From 5f2c71756ef90e71848aaecbd4e8f6005b510acb Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 12 Sep 2022 08:34:29 +0100 Subject: [PATCH 001/481] add basic realtime auth --- ably/__init__.py | 1 + ably/realtime/__init__.py | 0 ably/realtime/realtime.py | 34 ++++++++++++++++++++++++++++++++++ test/ably/realtimeauthtest.py | 21 +++++++++++++++++++++ 4 files changed, 56 insertions(+) create mode 100644 ably/realtime/__init__.py create mode 100644 ably/realtime/realtime.py create mode 100644 test/ably/realtimeauthtest.py diff --git a/ably/__init__.py b/ably/__init__.py index 578a1537..128e3d08 100644 --- a/ably/__init__.py +++ b/ably/__init__.py @@ -1,4 +1,5 @@ from ably.rest.rest import AblyRest +from ably.realtime.realtime import AblyRealtime from ably.rest.auth import Auth from ably.rest.push import Push from ably.types.capability import Capability diff --git a/ably/realtime/__init__.py b/ably/realtime/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py new file mode 100644 index 00000000..d9baff7c --- /dev/null +++ b/ably/realtime/realtime.py @@ -0,0 +1,34 @@ +import logging +from ably.rest.auth import Auth +from ably.types.options import Options + + +log = logging.getLogger(__name__) + +class AblyRealtime: + """Ably Realtime Client""" + + def __init__(self, key=None, **kwargs): + """Create an AblyRealtime instance. + + :Parameters: + **Credentials** + - `key`: a valid ably key string + """ + + if key is not None: + options = Options(key=key, **kwargs) + else: + options = Options(**kwargs) + + self.__auth = Auth(self, options) + + self.__options = options + + @property + def auth(self): + return self.__auth + + @property + def options(self): + return self.__options diff --git a/test/ably/realtimeauthtest.py b/test/ably/realtimeauthtest.py new file mode 100644 index 00000000..2c759481 --- /dev/null +++ b/test/ably/realtimeauthtest.py @@ -0,0 +1,21 @@ +import pytest +from ably import Auth, AblyRealtime +from ably.util.exceptions import AblyAuthException +from test.ably.utils import BaseAsyncTestCase + + +class TestRealtimeAuth(BaseAsyncTestCase): + async def setUp(self): + self.invalid_key = "some key" + self.valid_key_format = "Vjhddw.owt:R97sjjbdERJdjwer" + + def test_auth_with_correct_key_format(self): + key = self.valid_key_format.split(":") + ably = AblyRealtime(self.valid_key_format) + assert Auth.Method.BASIC == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" + assert ably.auth.auth_options.key_name == key[0] + assert ably.auth.auth_options.key_secret == key[1] + + def test_auth_incorrect_key_format(self): + with pytest.raises(AblyAuthException): + ably = AblyRealtime(self.invalid_key) \ No newline at end of file From bc1b8a492576825e9873196f5b3701c204fecb87 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 13 Sep 2022 09:28:14 +0100 Subject: [PATCH 002/481] create connection --- ably/realtime/connection.py | 33 +++++++++++++ ably/realtime/realtime.py | 11 ++++- poetry.lock | 94 ++++++++++++++++++++++++++++++------- pyproject.toml | 1 + 4 files changed, 119 insertions(+), 20 deletions(-) create mode 100644 ably/realtime/connection.py diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py new file mode 100644 index 00000000..0fab035a --- /dev/null +++ b/ably/realtime/connection.py @@ -0,0 +1,33 @@ +import asyncio +import websockets +import json + + +class RealtimeConnection: + def __init__(self, realtime): + self.options = realtime.options + self.__ably = realtime + + async def connect(self): + self.connected_future = asyncio.Future() + asyncio.create_task(self.connect_impl()) + return await self.connected_future + + async def connect_impl(self): + async with websockets.connect(f'wss://realtime.ably.io?key={self.ably.key}') as websocket: + self.websocket = websocket + task = asyncio.create_task(self.ws_read_loop()) + await task + + async def ws_read_loop(self): + while True: + raw = await self.websocket.recv() + msg = json.loads(raw) + action = msg['action'] + if (action == 4): # CONNECTED + self.connected_future.set_result(msg) + return msg + + @property + def ably(self): + return self.__ably diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index d9baff7c..475a728e 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -1,4 +1,5 @@ import logging +from ably.realtime.connection import RealtimeConnection from ably.rest.auth import Auth from ably.types.options import Options @@ -8,7 +9,7 @@ class AblyRealtime: """Ably Realtime Client""" - def __init__(self, key=None, **kwargs): + def __init__(self, key=None, token=None, token_details=None, **kwargs): """Create an AblyRealtime instance. :Parameters: @@ -22,8 +23,9 @@ def __init__(self, key=None, **kwargs): options = Options(**kwargs) self.__auth = Auth(self, options) - self.__options = options + self.key = key + self.__connection = RealtimeConnection(self) @property def auth(self): @@ -32,3 +34,8 @@ def auth(self): @property def options(self): return self.__options + + @property + def connection(self): + """Returns the channels container object""" + return self.__connection diff --git a/poetry.lock b/poetry.lock index 18243444..27c6cbb5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -12,8 +12,8 @@ sniffio = ">=1.1" typing-extensions = {version = "*", markers = "python_version < \"3.8\""} [package.extras] -doc = ["packaging", "sphinx-rtd-theme", "sphinx-autodoc-typehints (>=1.2.0)"] -test = ["coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "contextlib2", "uvloop (<0.15)", "mock (>=4)", "uvloop (>=0.15)"] +doc = ["packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["contextlib2", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (<0.15)", "uvloop (>=0.15)"] trio = ["trio (>=0.16)"] [[package]] @@ -33,10 +33,10 @@ optional = false python-versions = ">=3.5" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] -docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "cloudpickle"] +dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] +docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] +tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] +tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] [[package]] name = "certifi" @@ -161,8 +161,8 @@ rfc3986 = {version = ">=1.3,<2", extras = ["idna2008"]} sniffio = "*" [package.extras] -brotli = ["brotlicffi", "brotli"] -cli = ["click (>=8.0.0,<9.0.0)", "rich (>=10.0.0,<11.0.0)", "pygments (>=2.0.0,<3.0.0)"] +brotli = ["brotli", "brotlicffi"] +cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10.0.0,<11.0.0)"] http2 = ["h2 (>=3,<5)"] [[package]] @@ -194,9 +194,9 @@ typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] +docs = ["jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx"] perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] +testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] [[package]] name = "iniconfig" @@ -235,7 +235,7 @@ pbr = ">=0.11" six = ">=1.7" [package.extras] -docs = ["sphinx", "jinja2 (<2.7)", "Pygments (<2)", "sphinx (<1.3)"] +docs = ["Pygments (<2)", "jinja2 (<2.7)", "sphinx", "sphinx (<1.3)"] test = ["unittest2 (>=1.1.0)"] [[package]] @@ -285,8 +285,8 @@ python-versions = ">=3.6" importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} [package.extras] -testing = ["pytest-benchmark", "pytest"] -dev = ["tox", "pre-commit"] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] [[package]] name = "py" @@ -337,7 +337,7 @@ optional = false python-versions = ">=3.6.8" [package.extras] -diagrams = ["railroad-diagrams", "jinja2"] +diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" @@ -374,7 +374,7 @@ pytest = ">=4.6" toml = "*" [package.extras] -testing = ["fields", "hunter", "process-tests", "six", "pytest-xdist", "virtualenv"] +testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] [[package]] name = "pytest-flake8" @@ -482,6 +482,14 @@ category = "main" optional = false python-versions = ">=3.7" +[[package]] +name = "websockets" +version = "10.3" +description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" +category = "main" +optional = false +python-versions = ">=3.7" + [[package]] name = "zipp" version = "3.8.1" @@ -491,8 +499,8 @@ optional = false python-versions = ">=3.7" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "jaraco.tidelift (>=1.4)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] +docs = ["jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx"] +testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [extras] crypto = ["pycryptodome"] @@ -501,7 +509,7 @@ oldcrypto = ["pycrypto"] [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "0f5fa1c07bd116047635d4d34692f7f9ca1bb194988445ef61854469c2ce214d" +content-hash = "e7cc9e61014182ddade6f85e62d97616a52fb4a60a9a471e0b963eb1e82630aa" [metadata.files] anyio = [ @@ -805,6 +813,56 @@ typing-extensions = [ {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, ] +websockets = [ + {file = "websockets-10.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:661f641b44ed315556a2fa630239adfd77bd1b11cb0b9d96ed8ad90b0b1e4978"}, + {file = "websockets-10.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b529fdfa881b69fe563dbd98acce84f3e5a67df13de415e143ef053ff006d500"}, + {file = "websockets-10.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f351c7d7d92f67c0609329ab2735eee0426a03022771b00102816a72715bb00b"}, + {file = "websockets-10.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:379e03422178436af4f3abe0aa8f401aa77ae2487843738542a75faf44a31f0c"}, + {file = "websockets-10.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e904c0381c014b914136c492c8fa711ca4cced4e9b3d110e5e7d436d0fc289e8"}, + {file = "websockets-10.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e7e6f2d6fd48422071cc8a6f8542016f350b79cc782752de531577d35e9bd677"}, + {file = "websockets-10.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b9c77f0d1436ea4b4dc089ed8335fa141e6a251a92f75f675056dac4ab47a71e"}, + {file = "websockets-10.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e6fa05a680e35d0fcc1470cb070b10e6fe247af54768f488ed93542e71339d6f"}, + {file = "websockets-10.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2f94fa3ae454a63ea3a19f73b95deeebc9f02ba2d5617ca16f0bbdae375cda47"}, + {file = "websockets-10.3-cp310-cp310-win32.whl", hash = "sha256:6ed1d6f791eabfd9808afea1e068f5e59418e55721db8b7f3bfc39dc831c42ae"}, + {file = "websockets-10.3-cp310-cp310-win_amd64.whl", hash = "sha256:347974105bbd4ea068106ec65e8e8ebd86f28c19e529d115d89bd8cc5cda3079"}, + {file = "websockets-10.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fab7c640815812ed5f10fbee7abbf58788d602046b7bb3af9b1ac753a6d5e916"}, + {file = "websockets-10.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:994cdb1942a7a4c2e10098d9162948c9e7b235df755de91ca33f6e0481366fdb"}, + {file = "websockets-10.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:aad5e300ab32036eb3fdc350ad30877210e2f51bceaca83fb7fef4d2b6c72b79"}, + {file = "websockets-10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e49ea4c1a9543d2bd8a747ff24411509c29e4bdcde05b5b0895e2120cb1a761d"}, + {file = "websockets-10.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6ea6b300a6bdd782e49922d690e11c3669828fe36fc2471408c58b93b5535a98"}, + {file = "websockets-10.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:ef5ce841e102278c1c2e98f043db99d6755b1c58bde475516aef3a008ed7f28e"}, + {file = "websockets-10.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d1655a6fc7aecd333b079d00fb3c8132d18988e47f19740c69303bf02e9883c6"}, + {file = "websockets-10.3-cp37-cp37m-win32.whl", hash = "sha256:83e5ca0d5b743cde3d29fda74ccab37bdd0911f25bd4cdf09ff8b51b7b4f2fa1"}, + {file = "websockets-10.3-cp37-cp37m-win_amd64.whl", hash = "sha256:da4377904a3379f0c1b75a965fff23b28315bcd516d27f99a803720dfebd94d4"}, + {file = "websockets-10.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a1e15b230c3613e8ea82c9fc6941b2093e8eb939dd794c02754d33980ba81e36"}, + {file = "websockets-10.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:31564a67c3e4005f27815634343df688b25705cccb22bc1db621c781ddc64c69"}, + {file = "websockets-10.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c8d1d14aa0f600b5be363077b621b1b4d1eb3fbf90af83f9281cda668e6ff7fd"}, + {file = "websockets-10.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8fbd7d77f8aba46d43245e86dd91a8970eac4fb74c473f8e30e9c07581f852b2"}, + {file = "websockets-10.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:210aad7fdd381c52e58777560860c7e6110b6174488ef1d4b681c08b68bf7f8c"}, + {file = "websockets-10.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6075fd24df23133c1b078e08a9b04a3bc40b31a8def4ee0b9f2c8865acce913e"}, + {file = "websockets-10.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7f6d96fdb0975044fdd7953b35d003b03f9e2bcf85f2d2cf86285ece53e9f991"}, + {file = "websockets-10.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c7250848ce69559756ad0086a37b82c986cd33c2d344ab87fea596c5ac6d9442"}, + {file = "websockets-10.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:28dd20b938a57c3124028680dc1600c197294da5db4292c76a0b48efb3ed7f76"}, + {file = "websockets-10.3-cp38-cp38-win32.whl", hash = "sha256:54c000abeaff6d8771a4e2cef40900919908ea7b6b6a30eae72752607c6db559"}, + {file = "websockets-10.3-cp38-cp38-win_amd64.whl", hash = "sha256:7ab36e17af592eec5747c68ef2722a74c1a4a70f3772bc661079baf4ae30e40d"}, + {file = "websockets-10.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a141de3d5a92188234afa61653ed0bbd2dde46ad47b15c3042ffb89548e77094"}, + {file = "websockets-10.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:97bc9d41e69a7521a358f9b8e44871f6cdeb42af31815c17aed36372d4eec667"}, + {file = "websockets-10.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d6353ba89cfc657a3f5beabb3b69be226adbb5c6c7a66398e17809b0ce3c4731"}, + {file = "websockets-10.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec2b0ab7edc8cd4b0eb428b38ed89079bdc20c6bdb5f889d353011038caac2f9"}, + {file = "websockets-10.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:85506b3328a9e083cc0a0fb3ba27e33c8db78341b3eb12eb72e8afd166c36680"}, + {file = "websockets-10.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8af75085b4bc0b5c40c4a3c0e113fa95e84c60f4ed6786cbb675aeb1ee128247"}, + {file = "websockets-10.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:07cdc0a5b2549bcfbadb585ad8471ebdc7bdf91e32e34ae3889001c1c106a6af"}, + {file = "websockets-10.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:5b936bf552e4f6357f5727579072ff1e1324717902127ffe60c92d29b67b7be3"}, + {file = "websockets-10.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e4e08305bfd76ba8edab08dcc6496f40674f44eb9d5e23153efa0a35750337e8"}, + {file = "websockets-10.3-cp39-cp39-win32.whl", hash = "sha256:bb621ec2dbbbe8df78a27dbd9dd7919f9b7d32a73fafcb4d9252fc4637343582"}, + {file = "websockets-10.3-cp39-cp39-win_amd64.whl", hash = "sha256:51695d3b199cd03098ae5b42833006a0f43dc5418d3102972addc593a783bc02"}, + {file = "websockets-10.3-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:907e8247480f287aa9bbc9391bd6de23c906d48af54c8c421df84655eef66af7"}, + {file = "websockets-10.3-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b1359aba0ff810d5830d5ab8e2c4a02bebf98a60aa0124fb29aa78cfdb8031f"}, + {file = "websockets-10.3-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:93d5ea0b5da8d66d868b32c614d2b52d14304444e39e13a59566d4acb8d6e2e4"}, + {file = "websockets-10.3-pp37-pypy37_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7934e055fd5cd9dee60f11d16c8d79c4567315824bacb1246d0208a47eca9755"}, + {file = "websockets-10.3-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:3eda1cb7e9da1b22588cefff09f0951771d6ee9fa8dbe66f5ae04cc5f26b2b55"}, + {file = "websockets-10.3.tar.gz", hash = "sha256:fc06cc8073c8e87072138ba1e431300e2d408f054b27047d047b549455066ff4"}, +] zipp = [ {file = "zipp-3.8.1-py3-none-any.whl", hash = "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"}, {file = "zipp-3.8.1.tar.gz", hash = "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2"}, diff --git a/pyproject.toml b/pyproject.toml index 355ed464..51cb1353 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ h2 = "^4.0.0" # Optional dependencies pycrypto = { version = "^2.6.1", optional = true } pycryptodome = { version = "*", optional = true } +websockets = "^10.3" [tool.poetry.extras] oldcrypto = ["pycrypto"] From 4b321368bd3b421aa92ede128e7e4b9c41a75ff6 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 13 Sep 2022 15:27:26 +0100 Subject: [PATCH 003/481] update connection --- ably/realtime/connection.py | 9 +++++++-- ably/realtime/realtime.py | 4 +++- test/ably/realtimeauthtest.py | 29 ++++++++++++++++++++++++----- test/ably/restsetup.py | 2 ++ 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 0fab035a..c870bd15 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -1,6 +1,7 @@ import asyncio import websockets import json +from ably.util.exceptions import AblyAuthException class RealtimeConnection: @@ -13,8 +14,9 @@ async def connect(self): asyncio.create_task(self.connect_impl()) return await self.connected_future + async def connect_impl(self): - async with websockets.connect(f'wss://realtime.ably.io?key={self.ably.key}') as websocket: + async with websockets.connect(f'{self.options.realtime_host}?key={self.ably.key}') as websocket: self.websocket = websocket task = asyncio.create_task(self.ws_read_loop()) await task @@ -26,7 +28,10 @@ async def ws_read_loop(self): action = msg['action'] if (action == 4): # CONNECTED self.connected_future.set_result(msg) - return msg + if (action == 9): # ERROR + error = msg["error"] + self.connected_future.set_exception(AblyAuthException(error["message"], error["statusCode"], error["code"])) + @property def ably(self): diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 475a728e..059a43ec 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -1,4 +1,5 @@ import logging +import os from ably.realtime.connection import RealtimeConnection from ably.rest.auth import Auth from ably.types.options import Options @@ -9,7 +10,7 @@ class AblyRealtime: """Ably Realtime Client""" - def __init__(self, key=None, token=None, token_details=None, **kwargs): + def __init__(self, key=None, **kwargs): """Create an AblyRealtime instance. :Parameters: @@ -22,6 +23,7 @@ def __init__(self, key=None, token=None, token_details=None, **kwargs): else: options = Options(**kwargs) + options.realtime_host = os.environ.get('ABLY_REALTIME_HOST') self.__auth = Auth(self, options) self.__options = options self.key = key diff --git a/test/ably/realtimeauthtest.py b/test/ably/realtimeauthtest.py index 2c759481..f696569b 100644 --- a/test/ably/realtimeauthtest.py +++ b/test/ably/realtimeauthtest.py @@ -1,21 +1,40 @@ import pytest from ably import Auth, AblyRealtime from ably.util.exceptions import AblyAuthException +from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase class TestRealtimeAuth(BaseAsyncTestCase): async def setUp(self): - self.invalid_key = "some key" - self.valid_key_format = "Vjhddw.owt:R97sjjbdERJdjwer" + self.test_vars = await RestSetup.get_test_vars() + self.valid_key_format = "Vjdw.owt:R97sjjjwer" - def test_auth_with_correct_key_format(self): + async def test_auth_with_valid_key(self): + ably = AblyRealtime(self.test_vars["keys"][0]["key_str"]) + assert Auth.Method.BASIC == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" + assert ably.auth.auth_options.key_name == self.test_vars["keys"][0]['key_name'] + assert ably.auth.auth_options.key_secret == self.test_vars["keys"][0]['key_secret'] + + async def test_auth_incorrect_key(self): + with pytest.raises(AblyAuthException): + AblyRealtime("some invalid key") + + async def test_auth_with_valid_key_format(self): key = self.valid_key_format.split(":") ably = AblyRealtime(self.valid_key_format) assert Auth.Method.BASIC == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" assert ably.auth.auth_options.key_name == key[0] assert ably.auth.auth_options.key_secret == key[1] - def test_auth_incorrect_key_format(self): + # async def test_auth_connection(self): + # ably = AblyRealtime(self.test_vars["keys"][0]["key_str"]) + # conn = await ably.connection.connect() + # assert conn["action"] == 4 + # assert "connectionDetails" in conn + + async def test_auth_invalid_key(self): + ably = AblyRealtime(self.valid_key_format) with pytest.raises(AblyAuthException): - ably = AblyRealtime(self.invalid_key) \ No newline at end of file + await ably.connection.connect() + diff --git a/test/ably/restsetup.py b/test/ably/restsetup.py index 3c681005..9babdd05 100644 --- a/test/ably/restsetup.py +++ b/test/ably/restsetup.py @@ -14,6 +14,7 @@ tls = (os.environ.get('ABLY_TLS') or "true").lower() == "true" host = os.environ.get('ABLY_HOST', 'sandbox-rest.ably.io') +realtime_host = os.environ.get('ABLY_REALTIME_HOST', 'sandbox-realtime.ably.io') environment = os.environ.get('ABLY_ENV') port = 80 @@ -51,6 +52,7 @@ async def get_test_vars(sender=None): "tls_port": tls_port, "tls": tls, "environment": environment, + "realtime_host": realtime_host, "keys": [{ "key_name": "%s.%s" % (app_id, k.get("id", "")), "key_secret": k.get("value", ""), From 91f3f8fc9a48167048a7340c9c99bf8863694b11 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 16:26:57 +0100 Subject: [PATCH 004/481] Add get_ably_realtime test helper --- test/ably/restsetup.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/test/ably/restsetup.py b/test/ably/restsetup.py index 9babdd05..efab592d 100644 --- a/test/ably/restsetup.py +++ b/test/ably/restsetup.py @@ -6,6 +6,7 @@ from ably.types.capability import Capability from ably.types.options import Options from ably.util.exceptions import AblyException +from ably.realtime.realtime import AblyRealtime log = logging.getLogger(__name__) @@ -14,7 +15,7 @@ tls = (os.environ.get('ABLY_TLS') or "true").lower() == "true" host = os.environ.get('ABLY_HOST', 'sandbox-rest.ably.io') -realtime_host = os.environ.get('ABLY_REALTIME_HOST', 'sandbox-realtime.ably.io') +realtime_host = 'sandbox-realtime.ably.io' environment = os.environ.get('ABLY_ENV') port = 80 @@ -81,6 +82,20 @@ async def get_ably_rest(cls, **kw): options.update(kw) return AblyRest(**options) + @classmethod + async def get_ably_realtime(cls, **kw): + test_vars = await RestSetup.get_test_vars() + options = { + 'key': test_vars["keys"][0]["key_str"], + 'realtime_host': realtime_host, + 'port': test_vars["port"], + 'tls_port': test_vars["tls_port"], + 'tls': test_vars["tls"], + 'environment': test_vars["environment"], + } + options.update(kw) + return AblyRealtime(**options) + @classmethod async def clear_test_vars(cls): test_vars = RestSetup.__test_vars From ef063949d17b06a34efa2707d845eb9b0c20503a Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 16:27:21 +0100 Subject: [PATCH 005/481] Use configured realtime_host for websocket connections --- ably/realtime/connection.py | 3 +-- ably/realtime/realtime.py | 2 -- ably/types/options.py | 3 +++ 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index c870bd15..bedfda18 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -14,9 +14,8 @@ async def connect(self): asyncio.create_task(self.connect_impl()) return await self.connected_future - async def connect_impl(self): - async with websockets.connect(f'{self.options.realtime_host}?key={self.ably.key}') as websocket: + async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}') as websocket: self.websocket = websocket task = asyncio.create_task(self.ws_read_loop()) await task diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 059a43ec..9f44f7ff 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -1,5 +1,4 @@ import logging -import os from ably.realtime.connection import RealtimeConnection from ably.rest.auth import Auth from ably.types.options import Options @@ -23,7 +22,6 @@ def __init__(self, key=None, **kwargs): else: options = Options(**kwargs) - options.realtime_host = os.environ.get('ABLY_REALTIME_HOST') self.__auth = Auth(self, options) self.__options = options self.key = key diff --git a/ably/types/options.py b/ably/types/options.py index 38ef8ed9..441d87b6 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -27,6 +27,9 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, from ably import api_version idempotent_rest_publishing = api_version >= '1.2' + if realtime_host is None: + realtime_host = Defaults.realtime_host + self.__client_id = client_id self.__log_level = log_level self.__tls = tls From 82a42f47aa860563e53793540bfaeec98d983082 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 16:27:33 +0100 Subject: [PATCH 006/481] Update tests to use realtime helper method --- test/ably/realtimeauthtest.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/ably/realtimeauthtest.py b/test/ably/realtimeauthtest.py index f696569b..626eb12d 100644 --- a/test/ably/realtimeauthtest.py +++ b/test/ably/realtimeauthtest.py @@ -8,21 +8,21 @@ class TestRealtimeAuth(BaseAsyncTestCase): async def setUp(self): self.test_vars = await RestSetup.get_test_vars() - self.valid_key_format = "Vjdw.owt:R97sjjjwer" + self.valid_key_format = "api:key" async def test_auth_with_valid_key(self): - ably = AblyRealtime(self.test_vars["keys"][0]["key_str"]) + ably = await RestSetup.get_ably_realtime(key=self.test_vars["keys"][0]["key_str"]) assert Auth.Method.BASIC == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" assert ably.auth.auth_options.key_name == self.test_vars["keys"][0]['key_name'] assert ably.auth.auth_options.key_secret == self.test_vars["keys"][0]['key_secret'] async def test_auth_incorrect_key(self): with pytest.raises(AblyAuthException): - AblyRealtime("some invalid key") + await RestSetup.get_ably_realtime(key="some invalid key") async def test_auth_with_valid_key_format(self): key = self.valid_key_format.split(":") - ably = AblyRealtime(self.valid_key_format) + ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) assert Auth.Method.BASIC == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" assert ably.auth.auth_options.key_name == key[0] assert ably.auth.auth_options.key_secret == key[1] @@ -34,7 +34,6 @@ async def test_auth_with_valid_key_format(self): # assert "connectionDetails" in conn async def test_auth_invalid_key(self): - ably = AblyRealtime(self.valid_key_format) + ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) with pytest.raises(AblyAuthException): await ably.connection.connect() - From 8daa1075142b82392e83b0f5f9f484779e1de1bd Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 16:29:37 +0100 Subject: [PATCH 007/481] Add Realtime.connect method --- ably/realtime/realtime.py | 8 ++++++-- test/ably/realtimeauthtest.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 9f44f7ff..71dc5b38 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -6,6 +6,7 @@ log = logging.getLogger(__name__) + class AblyRealtime: """Ably Realtime Client""" @@ -26,7 +27,10 @@ def __init__(self, key=None, **kwargs): self.__options = options self.key = key self.__connection = RealtimeConnection(self) - + + async def connect(self): + await self.connection.connect() + @property def auth(self): return self.__auth @@ -34,7 +38,7 @@ def auth(self): @property def options(self): return self.__options - + @property def connection(self): """Returns the channels container object""" diff --git a/test/ably/realtimeauthtest.py b/test/ably/realtimeauthtest.py index 626eb12d..e84b5703 100644 --- a/test/ably/realtimeauthtest.py +++ b/test/ably/realtimeauthtest.py @@ -36,4 +36,4 @@ async def test_auth_with_valid_key_format(self): async def test_auth_invalid_key(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) with pytest.raises(AblyAuthException): - await ably.connection.connect() + await ably.connect() From 1cad09725ef3fbb5c5fa747529f352b38b8551e4 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 16:36:01 +0100 Subject: [PATCH 008/481] Make Realtime.connect return None when successful --- ably/realtime/connection.py | 4 ++-- test/ably/realtimeauthtest.py | 8 +++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index bedfda18..bf93dfbf 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -12,7 +12,7 @@ def __init__(self, realtime): async def connect(self): self.connected_future = asyncio.Future() asyncio.create_task(self.connect_impl()) - return await self.connected_future + await self.connected_future async def connect_impl(self): async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}') as websocket: @@ -26,7 +26,7 @@ async def ws_read_loop(self): msg = json.loads(raw) action = msg['action'] if (action == 4): # CONNECTED - self.connected_future.set_result(msg) + self.connected_future.set_result(None) if (action == 9): # ERROR error = msg["error"] self.connected_future.set_exception(AblyAuthException(error["message"], error["statusCode"], error["code"])) diff --git a/test/ably/realtimeauthtest.py b/test/ably/realtimeauthtest.py index e84b5703..7297c019 100644 --- a/test/ably/realtimeauthtest.py +++ b/test/ably/realtimeauthtest.py @@ -27,11 +27,9 @@ async def test_auth_with_valid_key_format(self): assert ably.auth.auth_options.key_name == key[0] assert ably.auth.auth_options.key_secret == key[1] - # async def test_auth_connection(self): - # ably = AblyRealtime(self.test_vars["keys"][0]["key_str"]) - # conn = await ably.connection.connect() - # assert conn["action"] == 4 - # assert "connectionDetails" in conn + async def test_auth_connection(self): + ably = await RestSetup.get_ably_realtime() + await ably.connect() async def test_auth_invalid_key(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) From 3d1a83527bfc64d27cc3b4d4f4abee776e11f718 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 16:41:22 +0100 Subject: [PATCH 009/481] Add Connection.state --- ably/realtime/connection.py | 15 ++++++++++++++- test/ably/realtimeauthtest.py | 3 +++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index bf93dfbf..18485afe 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -2,17 +2,27 @@ import websockets import json from ably.util.exceptions import AblyAuthException +from enum import Enum + + +class ConnectionState(Enum): + INITIALIZED = 'initialized' + CONNECTING = 'connecting' + CONNECTED = 'connected' class RealtimeConnection: def __init__(self, realtime): self.options = realtime.options self.__ably = realtime + self.__state = ConnectionState.INITIALIZED async def connect(self): + self.__state = ConnectionState.CONNECTING self.connected_future = asyncio.Future() asyncio.create_task(self.connect_impl()) await self.connected_future + self.__state = ConnectionState.CONNECTED async def connect_impl(self): async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}') as websocket: @@ -30,8 +40,11 @@ async def ws_read_loop(self): if (action == 9): # ERROR error = msg["error"] self.connected_future.set_exception(AblyAuthException(error["message"], error["statusCode"], error["code"])) - @property def ably(self): return self.__ably + + @property + def state(self): + return self.__state diff --git a/test/ably/realtimeauthtest.py b/test/ably/realtimeauthtest.py index 7297c019..bda9a530 100644 --- a/test/ably/realtimeauthtest.py +++ b/test/ably/realtimeauthtest.py @@ -1,3 +1,4 @@ +from ably.realtime.connection import ConnectionState import pytest from ably import Auth, AblyRealtime from ably.util.exceptions import AblyAuthException @@ -29,7 +30,9 @@ async def test_auth_with_valid_key_format(self): async def test_auth_connection(self): ably = await RestSetup.get_ably_realtime() + assert ably.connection.state == ConnectionState.INITIALIZED await ably.connect() + assert ably.connection.state == ConnectionState.CONNECTED async def test_auth_invalid_key(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) From d5b7d0bdeb43b7b88d8e6f0dd5a679f6499cc684 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 16:48:39 +0100 Subject: [PATCH 010/481] Add Realtime.close method --- ably/realtime/connection.py | 7 +++++++ ably/realtime/realtime.py | 3 +++ test/ably/realtimeauthtest.py | 3 +++ 3 files changed, 13 insertions(+) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 18485afe..baa24922 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -9,6 +9,8 @@ class ConnectionState(Enum): INITIALIZED = 'initialized' CONNECTING = 'connecting' CONNECTED = 'connected' + CLOSING = 'closing' + CLOSED = 'closed' class RealtimeConnection: @@ -24,6 +26,11 @@ async def connect(self): await self.connected_future self.__state = ConnectionState.CONNECTED + async def close(self): + self.__state = ConnectionState.CLOSING + await self.websocket.close() + self.__state = ConnectionState.CLOSED + async def connect_impl(self): async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}') as websocket: self.websocket = websocket diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 71dc5b38..25f57a2a 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -31,6 +31,9 @@ def __init__(self, key=None, **kwargs): async def connect(self): await self.connection.connect() + async def close(self): + await self.connection.close() + @property def auth(self): return self.__auth diff --git a/test/ably/realtimeauthtest.py b/test/ably/realtimeauthtest.py index bda9a530..e9110b0d 100644 --- a/test/ably/realtimeauthtest.py +++ b/test/ably/realtimeauthtest.py @@ -33,8 +33,11 @@ async def test_auth_connection(self): assert ably.connection.state == ConnectionState.INITIALIZED await ably.connect() assert ably.connection.state == ConnectionState.CONNECTED + await ably.close() + assert ably.connection.state == ConnectionState.CLOSED async def test_auth_invalid_key(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) with pytest.raises(AblyAuthException): await ably.connect() + await ably.close() From 615a87329a59dc60da93076e2821c04b71d470d1 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 16:49:37 +0100 Subject: [PATCH 011/481] Move realimteauthtest.py to realtimeinit_test.py --- test/ably/{realtimeauthtest.py => realtimeinit_test.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/ably/{realtimeauthtest.py => realtimeinit_test.py} (100%) diff --git a/test/ably/realtimeauthtest.py b/test/ably/realtimeinit_test.py similarity index 100% rename from test/ably/realtimeauthtest.py rename to test/ably/realtimeinit_test.py From cbe3a07bb5c756019a520dff4e3d857b556ff358 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 16:52:00 +0100 Subject: [PATCH 012/481] Move connection tests to new file --- test/ably/realtimeconnection_test.py | 25 +++++++++++++++++++++++++ test/ably/realtimeinit_test.py | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 test/ably/realtimeconnection_test.py diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py new file mode 100644 index 00000000..00e32759 --- /dev/null +++ b/test/ably/realtimeconnection_test.py @@ -0,0 +1,25 @@ +from ably.realtime.connection import ConnectionState +import pytest +from ably.util.exceptions import AblyAuthException +from test.ably.restsetup import RestSetup +from test.ably.utils import BaseAsyncTestCase + + +class TestRealtimeAuth(BaseAsyncTestCase): + async def setUp(self): + self.test_vars = await RestSetup.get_test_vars() + self.valid_key_format = "api:key" + + async def test_auth_connection(self): + ably = await RestSetup.get_ably_realtime() + assert ably.connection.state == ConnectionState.INITIALIZED + await ably.connect() + assert ably.connection.state == ConnectionState.CONNECTED + await ably.close() + assert ably.connection.state == ConnectionState.CLOSED + + async def test_auth_invalid_key(self): + ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) + with pytest.raises(AblyAuthException): + await ably.connect() + await ably.close() diff --git a/test/ably/realtimeinit_test.py b/test/ably/realtimeinit_test.py index e9110b0d..a85f9576 100644 --- a/test/ably/realtimeinit_test.py +++ b/test/ably/realtimeinit_test.py @@ -1,6 +1,6 @@ from ably.realtime.connection import ConnectionState import pytest -from ably import Auth, AblyRealtime +from ably import Auth from ably.util.exceptions import AblyAuthException from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase From 16fabf295162295d326c228de758847253b81119 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 17:01:41 +0100 Subject: [PATCH 013/481] Ensure connected_future is resolved once --- ably/realtime/connection.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index baa24922..3e4562f6 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -1,9 +1,12 @@ +import logging import asyncio import websockets import json from ably.util.exceptions import AblyAuthException from enum import Enum +log = logging.getLogger(__name__) + class ConnectionState(Enum): INITIALIZED = 'initialized' @@ -18,6 +21,8 @@ def __init__(self, realtime): self.options = realtime.options self.__ably = realtime self.__state = ConnectionState.INITIALIZED + self.connected_future = None + self.websocket = None async def connect(self): self.__state = ConnectionState.CONNECTING @@ -42,11 +47,18 @@ async def ws_read_loop(self): raw = await self.websocket.recv() msg = json.loads(raw) action = msg['action'] - if (action == 4): # CONNECTED - self.connected_future.set_result(None) - if (action == 9): # ERROR + if action == 4: # CONNECTED + if self.connected_future: + self.connected_future.set_result(None) + self.connected_future = None + else: + log.warn('CONNECTED message receieved but connected_future not set') + if action == 9: # ERROR error = msg["error"] - self.connected_future.set_exception(AblyAuthException(error["message"], error["statusCode"], error["code"])) + if error['nonfatal'] is False: + if self.connected_future: + self.connected_future.set_exception(AblyAuthException(error["message"], error["statusCode"], error["code"])) + self.connected_future = None @property def ably(self): From c479e1240247cf2fed08176b997b927b83884d8f Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 17:08:40 +0100 Subject: [PATCH 014/481] Add some state validation to Connection methods --- ably/realtime/connection.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 3e4562f6..bd3b21df 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -25,15 +25,27 @@ def __init__(self, realtime): self.websocket = None async def connect(self): - self.__state = ConnectionState.CONNECTING - self.connected_future = asyncio.Future() - asyncio.create_task(self.connect_impl()) - await self.connected_future - self.__state = ConnectionState.CONNECTED + if self.__state == ConnectionState.CONNECTED: + return + + if self.__state == ConnectionState.CONNECTING: + if self.connected_future is None: + log.fatal('Connection state is CONNECTING but connected_future does not exits') + return + await self.connected_future + else: + self.__state = ConnectionState.CONNECTING + self.connected_future = asyncio.Future() + asyncio.create_task(self.connect_impl()) + await self.connected_future + self.__state = ConnectionState.CONNECTED async def close(self): self.__state = ConnectionState.CLOSING - await self.websocket.close() + if self.websocket: + await self.websocket.close() + else: + log.warn('Connection.closed called while connection already closed') self.__state = ConnectionState.CLOSED async def connect_impl(self): From 4d5f00f2e67d4447f47527933224a775b81b2535 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 17:14:34 +0100 Subject: [PATCH 015/481] Add tests for transient connection states --- test/ably/realtimeconnection_test.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 00e32759..134c1f9d 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -1,3 +1,4 @@ +import asyncio from ably.realtime.connection import ConnectionState import pytest from ably.util.exceptions import AblyAuthException @@ -18,6 +19,22 @@ async def test_auth_connection(self): await ably.close() assert ably.connection.state == ConnectionState.CLOSED + async def test_connecting_state(self): + ably = await RestSetup.get_ably_realtime() + task = asyncio.create_task(ably.connect()) + await asyncio.sleep(0) + assert ably.connection.state == ConnectionState.CONNECTING + await task + await ably.close() + + async def test_closing_state(self): + ably = await RestSetup.get_ably_realtime() + await ably.connect() + task = asyncio.create_task(ably.close()) + await asyncio.sleep(0) + assert ably.connection.state == ConnectionState.CLOSING + await task + async def test_auth_invalid_key(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) with pytest.raises(AblyAuthException): From 7e73898eb0dffaafbfbe94a8731752459ab2dc15 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 14 Sep 2022 08:29:34 +0100 Subject: [PATCH 016/481] add api key check --- ably/realtime/connection.py | 3 ++- ably/realtime/realtime.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index bd3b21df..a9e24341 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -69,7 +69,8 @@ async def ws_read_loop(self): error = msg["error"] if error['nonfatal'] is False: if self.connected_future: - self.connected_future.set_exception(AblyAuthException(error["message"], error["statusCode"], error["code"])) + self.connected_future.set_exception( + AblyAuthException(error["message"], error["statusCode"], error["code"])) self.connected_future = None @property diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 25f57a2a..36cf1cbe 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -21,7 +21,7 @@ def __init__(self, key=None, **kwargs): if key is not None: options = Options(key=key, **kwargs) else: - options = Options(**kwargs) + raise ValueError("Key is missing. Provide an API key") self.__auth = Auth(self, options) self.__options = options @@ -44,5 +44,5 @@ def options(self): @property def connection(self): - """Returns the channels container object""" + """Establish realtime connection""" return self.__connection From b8f3c9adac3f20978cf5300a030fd1da02d34f8e Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 14 Sep 2022 14:15:28 +0100 Subject: [PATCH 017/481] Change some connection fields to private --- ably/realtime/connection.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index a9e24341..3a154bae 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -21,57 +21,57 @@ def __init__(self, realtime): self.options = realtime.options self.__ably = realtime self.__state = ConnectionState.INITIALIZED - self.connected_future = None - self.websocket = None + self.__connected_future = None + self.__websocket = None async def connect(self): if self.__state == ConnectionState.CONNECTED: return if self.__state == ConnectionState.CONNECTING: - if self.connected_future is None: + if self.__connected_future is None: log.fatal('Connection state is CONNECTING but connected_future does not exits') return - await self.connected_future + await self.__connected_future else: self.__state = ConnectionState.CONNECTING - self.connected_future = asyncio.Future() + self.__connected_future = asyncio.Future() asyncio.create_task(self.connect_impl()) - await self.connected_future + await self.__connected_future self.__state = ConnectionState.CONNECTED async def close(self): self.__state = ConnectionState.CLOSING - if self.websocket: - await self.websocket.close() + if self.__websocket: + await self.__websocket.close() else: log.warn('Connection.closed called while connection already closed') self.__state = ConnectionState.CLOSED async def connect_impl(self): async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}') as websocket: - self.websocket = websocket + self.__websocket = websocket task = asyncio.create_task(self.ws_read_loop()) await task async def ws_read_loop(self): while True: - raw = await self.websocket.recv() + raw = await self.__websocket.recv() msg = json.loads(raw) action = msg['action'] if action == 4: # CONNECTED - if self.connected_future: - self.connected_future.set_result(None) - self.connected_future = None + if self.__connected_future: + self.__connected_future.set_result(None) + self.__connected_future = None else: log.warn('CONNECTED message receieved but connected_future not set') if action == 9: # ERROR error = msg["error"] if error['nonfatal'] is False: - if self.connected_future: - self.connected_future.set_exception( + if self.__connected_future: + self.__connected_future.set_exception( AblyAuthException(error["message"], error["statusCode"], error["code"])) - self.connected_future = None + self.__connected_future = None @property def ably(self): From fce2edf7028f6372dfdfc1edb36fed16666697ec Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 14 Sep 2022 14:18:02 +0100 Subject: [PATCH 018/481] Add failed ConnectionState --- ably/realtime/connection.py | 2 ++ test/ably/realtimeconnection_test.py | 1 + 2 files changed, 3 insertions(+) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 3a154bae..d5c79f0d 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -14,6 +14,7 @@ class ConnectionState(Enum): CONNECTED = 'connected' CLOSING = 'closing' CLOSED = 'closed' + FAILED = 'failed' class RealtimeConnection: @@ -68,6 +69,7 @@ async def ws_read_loop(self): if action == 9: # ERROR error = msg["error"] if error['nonfatal'] is False: + self.__state = ConnectionState.FAILED if self.__connected_future: self.__connected_future.set_exception( AblyAuthException(error["message"], error["statusCode"], error["code"])) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 134c1f9d..929161a7 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -39,4 +39,5 @@ async def test_auth_invalid_key(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) with pytest.raises(AblyAuthException): await ably.connect() + assert ably.connection.state == ConnectionState.FAILED await ably.close() From 940d0cc49c6754f3c48d6479e9a4e38eac1ebfb8 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 14 Sep 2022 15:32:22 +0100 Subject: [PATCH 019/481] change base type of ProtocolMessageAction to IntEnum fix hanging test --- ably/realtime/connection.py | 13 +++++++++---- ably/realtime/realtime.py | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index d5c79f0d..6cf3490f 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -3,7 +3,7 @@ import websockets import json from ably.util.exceptions import AblyAuthException -from enum import Enum +from enum import Enum, IntEnum log = logging.getLogger(__name__) @@ -17,6 +17,11 @@ class ConnectionState(Enum): FAILED = 'failed' +class ProtocolMessageAction(IntEnum): + CONNECTED = 4 + ERROR = 9 + + class RealtimeConnection: def __init__(self, realtime): self.options = realtime.options @@ -60,13 +65,13 @@ async def ws_read_loop(self): raw = await self.__websocket.recv() msg = json.loads(raw) action = msg['action'] - if action == 4: # CONNECTED + if action == ProtocolMessageAction.CONNECTED: # CONNECTED if self.__connected_future: self.__connected_future.set_result(None) self.__connected_future = None else: - log.warn('CONNECTED message receieved but connected_future not set') - if action == 9: # ERROR + log.warn('CONNECTED message received but connected_future not set') + if action == ProtocolMessageAction.ERROR: # ERROR error = msg["error"] if error['nonfatal'] is False: self.__state = ConnectionState.FAILED diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 36cf1cbe..de70e41c 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -21,7 +21,7 @@ def __init__(self, key=None, **kwargs): if key is not None: options = Options(key=key, **kwargs) else: - raise ValueError("Key is missing. Provide an API key") + raise ValueError("Key is missing. Provide an API key.") self.__auth = Auth(self, options) self.__options = options From ce26cf622d8d0f39729a8409768ea3a208920745 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Thu, 15 Sep 2022 16:25:34 +0100 Subject: [PATCH 020/481] send ably-agent header in realtime connection fix linting error --- ably/realtime/connection.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 6cf3490f..0ec73e67 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -2,6 +2,7 @@ import asyncio import websockets import json +from ably.http.httputils import HttpUtils from ably.util.exceptions import AblyAuthException from enum import Enum, IntEnum @@ -55,7 +56,9 @@ async def close(self): self.__state = ConnectionState.CLOSED async def connect_impl(self): - async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}') as websocket: + headers = HttpUtils.default_get_headers() + async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}', + extra_headers=headers) as websocket: self.__websocket = websocket task = asyncio.create_task(self.ws_read_loop()) await task From 102d884c33b05e3c2f5e99eb520de74aeccad5a7 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Thu, 15 Sep 2022 18:11:39 +0100 Subject: [PATCH 021/481] refactor default header --- ably/http/httputils.py | 12 ++++++++---- ably/realtime/connection.py | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ably/http/httputils.py b/ably/http/httputils.py index 0517f969..53a583a1 100644 --- a/ably/http/httputils.py +++ b/ably/http/httputils.py @@ -15,10 +15,7 @@ class HttpUtils: @staticmethod def default_get_headers(binary=False): - headers = { - "X-Ably-Version": ably.api_version, - "Ably-Agent": 'ably-python/%s python/%s' % (ably.lib_version, platform.python_version()) - } + headers = HttpUtils.default_headers() if binary: headers["Accept"] = HttpUtils.mime_types['binary'] else: @@ -36,3 +33,10 @@ def get_host_header(host): return { 'Host': host, } + + @staticmethod + def default_headers(): + return { + "X-Ably-Version": ably.api_version, + "Ably-Agent": 'ably-python/%s python/%s' % (ably.lib_version, platform.python_version()) + } diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 0ec73e67..0e5cabb8 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -56,7 +56,7 @@ async def close(self): self.__state = ConnectionState.CLOSED async def connect_impl(self): - headers = HttpUtils.default_get_headers() + headers = HttpUtils.default_headers() async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}', extra_headers=headers) as websocket: self.__websocket = websocket From 23eb3c8ddf24b9cafdde1efe1c4e550aba07e777 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Thu, 22 Sep 2022 10:56:35 +0100 Subject: [PATCH 022/481] send close protocol message to ably --- ably/realtime/connection.py | 21 ++++++++++++++++++--- test/ably/realtimeconnection_test.py | 4 ++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 0e5cabb8..8af853c1 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -21,6 +21,8 @@ class ConnectionState(Enum): class ProtocolMessageAction(IntEnum): CONNECTED = 4 ERROR = 9 + CLOSE = 7 + CLOSED = 8 class RealtimeConnection: @@ -29,6 +31,7 @@ def __init__(self, realtime): self.__ably = realtime self.__state = ConnectionState.INITIALIZED self.__connected_future = None + self.__closed_future = None self.__websocket = None async def connect(self): @@ -49,12 +52,20 @@ async def connect(self): async def close(self): self.__state = ConnectionState.CLOSING - if self.__websocket: - await self.__websocket.close() + self.__closed_future = asyncio.Future() + if self.__websocket and self.__state == ConnectionState.CONNECTED: + task = asyncio.create_task(self.close_connection()) + await task else: - log.warn('Connection.closed called while connection already closed') + log.warn('Connection.closed called while connection already closed or not established') self.__state = ConnectionState.CLOSED + async def close_connection(self): + await self.sendProtocolMessage({"action": ProtocolMessageAction.CLOSE}) + + async def sendProtocolMessage(self, protocolMessage): + await self.__websocket.send(json.dumps(protocolMessage)) + async def connect_impl(self): headers = HttpUtils.default_headers() async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}', @@ -82,6 +93,10 @@ async def ws_read_loop(self): self.__connected_future.set_exception( AblyAuthException(error["message"], error["statusCode"], error["code"])) self.__connected_future = None + if action == ProtocolMessageAction.CLOSED: + await self.__websocket.close() + self.__closed_future.set_result(None) + break @property def ably(self): diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 929161a7..203cc0f5 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -27,12 +27,12 @@ async def test_connecting_state(self): await task await ably.close() - async def test_closing_state(self): + async def test_closed_state(self): ably = await RestSetup.get_ably_realtime() await ably.connect() task = asyncio.create_task(ably.close()) await asyncio.sleep(0) - assert ably.connection.state == ConnectionState.CLOSING + assert ably.connection.state == ConnectionState.CLOSED await task async def test_auth_invalid_key(self): From dcdea9dbb7fc93691cd8fb77e30554735367ea6c Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 26 Sep 2022 11:31:19 +0100 Subject: [PATCH 023/481] review: await closed future --- ably/realtime/connection.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 8af853c1..3a5a21a4 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -54,13 +54,13 @@ async def close(self): self.__state = ConnectionState.CLOSING self.__closed_future = asyncio.Future() if self.__websocket and self.__state == ConnectionState.CONNECTED: - task = asyncio.create_task(self.close_connection()) - await task + await self.send_close_message() + await self.__closed_future else: log.warn('Connection.closed called while connection already closed or not established') self.__state = ConnectionState.CLOSED - async def close_connection(self): + async def send_close_message(self): await self.sendProtocolMessage({"action": ProtocolMessageAction.CLOSE}) async def sendProtocolMessage(self, protocolMessage): From e7576e7f078ee0f32ec99a9b060c40d000934401 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 26 Sep 2022 23:23:20 +0100 Subject: [PATCH 024/481] refactor: extract Connection internals to ConnectionManager --- ably/realtime/connection.py | 36 ++++++++++++++++++++++++++++++------ ably/realtime/realtime.py | 4 ++-- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 3a5a21a4..0699e2f6 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -25,7 +25,27 @@ class ProtocolMessageAction(IntEnum): CLOSED = 8 -class RealtimeConnection: +class Connection: + def __init__(self, realtime): + self.__realtime = realtime + self.__connection_manager = ConnectionManager(realtime) + self.__state = ConnectionState.INITIALIZED + + async def connect(self): + await self.__connection_manager.connect() + + async def close(self): + await self.__connection_manager.close() + + def on_state_update(self, state): + self.__state = state + + @property + def state(self): + return self.__state + + +class ConnectionManager: def __init__(self, realtime): self.options = realtime.options self.__ably = realtime @@ -34,6 +54,10 @@ def __init__(self, realtime): self.__closed_future = None self.__websocket = None + def enact_state_change(self, state): + self.__state = state + self.ably.connection.on_state_update(state) + async def connect(self): if self.__state == ConnectionState.CONNECTED: return @@ -44,21 +68,21 @@ async def connect(self): return await self.__connected_future else: - self.__state = ConnectionState.CONNECTING + self.enact_state_change(ConnectionState.CONNECTING) self.__connected_future = asyncio.Future() asyncio.create_task(self.connect_impl()) await self.__connected_future - self.__state = ConnectionState.CONNECTED + self.enact_state_change(ConnectionState.CONNECTED) async def close(self): - self.__state = ConnectionState.CLOSING + self.enact_state_change(ConnectionState.CLOSING) self.__closed_future = asyncio.Future() if self.__websocket and self.__state == ConnectionState.CONNECTED: await self.send_close_message() await self.__closed_future else: log.warn('Connection.closed called while connection already closed or not established') - self.__state = ConnectionState.CLOSED + self.enact_state_change(ConnectionState.CLOSED) async def send_close_message(self): await self.sendProtocolMessage({"action": ProtocolMessageAction.CLOSE}) @@ -88,7 +112,7 @@ async def ws_read_loop(self): if action == ProtocolMessageAction.ERROR: # ERROR error = msg["error"] if error['nonfatal'] is False: - self.__state = ConnectionState.FAILED + self.enact_state_change(ConnectionState.FAILED) if self.__connected_future: self.__connected_future.set_exception( AblyAuthException(error["message"], error["statusCode"], error["code"])) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index de70e41c..4f62d576 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -1,5 +1,5 @@ import logging -from ably.realtime.connection import RealtimeConnection +from ably.realtime.connection import Connection from ably.rest.auth import Auth from ably.types.options import Options @@ -26,7 +26,7 @@ def __init__(self, key=None, **kwargs): self.__auth = Auth(self, options) self.__options = options self.key = key - self.__connection = RealtimeConnection(self) + self.__connection = Connection(self) async def connect(self): await self.connection.connect() From 0bff8343fe768395f80f4aba993a04e2b3afa21b Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 26 Sep 2022 23:33:17 +0100 Subject: [PATCH 025/481] chore: add loop option --- ably/realtime/realtime.py | 11 +++++++++-- ably/types/options.py | 10 +++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 4f62d576..e22b1da9 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -1,4 +1,5 @@ import logging +import asyncio from ably.realtime.connection import Connection from ably.rest.auth import Auth from ably.types.options import Options @@ -10,7 +11,7 @@ class AblyRealtime: """Ably Realtime Client""" - def __init__(self, key=None, **kwargs): + def __init__(self, key=None, loop=None, **kwargs): """Create an AblyRealtime instance. :Parameters: @@ -18,8 +19,14 @@ def __init__(self, key=None, **kwargs): - `key`: a valid ably key string """ + if loop is None: + try: + loop = asyncio.get_running_loop() + except RuntimeError: + log.warning('Realtime client created outside event loop') + if key is not None: - options = Options(key=key, **kwargs) + options = Options(key=key, loop=loop, **kwargs) else: raise ValueError("Key is missing. Provide an API key.") diff --git a/ably/types/options.py b/ably/types/options.py index 441d87b6..9a4791e0 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -1,9 +1,12 @@ import random import warnings +import logging from ably.transport.defaults import Defaults from ably.types.authoptions import AuthOptions +log = logging.getLogger(__name__) + class Options(AuthOptions): def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, @@ -12,7 +15,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, http_open_timeout=None, http_request_timeout=None, http_max_retry_count=None, http_max_retry_duration=None, fallback_hosts=None, fallback_hosts_use_default=None, fallback_retry_timeout=None, - idempotent_rest_publishing=None, + idempotent_rest_publishing=None, loop=None, **kwargs): super().__init__(**kwargs) @@ -49,6 +52,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, self.__fallback_hosts_use_default = fallback_hosts_use_default self.__fallback_retry_timeout = fallback_retry_timeout self.__idempotent_rest_publishing = idempotent_rest_publishing + self.__loop = loop self.__rest_hosts = self.__get_rest_hosts() @@ -184,6 +188,10 @@ def fallback_retry_timeout(self): def idempotent_rest_publishing(self): return self.__idempotent_rest_publishing + @property + def loop(self): + return self.__loop + def __get_rest_hosts(self): """ Return the list of hosts as they should be tried. First comes the main From 3db66c9b9a08eb8a239c6fbe85245ffe2c1a66ae Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 27 Sep 2022 00:12:16 +0100 Subject: [PATCH 026/481] chore: add pyee dependency --- poetry.lock | 51 ++++++++++++++++++++++++++++++++------------------ pyproject.toml | 1 + 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/poetry.lock b/poetry.lock index 27c6cbb5..028e8ae3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -12,8 +12,8 @@ sniffio = ">=1.1" typing-extensions = {version = "*", markers = "python_version < \"3.8\""} [package.extras] -doc = ["packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] -test = ["contextlib2", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (<0.15)", "uvloop (>=0.15)"] +doc = ["packaging", "sphinx-rtd-theme", "sphinx-autodoc-typehints (>=1.2.0)"] +test = ["coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "contextlib2", "uvloop (<0.15)", "mock (>=4)", "uvloop (>=0.15)"] trio = ["trio (>=0.16)"] [[package]] @@ -33,10 +33,10 @@ optional = false python-versions = ">=3.5" [package.extras] -dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] -docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] -tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] -tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] +docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "cloudpickle"] [[package]] name = "certifi" @@ -161,8 +161,8 @@ rfc3986 = {version = ">=1.3,<2", extras = ["idna2008"]} sniffio = "*" [package.extras] -brotli = ["brotli", "brotlicffi"] -cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10.0.0,<11.0.0)"] +brotli = ["brotlicffi", "brotli"] +cli = ["click (>=8.0.0,<9.0.0)", "rich (>=10.0.0,<11.0.0)", "pygments (>=2.0.0,<3.0.0)"] http2 = ["h2 (>=3,<5)"] [[package]] @@ -194,9 +194,9 @@ typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] -docs = ["jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx"] +docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] perf = ["ipython"] -testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] [[package]] name = "iniconfig" @@ -235,7 +235,7 @@ pbr = ">=0.11" six = ">=1.7" [package.extras] -docs = ["Pygments (<2)", "jinja2 (<2.7)", "sphinx", "sphinx (<1.3)"] +docs = ["sphinx", "jinja2 (<2.7)", "Pygments (<2)", "sphinx (<1.3)"] test = ["unittest2 (>=1.1.0)"] [[package]] @@ -285,8 +285,8 @@ python-versions = ">=3.6" importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} [package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] +testing = ["pytest-benchmark", "pytest"] +dev = ["tox", "pre-commit"] [[package]] name = "py" @@ -320,6 +320,17 @@ category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "pyee" +version = "9.0.4" +description = "A port of node.js's EventEmitter to python." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +typing-extensions = "*" + [[package]] name = "pyflakes" version = "2.3.1" @@ -337,7 +348,7 @@ optional = false python-versions = ">=3.6.8" [package.extras] -diagrams = ["jinja2", "railroad-diagrams"] +diagrams = ["railroad-diagrams", "jinja2"] [[package]] name = "pytest" @@ -374,7 +385,7 @@ pytest = ">=4.6" toml = "*" [package.extras] -testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] +testing = ["fields", "hunter", "process-tests", "six", "pytest-xdist", "virtualenv"] [[package]] name = "pytest-flake8" @@ -499,8 +510,8 @@ optional = false python-versions = ">=3.7" [package.extras] -docs = ["jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx"] -testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] [extras] crypto = ["pycryptodome"] @@ -509,7 +520,7 @@ oldcrypto = ["pycrypto"] [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "e7cc9e61014182ddade6f85e62d97616a52fb4a60a9a471e0b963eb1e82630aa" +content-hash = "13501b1a92c40a2047c4b3c8129700acbce42f4feb7119a608c467a9f8a2830a" [metadata.files] anyio = [ @@ -757,6 +768,10 @@ pycryptodome = [ {file = "pycryptodome-3.15.0-pp36-pypy36_pp73-win32.whl", hash = "sha256:9c772c485b27967514d0df1458b56875f4b6d025566bf27399d0c239ff1b369f"}, {file = "pycryptodome-3.15.0.tar.gz", hash = "sha256:9135dddad504592bcc18b0d2d95ce86c3a5ea87ec6447ef25cfedea12d6018b8"}, ] +pyee = [ + {file = "pyee-9.0.4-py2.py3-none-any.whl", hash = "sha256:9f066570130c554e9cc12de5a9d86f57c7ee47fece163bbdaa3e9c933cfbdfa5"}, + {file = "pyee-9.0.4.tar.gz", hash = "sha256:2770c4928abc721f46b705e6a72b0c59480c4a69c9a83ca0b00bb994f1ea4b32"}, +] pyflakes = [ {file = "pyflakes-2.3.1-py2.py3-none-any.whl", hash = "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3"}, {file = "pyflakes-2.3.1.tar.gz", hash = "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"}, diff --git a/pyproject.toml b/pyproject.toml index 51cb1353..e977e457 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ h2 = "^4.0.0" pycrypto = { version = "^2.6.1", optional = true } pycryptodome = { version = "*", optional = true } websockets = "^10.3" +pyee = "^9.0.4" [tool.poetry.extras] oldcrypto = ["pycrypto"] From 8749dca0b30bb3e7a98c8a5da79f24906667bbf1 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 27 Sep 2022 00:14:56 +0100 Subject: [PATCH 027/481] feat: queryable connection state --- ably/realtime/connection.py | 22 ++++++++++++++++++---- test/ably/realtimeconnection_test.py | 4 ++-- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 0699e2f6..898b226d 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -1,3 +1,4 @@ +import functools import logging import asyncio import websockets @@ -5,6 +6,7 @@ from ably.http.httputils import HttpUtils from ably.util.exceptions import AblyAuthException from enum import Enum, IntEnum +from pyee.asyncio import AsyncIOEventEmitter log = logging.getLogger(__name__) @@ -25,11 +27,13 @@ class ProtocolMessageAction(IntEnum): CLOSED = 8 -class Connection: +class Connection(AsyncIOEventEmitter): def __init__(self, realtime): self.__realtime = realtime self.__connection_manager = ConnectionManager(realtime) self.__state = ConnectionState.INITIALIZED + self.__connection_manager.on('connectionstate', self.on_state_update) + super().__init__() async def connect(self): await self.__connection_manager.connect() @@ -39,13 +43,18 @@ async def close(self): def on_state_update(self, state): self.__state = state + self.__realtime.options.loop.call_soon(functools.partial(self.emit, state)) @property def state(self): return self.__state + @state.setter + def state(self, value): + self.__state = value -class ConnectionManager: + +class ConnectionManager(AsyncIOEventEmitter): def __init__(self, realtime): self.options = realtime.options self.__ably = realtime @@ -53,10 +62,11 @@ def __init__(self, realtime): self.__connected_future = None self.__closed_future = None self.__websocket = None + super().__init__() def enact_state_change(self, state): self.__state = state - self.ably.connection.on_state_update(state) + self.emit('connectionstate', state) async def connect(self): if self.__state == ConnectionState.CONNECTED: @@ -75,9 +85,11 @@ async def connect(self): self.enact_state_change(ConnectionState.CONNECTED) async def close(self): + if self.__state != ConnectionState.CONNECTED: + log.warn('Connection.closed called while connection state not connected') self.enact_state_change(ConnectionState.CLOSING) self.__closed_future = asyncio.Future() - if self.__websocket and self.__state == ConnectionState.CONNECTED: + if self.__websocket: await self.send_close_message() await self.__closed_future else: @@ -117,8 +129,10 @@ async def ws_read_loop(self): self.__connected_future.set_exception( AblyAuthException(error["message"], error["statusCode"], error["code"])) self.__connected_future = None + self.__websocket = None if action == ProtocolMessageAction.CLOSED: await self.__websocket.close() + self.__websocket = None self.__closed_future.set_result(None) break diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 203cc0f5..929161a7 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -27,12 +27,12 @@ async def test_connecting_state(self): await task await ably.close() - async def test_closed_state(self): + async def test_closing_state(self): ably = await RestSetup.get_ably_realtime() await ably.connect() task = asyncio.create_task(ably.close()) await asyncio.sleep(0) - assert ably.connection.state == ConnectionState.CLOSED + assert ably.connection.state == ConnectionState.CLOSING await task async def test_auth_invalid_key(self): From efb2a2449a02e8fca3d39706988462d6c91bb272 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 27 Sep 2022 00:15:20 +0100 Subject: [PATCH 028/481] test: add tests for connection eventemitter interface --- test/ably/eventemitter_test.py | 51 ++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 test/ably/eventemitter_test.py diff --git a/test/ably/eventemitter_test.py b/test/ably/eventemitter_test.py new file mode 100644 index 00000000..d57f046a --- /dev/null +++ b/test/ably/eventemitter_test.py @@ -0,0 +1,51 @@ +import asyncio +from ably.realtime.connection import ConnectionState +from unittest.mock import Mock +from test.ably.restsetup import RestSetup +from test.ably.utils import BaseAsyncTestCase + + +class TestEventEmitter(BaseAsyncTestCase): + async def setUp(self): + self.test_vars = await RestSetup.get_test_vars() + + async def test_connection_events(self): + realtime = await RestSetup.get_ably_realtime() + listener = Mock() + realtime.connection.add_listener(ConnectionState.CONNECTED, listener) + + await realtime.connect() + + # Listener is only called once event loop is free + listener.assert_not_called() + await asyncio.sleep(0) + listener.assert_called_once() + await realtime.close() + + async def test_event_listener_error(self): + realtime = await RestSetup.get_ably_realtime() + listener = Mock() + + # If a listener throws an exception it should not propagate (#RTE6) + listener.side_effect = Exception() + realtime.connection.add_listener(ConnectionState.CONNECTED, listener) + + await realtime.connect() + + listener.assert_not_called() + await asyncio.sleep(0) + listener.assert_called_once() + await realtime.close() + + async def test_event_emitter_off(self): + realtime = await RestSetup.get_ably_realtime() + listener = Mock() + realtime.connection.add_listener(ConnectionState.CONNECTED, listener) + realtime.connection.remove_listener(ConnectionState.CONNECTED, listener) + + await realtime.connect() + + listener.assert_not_called() + await asyncio.sleep(0) + listener.assert_not_called() + await realtime.close() From dc636cd01e8657daa3c90054ee426c3ae9cc1453 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 27 Sep 2022 00:57:34 +0100 Subject: [PATCH 029/481] fix: finish tasks gracefully on failed connection --- ably/realtime/connection.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 898b226d..429552a7 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -62,6 +62,7 @@ def __init__(self, realtime): self.__connected_future = None self.__closed_future = None self.__websocket = None + self.connect_impl_task = None super().__init__() def enact_state_change(self, state): @@ -80,7 +81,7 @@ async def connect(self): else: self.enact_state_change(ConnectionState.CONNECTING) self.__connected_future = asyncio.Future() - asyncio.create_task(self.connect_impl()) + self.connect_impl_task = self.ably.options.loop.create_task(self.connect_impl()) await self.__connected_future self.enact_state_change(ConnectionState.CONNECTED) @@ -89,12 +90,14 @@ async def close(self): log.warn('Connection.closed called while connection state not connected') self.enact_state_change(ConnectionState.CLOSING) self.__closed_future = asyncio.Future() - if self.__websocket: + if self.__websocket and self.__state != ConnectionState.FAILED: await self.send_close_message() await self.__closed_future else: log.warn('Connection.closed called while connection already closed or not established') self.enact_state_change(ConnectionState.CLOSED) + if self.connect_impl_task: + await self.connect_impl_task async def send_close_message(self): await self.sendProtocolMessage({"action": ProtocolMessageAction.CLOSE}) @@ -107,8 +110,11 @@ async def connect_impl(self): async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}', extra_headers=headers) as websocket: self.__websocket = websocket - task = asyncio.create_task(self.ws_read_loop()) - await task + task = self.ably.options.loop.create_task(self.ws_read_loop()) + try: + await task + except AblyAuthException: + return async def ws_read_loop(self): while True: @@ -125,11 +131,12 @@ async def ws_read_loop(self): error = msg["error"] if error['nonfatal'] is False: self.enact_state_change(ConnectionState.FAILED) + exception = AblyAuthException(error["message"], error["statusCode"], error["code"]) if self.__connected_future: - self.__connected_future.set_exception( - AblyAuthException(error["message"], error["statusCode"], error["code"])) + self.__connected_future.set_exception(exception) self.__connected_future = None self.__websocket = None + raise exception if action == ProtocolMessageAction.CLOSED: await self.__websocket.close() self.__websocket = None From 8eeb66bad242784c57121291e8cf5626cad3d03f Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 27 Sep 2022 12:42:35 +0100 Subject: [PATCH 030/481] implement realtime ping --- ably/realtime/connection.py | 26 ++++++++++++++++++++++++- ably/realtime/realtime.py | 3 +++ ably/util/helper.py | 9 +++++++++ test/ably/realtimeconnection_test.py | 29 ++++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 ably/util/helper.py diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 429552a7..6ac5bde3 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -7,6 +7,8 @@ from ably.util.exceptions import AblyAuthException from enum import Enum, IntEnum from pyee.asyncio import AsyncIOEventEmitter +from datetime import datetime +from ably.util import helper log = logging.getLogger(__name__) @@ -21,6 +23,7 @@ class ConnectionState(Enum): class ProtocolMessageAction(IntEnum): + HEARTBEAT = 0 CONNECTED = 4 ERROR = 9 CLOSE = 7 @@ -68,16 +71,19 @@ def __init__(self, realtime): def enact_state_change(self, state): self.__state = state self.emit('connectionstate', state) + self.__ping_future = None async def connect(self): if self.__state == ConnectionState.CONNECTED: + await self.ping() return if self.__state == ConnectionState.CONNECTING: if self.__connected_future is None: - log.fatal('Connection state is CONNECTING but connected_future does not exits') + log.fatal('Connection state is CONNECTING but connected_future does not exist') return await self.__connected_future + await self.ping() else: self.enact_state_change(ConnectionState.CONNECTING) self.__connected_future = asyncio.Future() @@ -116,6 +122,20 @@ async def connect_impl(self): except AblyAuthException: return + async def ping(self): + self.__ping_future = asyncio.Future() + if self.__state in [ConnectionState.CONNECTED, ConnectionState.CONNECTING]: + ping_start_time = datetime.now().timestamp() + await self.sendProtocolMessage({"action": ProtocolMessageAction.HEARTBEAT, + "id": helper.get_random_id()}) + else: + log.error("Cannot send ping request. Connection not in connected or connecting") + return + await self.__ping_future + ping_end_time = datetime.now().timestamp() + response_time_ms = (ping_end_time - ping_start_time) * 1000 + return round(response_time_ms, 2) + async def ws_read_loop(self): while True: raw = await self.__websocket.recv() @@ -142,6 +162,10 @@ async def ws_read_loop(self): self.__websocket = None self.__closed_future.set_result(None) break + if action == ProtocolMessageAction.HEARTBEAT: + if self.__ping_future: + self.__ping_future.set_result(None) + self.__ping_future = None @property def ably(self): diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index e22b1da9..5e3edba2 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -41,6 +41,9 @@ async def connect(self): async def close(self): await self.connection.close() + async def ping(self): + return await self.connection.ping() + @property def auth(self): return self.__auth diff --git a/ably/util/helper.py b/ably/util/helper.py new file mode 100644 index 00000000..0ca32ba1 --- /dev/null +++ b/ably/util/helper.py @@ -0,0 +1,9 @@ +import random +import string + + +def get_random_id(): + # get random string of letters and digits + source = string.ascii_letters + string.digits + random_id = ''.join((random.choice(source) for i in range(8))) + return random_id diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 929161a7..2928e6a7 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -41,3 +41,32 @@ async def test_auth_invalid_key(self): await ably.connect() assert ably.connection.state == ConnectionState.FAILED await ably.close() + + async def test_connection_ping_connected(self): + ably = await RestSetup.get_ably_realtime() + await ably.connect() + response_time_ms = await ably.ping() + assert response_time_ms is not None + assert type(response_time_ms) is float + + async def test_connection_ping_initialized(self): + ably = await RestSetup.get_ably_realtime() + assert ably.connection.state == ConnectionState.INITIALIZED + response_time_ms = await ably.ping() + assert response_time_ms is None + + async def test_connection_ping_failed(self): + ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) + with pytest.raises(AblyAuthException): + await ably.connect() + assert ably.connection.state == ConnectionState.FAILED + response_time_ms = await ably.ping() + assert response_time_ms is None + + async def test_connection_ping_closed(self): + ably = await RestSetup.get_ably_realtime() + await ably.connect() + assert ably.connection.state == ConnectionState.CONNECTED + await ably.close() + response_time_ms = await ably.ping() + assert response_time_ms is None From 8e7474d9ff403d52021c5b602f44a0f387ef1a60 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 27 Sep 2022 18:50:11 +0100 Subject: [PATCH 031/481] review: correct rtn13b and rtn13e --- ably/realtime/connection.py | 21 ++++++++++++++------- test/ably/realtimeconnection_test.py | 14 +++++++------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 6ac5bde3..ae2ab701 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -4,7 +4,7 @@ import websockets import json from ably.http.httputils import HttpUtils -from ably.util.exceptions import AblyAuthException +from ably.util.exceptions import AblyAuthException, AblyException from enum import Enum, IntEnum from pyee.asyncio import AsyncIOEventEmitter from datetime import datetime @@ -44,6 +44,9 @@ async def connect(self): async def close(self): await self.__connection_manager.close() + async def ping(self): + return await self.__connection_manager.ping() + def on_state_update(self, state): self.__state = state self.__realtime.options.loop.call_soon(functools.partial(self.emit, state)) @@ -66,12 +69,12 @@ def __init__(self, realtime): self.__closed_future = None self.__websocket = None self.connect_impl_task = None + self.__ping_future = None super().__init__() def enact_state_change(self, state): self.__state = state self.emit('connectionstate', state) - self.__ping_future = None async def connect(self): if self.__state == ConnectionState.CONNECTED: @@ -125,13 +128,14 @@ async def connect_impl(self): async def ping(self): self.__ping_future = asyncio.Future() if self.__state in [ConnectionState.CONNECTED, ConnectionState.CONNECTING]: + self.__ping_id = helper.get_random_id() ping_start_time = datetime.now().timestamp() await self.sendProtocolMessage({"action": ProtocolMessageAction.HEARTBEAT, - "id": helper.get_random_id()}) + "id": self.__ping_id}) else: - log.error("Cannot send ping request. Connection not in connected or connecting") - return - await self.__ping_future + raise AblyException("Cannot send ping request. Calling ping in invalid state", 40000, 400) + if self.__ping_future: + await self.__ping_future ping_end_time = datetime.now().timestamp() response_time_ms = (ping_end_time - ping_start_time) * 1000 return round(response_time_ms, 2) @@ -164,7 +168,10 @@ async def ws_read_loop(self): break if action == ProtocolMessageAction.HEARTBEAT: if self.__ping_future: - self.__ping_future.set_result(None) + # Resolve on heartbeat from ping request. + # TODO: Handle Normal heartbeat if required + if self.__ping_id == msg["id"]: + self.__ping_future.set_result(None) self.__ping_future = None @property diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 2928e6a7..aa27e50a 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -1,7 +1,7 @@ import asyncio from ably.realtime.connection import ConnectionState import pytest -from ably.util.exceptions import AblyAuthException +from ably.util.exceptions import AblyAuthException, AblyException from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase @@ -52,21 +52,21 @@ async def test_connection_ping_connected(self): async def test_connection_ping_initialized(self): ably = await RestSetup.get_ably_realtime() assert ably.connection.state == ConnectionState.INITIALIZED - response_time_ms = await ably.ping() - assert response_time_ms is None + with pytest.raises(AblyException): + await ably.ping() async def test_connection_ping_failed(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) with pytest.raises(AblyAuthException): await ably.connect() assert ably.connection.state == ConnectionState.FAILED - response_time_ms = await ably.ping() - assert response_time_ms is None + with pytest.raises(AblyException): + await ably.ping() async def test_connection_ping_closed(self): ably = await RestSetup.get_ably_realtime() await ably.connect() assert ably.connection.state == ConnectionState.CONNECTED await ably.close() - response_time_ms = await ably.ping() - assert response_time_ms is None + with pytest.raises(AblyException): + await ably.ping() From 6220399060b6fe1302b7b5f3a2874ffcb19fbea1 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 28 Sep 2022 14:32:00 +0100 Subject: [PATCH 032/481] refactor realtime ping --- ably/realtime/connection.py | 2 +- test/ably/realtimeconnection_test.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index ae2ab701..32408c6f 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -170,7 +170,7 @@ async def ws_read_loop(self): if self.__ping_future: # Resolve on heartbeat from ping request. # TODO: Handle Normal heartbeat if required - if self.__ping_id == msg["id"]: + if self.__ping_id == msg.get("id"): self.__ping_future.set_result(None) self.__ping_future = None diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index aa27e50a..7a4a2212 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -52,21 +52,28 @@ async def test_connection_ping_connected(self): async def test_connection_ping_initialized(self): ably = await RestSetup.get_ably_realtime() assert ably.connection.state == ConnectionState.INITIALIZED - with pytest.raises(AblyException): + with pytest.raises(AblyException) as exception: await ably.ping() + assert exception.value.code == 400 + assert exception.value.status_code == 40000 async def test_connection_ping_failed(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) with pytest.raises(AblyAuthException): await ably.connect() assert ably.connection.state == ConnectionState.FAILED - with pytest.raises(AblyException): + with pytest.raises(AblyException) as exception: await ably.ping() + assert exception.value.code == 400 + assert exception.value.status_code == 40000 + await ably.close() async def test_connection_ping_closed(self): ably = await RestSetup.get_ably_realtime() await ably.connect() assert ably.connection.state == ConnectionState.CONNECTED await ably.close() - with pytest.raises(AblyException): + with pytest.raises(AblyException) as exception: await ably.ping() + assert exception.value.code == 400 + assert exception.value.status_code == 40000 From 2845af390f6e2a8b3633870ff3316b44d9f7fbb3 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 27 Sep 2022 23:30:05 +0100 Subject: [PATCH 033/481] feat: RealtimeChannels.get/release --- ably/realtime/realtime.py | 21 +++++++++++++++++++++ ably/realtime/realtime_channel.py | 7 +++++++ 2 files changed, 28 insertions(+) create mode 100644 ably/realtime/realtime_channel.py diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 5e3edba2..7ff1685f 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -3,6 +3,7 @@ from ably.realtime.connection import Connection from ably.rest.auth import Auth from ably.types.options import Options +from ably.realtime.realtime_channel import RealtimeChannel log = logging.getLogger(__name__) @@ -34,6 +35,7 @@ def __init__(self, key=None, loop=None, **kwargs): self.__options = options self.key = key self.__connection = Connection(self) + self.__channels = Channels() async def connect(self): await self.connection.connect() @@ -56,3 +58,22 @@ def options(self): def connection(self): """Establish realtime connection""" return self.__connection + + @property + def channels(self): + return self.__channels + + +class Channels: + def __init__(self): + self.all = {} + + def get(self, name): + if not self.all.get(name): + self.all[name] = RealtimeChannel(name) + return self.all[name] + + def release(self, name): + if not self.all.get(name): + return + del self.all[name] diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py new file mode 100644 index 00000000..a423c722 --- /dev/null +++ b/ably/realtime/realtime_channel.py @@ -0,0 +1,7 @@ +class RealtimeChannel(): + def __init__(self, name): + self.__name = name + + @property + def name(self): + return self.__name From 3f5841fcfb9f69b26887c3eeb6dea2ea2c62e7a6 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 27 Sep 2022 23:30:38 +0100 Subject: [PATCH 034/481] test: RealtimeChannels.get/release --- test/ably/realtimechannel_test.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 test/ably/realtimechannel_test.py diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py new file mode 100644 index 00000000..2b4d162a --- /dev/null +++ b/test/ably/realtimechannel_test.py @@ -0,0 +1,21 @@ +from test.ably.restsetup import RestSetup +from test.ably.utils import BaseAsyncTestCase + + +class TestRealtimeChannel(BaseAsyncTestCase): + async def setUp(self): + self.test_vars = await RestSetup.get_test_vars() + self.valid_key_format = "api:key" + + async def test_channels_get(self): + ably = await RestSetup.get_ably_realtime() + channel = ably.channels.get('my_channel') + assert channel == ably.channels.all['my_channel'] + await ably.close() + + async def test_channels_release(self): + ably = await RestSetup.get_ably_realtime() + ably.channels.get('my_channel') + ably.channels.release('my_channel') + assert ably.channels.all.get('my_channel') is None + await ably.close() From b1044f33fa3c940ea340af7d9cf2dcc1272f91ce Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 28 Sep 2022 00:39:13 +0100 Subject: [PATCH 035/481] feat: RealtimeChannel attach/detach --- ably/realtime/connection.py | 10 +++ ably/realtime/realtime.py | 13 +++- ably/realtime/realtime_channel.py | 118 +++++++++++++++++++++++++++++- 3 files changed, 136 insertions(+), 5 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 32408c6f..a0896a00 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -28,6 +28,10 @@ class ProtocolMessageAction(IntEnum): ERROR = 9 CLOSE = 7 CLOSED = 8 + ATTACH = 10 + ATTACHED = 11 + DETACH = 12 + DETACHED = 13 class Connection(AsyncIOEventEmitter): @@ -59,6 +63,10 @@ def state(self): def state(self, value): self.__state = value + @property + def connection_manager(self): + return self.__connection_manager + class ConnectionManager(AsyncIOEventEmitter): def __init__(self, realtime): @@ -173,6 +181,8 @@ async def ws_read_loop(self): if self.__ping_id == msg.get("id"): self.__ping_future.set_result(None) self.__ping_future = None + if action in [ProtocolMessageAction.ATTACHED, ProtocolMessageAction.DETACHED]: + self.ably.channels.on_channel_message(msg) @property def ably(self): diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 7ff1685f..f8f658bf 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -35,7 +35,7 @@ def __init__(self, key=None, loop=None, **kwargs): self.__options = options self.key = key self.__connection = Connection(self) - self.__channels = Channels() + self.__channels = Channels(self) async def connect(self): await self.connection.connect() @@ -65,15 +65,22 @@ def channels(self): class Channels: - def __init__(self): + def __init__(self, realtime): self.all = {} + self.__realtime = realtime def get(self, name): if not self.all.get(name): - self.all[name] = RealtimeChannel(name) + self.all[name] = RealtimeChannel(self.__realtime, name) return self.all[name] def release(self, name): if not self.all.get(name): return del self.all[name] + + def on_channel_message(self, msg): + channel = self.all.get(msg.get('channel')) + if not channel: + log.warning('Channel message recieved but no channel instance found') + channel.on_message(msg) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index a423c722..34976438 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -1,7 +1,121 @@ -class RealtimeChannel(): - def __init__(self, name): +import asyncio +import logging +from ably.realtime.connection import ConnectionState, ProtocolMessageAction +from ably.util.exceptions import AblyException +from pyee.asyncio import AsyncIOEventEmitter +from enum import Enum + +log = logging.getLogger(__name__) + + +class ChannelState(Enum): + INITIALIZED = 'initialized' + ATTACHING = 'attaching' + ATTACHED = 'attached' + DETACHING = 'detaching' + DETACHED = 'detached' + + +class RealtimeChannel(AsyncIOEventEmitter): + def __init__(self, realtime, name): self.__name = name + self.__attach_future = None + self.__detach_future = None + self.__realtime = realtime + self.__state = ChannelState.INITIALIZED + super().__init__() + + async def attach(self): + # RTL4a - if channel is attached do nothing + if self.state == ChannelState.ATTACHED: + return + + # RTL4b + if self.__realtime.connection.state not in [ConnectionState.CONNECTING, ConnectionState.CONNECTED]: + raise AblyException( + message=f"Unable to attach; channel state = {self.state}", + code=90001, + status_code=400 + ) + + # RTL4h - wait for pending attach/detach + if self.state == ChannelState.ATTACHING: + await self.__attach_future + return + elif self.state == ChannelState.DETACHING: + await self.__detach_future + + self.set_state(ChannelState.ATTACHING) + + # RTL4i - wait for pending connection + if self.__realtime.connection.state == ConnectionState.CONNECTING: + await self.__realtime.connect() + + self.__attach_future = asyncio.Future() + await self.__realtime.connection.connection_manager.sendProtocolMessage( + { + "action": ProtocolMessageAction.ATTACH, + "channel": self.name, + } + ) + await self.__attach_future + self.set_state(ChannelState.ATTACHED) + + async def detach(self): + # RTL5g - raise exception if state invalid + if self.__realtime.connection.state in [ConnectionState.CLOSING, ConnectionState.FAILED]: + raise AblyException( + message=f"Unable to detach; channel state = {self.state}", + code=90001, + status_code=400 + ) + + # RTL5a - if channel already detached do nothing + if self.state in [ChannelState.INITIALIZED, ChannelState.DETACHED]: + return + + # RTL5i - wait for pending attach/detach + if self.state == ChannelState.DETACHING: + await self.__detach_future + return + elif self.state == ChannelState.ATTACHING: + await self.__attach_future + + self.set_state(ChannelState.DETACHING) + + # RTL5h - wait for pending connection + if self.__realtime.connection.state == ConnectionState.CONNECTING: + await self.__realtime.connect() + + self.__detach_future = asyncio.Future() + await self.__realtime.connection.connection_manager.sendProtocolMessage( + { + "action": ProtocolMessageAction.DETACH, + "channel": self.name, + } + ) + await self.__detach_future + self.set_state(ChannelState.DETACHED) + + def on_message(self, msg): + action = msg.get('action') + if action == ProtocolMessageAction.ATTACHED: + if self.__attach_future: + self.__attach_future.set_result(None) + self.__attach_future = None + elif action == ProtocolMessageAction.DETACHED: + if self.__detach_future: + self.__detach_future.set_result(None) + self.__detach_future = None + + def set_state(self, state): + self.__state = state + self.emit(state) @property def name(self): return self.__name + + @property + def state(self): + return self.__state From 57888b62886010c886c236d9eb2ef7b7f3b8d45d Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 28 Sep 2022 00:39:22 +0100 Subject: [PATCH 036/481] test: RealtimeChannel attach/detach --- test/ably/realtimechannel_test.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py index 2b4d162a..9f08736c 100644 --- a/test/ably/realtimechannel_test.py +++ b/test/ably/realtimechannel_test.py @@ -1,3 +1,4 @@ +from ably.realtime.realtime_channel import ChannelState from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase @@ -19,3 +20,21 @@ async def test_channels_release(self): ably.channels.release('my_channel') assert ably.channels.all.get('my_channel') is None await ably.close() + + async def test_channel_attach(self): + ably = await RestSetup.get_ably_realtime() + await ably.connect() + channel = ably.channels.get('my_channel') + assert channel.state == ChannelState.INITIALIZED + await channel.attach() + assert channel.state == ChannelState.ATTACHED + await ably.close() + + async def test_channel_detach(self): + ably = await RestSetup.get_ably_realtime() + await ably.connect() + channel = ably.channels.get('my_channel') + await channel.attach() + await channel.detach() + assert channel.state == ChannelState.DETACHED + await ably.close() From e7b4f1a61dddc36de47093651d42cd21487c66d8 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Fri, 30 Sep 2022 01:19:13 +0100 Subject: [PATCH 037/481] fix: ping behaviour fixups --- ably/realtime/connection.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index a0896a00..275c64f8 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -86,7 +86,6 @@ def enact_state_change(self, state): async def connect(self): if self.__state == ConnectionState.CONNECTED: - await self.ping() return if self.__state == ConnectionState.CONNECTING: @@ -94,7 +93,6 @@ async def connect(self): log.fatal('Connection state is CONNECTING but connected_future does not exist') return await self.__connected_future - await self.ping() else: self.enact_state_change(ConnectionState.CONNECTING) self.__connected_future = asyncio.Future() @@ -134,6 +132,10 @@ async def connect_impl(self): return async def ping(self): + if self.__ping_future: + response = await self.__ping_future + return response + self.__ping_future = asyncio.Future() if self.__state in [ConnectionState.CONNECTED, ConnectionState.CONNECTING]: self.__ping_id = helper.get_random_id() @@ -142,8 +144,6 @@ async def ping(self): "id": self.__ping_id}) else: raise AblyException("Cannot send ping request. Calling ping in invalid state", 40000, 400) - if self.__ping_future: - await self.__ping_future ping_end_time = datetime.now().timestamp() response_time_ms = (ping_end_time - ping_start_time) * 1000 return round(response_time_ms, 2) @@ -180,7 +180,7 @@ async def ws_read_loop(self): # TODO: Handle Normal heartbeat if required if self.__ping_id == msg.get("id"): self.__ping_future.set_result(None) - self.__ping_future = None + self.__ping_future = None if action in [ProtocolMessageAction.ATTACHED, ProtocolMessageAction.DETACHED]: self.ably.channels.on_channel_message(msg) From 8bae33a76f7bdc43b6d3e58019ff3607fa8cf4d0 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Fri, 30 Sep 2022 22:29:22 +0100 Subject: [PATCH 038/481] refactor: separate connection state checks from implementation --- ably/realtime/connection.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 275c64f8..43672b03 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -76,7 +76,7 @@ def __init__(self, realtime): self.__connected_future = None self.__closed_future = None self.__websocket = None - self.connect_impl_task = None + self.setup_ws_task = None self.__ping_future = None super().__init__() @@ -93,12 +93,11 @@ async def connect(self): log.fatal('Connection state is CONNECTING but connected_future does not exist') return await self.__connected_future + self.enact_state_change(ConnectionState.CONNECTED) else: self.enact_state_change(ConnectionState.CONNECTING) self.__connected_future = asyncio.Future() - self.connect_impl_task = self.ably.options.loop.create_task(self.connect_impl()) - await self.__connected_future - self.enact_state_change(ConnectionState.CONNECTED) + await self.connect_impl() async def close(self): if self.__state != ConnectionState.CONNECTED: @@ -111,8 +110,13 @@ async def close(self): else: log.warn('Connection.closed called while connection already closed or not established') self.enact_state_change(ConnectionState.CLOSED) - if self.connect_impl_task: - await self.connect_impl_task + if self.setup_ws_task: + await self.setup_ws_task + + async def connect_impl(self): + self.setup_ws_task = self.ably.options.loop.create_task(self.setup_ws()) + await self.__connected_future + self.enact_state_change(ConnectionState.CONNECTED) async def send_close_message(self): await self.sendProtocolMessage({"action": ProtocolMessageAction.CLOSE}) @@ -120,7 +124,7 @@ async def send_close_message(self): async def sendProtocolMessage(self, protocolMessage): await self.__websocket.send(json.dumps(protocolMessage)) - async def connect_impl(self): + async def setup_ws(self): headers = HttpUtils.default_headers() async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}', extra_headers=headers) as websocket: From 46d7bb066a86b413d7d892d4d74517080a961841 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Fri, 30 Sep 2022 22:31:06 +0100 Subject: [PATCH 039/481] refactor: single initial connection state --- ably/realtime/connection.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 43672b03..ff560fe1 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -38,7 +38,7 @@ class Connection(AsyncIOEventEmitter): def __init__(self, realtime): self.__realtime = realtime self.__connection_manager = ConnectionManager(realtime) - self.__state = ConnectionState.INITIALIZED + self.__connection_manager = ConnectionManager(realtime, self.state) self.__connection_manager.on('connectionstate', self.on_state_update) super().__init__() @@ -69,10 +69,10 @@ def connection_manager(self): class ConnectionManager(AsyncIOEventEmitter): - def __init__(self, realtime): + def __init__(self, realtime, initial_state): self.options = realtime.options self.__ably = realtime - self.__state = ConnectionState.INITIALIZED + self.__state = initial_state self.__connected_future = None self.__closed_future = None self.__websocket = None From 8f634e7ca5550829c7055979d6c8476e87a3e4aa Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 4 Oct 2022 22:26:24 +0100 Subject: [PATCH 040/481] feat: add autoconnect implementation and client option fixes #321 --- ably/realtime/connection.py | 4 ++-- ably/realtime/realtime.py | 3 +++ ably/types/options.py | 7 ++++++- test/ably/realtimeconnection_test.py | 5 +++-- test/ably/realtimeinit_test.py | 10 +++++----- 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index ff560fe1..cba28eaf 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -37,7 +37,7 @@ class ProtocolMessageAction(IntEnum): class Connection(AsyncIOEventEmitter): def __init__(self, realtime): self.__realtime = realtime - self.__connection_manager = ConnectionManager(realtime) + self.__state = ConnectionState.CONNECTING if realtime.options.auto_connect else ConnectionState.INITIALIZED self.__connection_manager = ConnectionManager(realtime, self.state) self.__connection_manager.on('connectionstate', self.on_state_update) super().__init__() @@ -73,7 +73,7 @@ def __init__(self, realtime, initial_state): self.options = realtime.options self.__ably = realtime self.__state = initial_state - self.__connected_future = None + self.__connected_future = asyncio.Future() if initial_state == ConnectionState.CONNECTING else None self.__closed_future = None self.__websocket = None self.setup_ws_task = None diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index f8f658bf..35f711c0 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -37,6 +37,9 @@ def __init__(self, key=None, loop=None, **kwargs): self.__connection = Connection(self) self.__channels = Channels(self) + if options.auto_connect: + asyncio.ensure_future(self.connection.connection_manager.connect_impl()) + async def connect(self): await self.connection.connect() diff --git a/ably/types/options.py b/ably/types/options.py index 9a4791e0..6d254440 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -15,7 +15,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, http_open_timeout=None, http_request_timeout=None, http_max_retry_count=None, http_max_retry_duration=None, fallback_hosts=None, fallback_hosts_use_default=None, fallback_retry_timeout=None, - idempotent_rest_publishing=None, loop=None, + idempotent_rest_publishing=None, loop=None, auto_connect=True, **kwargs): super().__init__(**kwargs) @@ -53,6 +53,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, self.__fallback_retry_timeout = fallback_retry_timeout self.__idempotent_rest_publishing = idempotent_rest_publishing self.__loop = loop + self.__auto_connect = auto_connect self.__rest_hosts = self.__get_rest_hosts() @@ -192,6 +193,10 @@ def idempotent_rest_publishing(self): def loop(self): return self.__loop + @property + def auto_connect(self): + return self.__auto_connect + def __get_rest_hosts(self): """ Return the list of hosts as they should be tried. First comes the main diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 7a4a2212..9695dd3c 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -12,7 +12,7 @@ async def setUp(self): self.valid_key_format = "api:key" async def test_auth_connection(self): - ably = await RestSetup.get_ably_realtime() + ably = await RestSetup.get_ably_realtime(auto_connect=False) assert ably.connection.state == ConnectionState.INITIALIZED await ably.connect() assert ably.connection.state == ConnectionState.CONNECTED @@ -48,9 +48,10 @@ async def test_connection_ping_connected(self): response_time_ms = await ably.ping() assert response_time_ms is not None assert type(response_time_ms) is float + await ably.close() async def test_connection_ping_initialized(self): - ably = await RestSetup.get_ably_realtime() + ably = await RestSetup.get_ably_realtime(auto_connect=False) assert ably.connection.state == ConnectionState.INITIALIZED with pytest.raises(AblyException) as exception: await ably.ping() diff --git a/test/ably/realtimeinit_test.py b/test/ably/realtimeinit_test.py index a85f9576..fdb99a8e 100644 --- a/test/ably/realtimeinit_test.py +++ b/test/ably/realtimeinit_test.py @@ -12,24 +12,24 @@ async def setUp(self): self.valid_key_format = "api:key" async def test_auth_with_valid_key(self): - ably = await RestSetup.get_ably_realtime(key=self.test_vars["keys"][0]["key_str"]) + ably = await RestSetup.get_ably_realtime(key=self.test_vars["keys"][0]["key_str"], auto_connect=False) assert Auth.Method.BASIC == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" assert ably.auth.auth_options.key_name == self.test_vars["keys"][0]['key_name'] assert ably.auth.auth_options.key_secret == self.test_vars["keys"][0]['key_secret'] async def test_auth_incorrect_key(self): with pytest.raises(AblyAuthException): - await RestSetup.get_ably_realtime(key="some invalid key") + await RestSetup.get_ably_realtime(key="some invalid key", auto_connect=False) async def test_auth_with_valid_key_format(self): key = self.valid_key_format.split(":") - ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) + ably = await RestSetup.get_ably_realtime(key=self.valid_key_format, auto_connect=False) assert Auth.Method.BASIC == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" assert ably.auth.auth_options.key_name == key[0] assert ably.auth.auth_options.key_secret == key[1] async def test_auth_connection(self): - ably = await RestSetup.get_ably_realtime() + ably = await RestSetup.get_ably_realtime(auto_connect=False) assert ably.connection.state == ConnectionState.INITIALIZED await ably.connect() assert ably.connection.state == ConnectionState.CONNECTED @@ -37,7 +37,7 @@ async def test_auth_connection(self): assert ably.connection.state == ConnectionState.CLOSED async def test_auth_invalid_key(self): - ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) + ably = await RestSetup.get_ably_realtime(key=self.valid_key_format, auto_connect=False) with pytest.raises(AblyAuthException): await ably.connect() await ably.close() From 66b9fccdd66071c2cc28202ade1d07089d109b06 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 4 Oct 2022 22:27:26 +0100 Subject: [PATCH 041/481] test: add test for autoconnect behaviour --- test/ably/realtimeconnection_test.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 9695dd3c..ec6980f1 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -78,3 +78,11 @@ async def test_connection_ping_closed(self): await ably.ping() assert exception.value.code == 400 assert exception.value.status_code == 40000 + + async def test_auto_connect(self): + ably = await RestSetup.get_ably_realtime() + connect_future = asyncio.Future() + ably.connection.on(ConnectionState.CONNECTED, lambda: connect_future.set_result(None)) + await connect_future + assert ably.connection.state == ConnectionState.CONNECTED + await ably.close() From 7a90806028e34360a11b56c9195d5523b05cab97 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 10 Oct 2022 01:44:42 +0100 Subject: [PATCH 042/481] feat: RealtimeChannel.subscribe --- ably/realtime/connection.py | 7 +++++- ably/realtime/realtime_channel.py | 39 +++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index cba28eaf..de267164 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -32,6 +32,7 @@ class ProtocolMessageAction(IntEnum): ATTACHED = 11 DETACH = 12 DETACHED = 13 + MESSAGE = 15 class Connection(AsyncIOEventEmitter): @@ -185,7 +186,11 @@ async def ws_read_loop(self): if self.__ping_id == msg.get("id"): self.__ping_future.set_result(None) self.__ping_future = None - if action in [ProtocolMessageAction.ATTACHED, ProtocolMessageAction.DETACHED]: + if action in ( + ProtocolMessageAction.ATTACHED, + ProtocolMessageAction.DETACHED, + ProtocolMessageAction.MESSAGE + ): self.ably.channels.on_channel_message(msg) @property diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 34976438..5819774b 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -1,6 +1,9 @@ import asyncio import logging +import types + from ably.realtime.connection import ConnectionState, ProtocolMessageAction +from ably.types.message import Message from ably.util.exceptions import AblyException from pyee.asyncio import AsyncIOEventEmitter from enum import Enum @@ -23,6 +26,8 @@ def __init__(self, realtime, name): self.__detach_future = None self.__realtime = realtime self.__state = ChannelState.INITIALIZED + self.__message_emitter = AsyncIOEventEmitter() + self.__all_messages_emitter = AsyncIOEventEmitter() super().__init__() async def attach(self): @@ -97,6 +102,35 @@ async def detach(self): await self.__detach_future self.set_state(ChannelState.DETACHED) + async def subscribe(self, *args): + if isinstance(args[0], str): + event = args[0] + listener = args[1] + elif isinstance(args[0], types.FunctionType) or asyncio.iscoroutinefunction(args[0]): + listener = args[0] + event = None + else: + raise ValueError('invalid subscribe arguments') + + if self.__realtime.connection.state == ConnectionState.CONNECTING: + await self.__realtime.connection.connect() + elif self.__realtime.connection.state != ConnectionState.CONNECTED: + raise AblyException( + 'Cannot subscribe to channel, invalid connection state: {self.__realtime.connection.state}', + 400, + 40000 + ) + + if self.state in (ChannelState.INITIALIZED, ChannelState.ATTACHING, ChannelState.DETACHED): + await self.attach() + + if event is not None: + self.__message_emitter.on(event, listener) + else: + self.__all_messages_emitter.on('message', listener) + + await self.attach() + def on_message(self, msg): action = msg.get('action') if action == ProtocolMessageAction.ATTACHED: @@ -107,6 +141,11 @@ def on_message(self, msg): if self.__detach_future: self.__detach_future.set_result(None) self.__detach_future = None + elif action == ProtocolMessageAction.MESSAGE: + messages = Message.from_encoded_array(msg.get('messages')) + for message in messages: + self.__message_emitter.emit(message.name, message) + self.__all_messages_emitter.emit('message', message) def set_state(self, state): self.__state = state From e07edab9b1ea0c076a7e2e0d5a4b4adc3c04f67a Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 10 Oct 2022 01:44:56 +0100 Subject: [PATCH 043/481] test: add tests for RealtimeChannel.Subscribe --- test/ably/realtimechannel_test.py | 105 ++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py index 9f08736c..4fc55180 100644 --- a/test/ably/realtimechannel_test.py +++ b/test/ably/realtimechannel_test.py @@ -1,4 +1,8 @@ +import asyncio +from unittest.mock import Mock +import types from ably.realtime.realtime_channel import ChannelState +from ably.types.message import Message from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase @@ -38,3 +42,104 @@ async def test_channel_detach(self): await channel.detach() assert channel.state == ChannelState.DETACHED await ably.close() + + # RTL7b + async def test_subscribe(self): + ably = await RestSetup.get_ably_realtime() + + first_message_future = asyncio.Future() + second_message_future = asyncio.Future() + + def listener(message): + if not first_message_future.done(): + first_message_future.set_result(message) + else: + second_message_future.set_result(message) + + await ably.connect() + channel = ably.channels.get('my_channel') + await channel.attach() + await channel.subscribe('event', listener) + + # publish a message using rest client + rest = await RestSetup.get_ably_rest() + rest_channel = rest.channels.get('my_channel') + await rest_channel.publish('event', 'data') + message = await first_message_future + + assert isinstance(message, Message) + assert message.name == 'event' + assert message.data == 'data' + + # test that the listener is called again for further publishes + await rest_channel.publish('event', 'data') + await second_message_future + + await ably.close() + await rest.close() + + async def test_subscribe_coroutine(self): + ably = await RestSetup.get_ably_realtime() + await ably.connect() + channel = ably.channels.get('my_channel') + await channel.attach() + + message_future = asyncio.Future() + + # AsyncMock doesn't work in python 3.7 so use an actual coroutine + async def listener(msg): + message_future.set_result(msg) + + await channel.subscribe('event', listener) + + # publish a message using rest client + rest = await RestSetup.get_ably_rest() + rest_channel = rest.channels.get('my_channel') + await rest_channel.publish('event', 'data') + + message = await message_future + assert isinstance(message, Message) + assert message.name == 'event' + assert message.data == 'data' + + await ably.close() + await rest.close() + + # RTL7a + async def test_subscribe_all_events(self): + ably = await RestSetup.get_ably_realtime() + await ably.connect() + channel = ably.channels.get('my_channel') + await channel.attach() + + message_future = asyncio.Future() + listener = Mock(spec=types.FunctionType, side_effect=lambda msg: message_future.set_result(msg)) + await channel.subscribe(listener) + + # publish a message using rest client + rest = await RestSetup.get_ably_rest() + rest_channel = rest.channels.get('my_channel') + await rest_channel.publish('event', 'data') + message = await message_future + + listener.assert_called_once() + assert isinstance(message, Message) + assert message.name == 'event' + assert message.data == 'data' + + await ably.close() + await rest.close() + + # RTL7c + async def test_subscribe_auto_attach(self): + ably = await RestSetup.get_ably_realtime() + await ably.connect() + channel = ably.channels.get('my_channel') + assert channel.state == ChannelState.INITIALIZED + + listener = Mock() + await channel.subscribe('event', listener) + + assert channel.state == ChannelState.ATTACHED + + await ably.close() From c95eaefa2eade92113b239f474dbd55ab16bbb5f Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 10 Oct 2022 01:55:55 +0100 Subject: [PATCH 044/481] feat: RealtimeChannel.unsubscribe --- ably/realtime/realtime_channel.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 5819774b..ad9bd224 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -131,6 +131,27 @@ async def subscribe(self, *args): await self.attach() + def unsubscribe(self, *args): + if len(args) == 0: + event = None + listener = None + elif isinstance(args[0], str): + event = args[0] + listener = args[1] + elif isinstance(args[0], types.FunctionType) or asyncio.iscoroutinefunction(args[0]): + listener = args[0] + event = None + else: + raise ValueError('invalid unsubscribe arguments') + + if listener is None: + self.__message_emitter.remove_all_listeners() + self.__all_messages_emitter.remove_all_listeners() + elif event is not None: + self.__message_emitter.remove_listener(event, listener) + else: + self.__all_messages_emitter.remove_listener('message', listener) + def on_message(self, msg): action = msg.get('action') if action == ProtocolMessageAction.ATTACHED: From c71cd747d368ecc7ec07d81b82da12634d75537d Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 10 Oct 2022 01:56:10 +0100 Subject: [PATCH 045/481] test: add tests for RealtimeChannel.unsubscribe --- test/ably/realtimechannel_test.py | 57 +++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py index 4fc55180..4ee30357 100644 --- a/test/ably/realtimechannel_test.py +++ b/test/ably/realtimechannel_test.py @@ -143,3 +143,60 @@ async def test_subscribe_auto_attach(self): assert channel.state == ChannelState.ATTACHED await ably.close() + + # RTL8b + async def test_unsubscribe(self): + ably = await RestSetup.get_ably_realtime() + await ably.connect() + channel = ably.channels.get('my_channel') + await channel.attach() + + message_future = asyncio.Future() + listener = Mock(side_effect=lambda msg: message_future.set_result(msg)) + await channel.subscribe('event', listener) + + # publish a message using rest client + rest = await RestSetup.get_ably_rest() + rest_channel = rest.channels.get('my_channel') + await rest_channel.publish('event', 'data') + await message_future + listener.assert_called_once() + + # unsubscribe the listener from the channel + channel.unsubscribe('event', listener) + + # test that the listener is not called again for further publishes + await rest_channel.publish('event', 'data') + await asyncio.sleep(1) + assert listener.call_count == 1 + + await ably.close() + await rest.close() + + # RTL8c + async def test_unsubscribe_all(self): + ably = await RestSetup.get_ably_realtime() + await ably.connect() + channel = ably.channels.get('my_channel') + await channel.attach() + message_future = asyncio.Future() + listener = Mock(side_effect=lambda msg: message_future.set_result(msg)) + await channel.subscribe('event', listener) + + # publish a message using rest client + rest = await RestSetup.get_ably_rest() + rest_channel = rest.channels.get('my_channel') + await rest_channel.publish('event', 'data') + await message_future + listener.assert_called_once() + + # unsubscribe all listeners from the channel + channel.unsubscribe() + + # test that the listener is not called again for further publishes + await rest_channel.publish('event', 'data') + await asyncio.sleep(1) + assert listener.call_count == 1 + + await ably.close() + await rest.close() From 8775932ad5b5b6b497ab9afdec8b03cd1acdbf9e Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 12 Oct 2022 20:52:07 +0100 Subject: [PATCH 046/481] refactor: improve error messages for subscribe args --- ably/realtime/realtime_channel.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index ad9bd224..ed84ce32 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -105,6 +105,8 @@ async def detach(self): async def subscribe(self, *args): if isinstance(args[0], str): event = args[0] + if not args[1]: + raise ValueError("channel.subscribe called without listener") listener = args[1] elif isinstance(args[0], types.FunctionType) or asyncio.iscoroutinefunction(args[0]): listener = args[0] @@ -137,6 +139,8 @@ def unsubscribe(self, *args): listener = None elif isinstance(args[0], str): event = args[0] + if not args[1]: + raise ValueError("channel.unsubscribe called without listener") listener = args[1] elif isinstance(args[0], types.FunctionType) or asyncio.iscoroutinefunction(args[0]): listener = args[0] From 087bc82fba7a5ea97685b600bcd915202b1f9e35 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 12 Oct 2022 22:52:31 +0100 Subject: [PATCH 047/481] feat: ConnectionStateChange fixes: #320 --- ably/realtime/connection.py | 22 ++++++++++++++++------ test/ably/realtimeconnection_test.py | 2 +- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index cba28eaf..37b61c64 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -9,6 +9,8 @@ from pyee.asyncio import AsyncIOEventEmitter from datetime import datetime from ably.util import helper +from dataclasses import dataclass +from typing import Optional log = logging.getLogger(__name__) @@ -22,6 +24,13 @@ class ConnectionState(Enum): FAILED = 'failed' +@dataclass +class ConnectionStateChange: + previous: ConnectionState + current: ConnectionState + reason: Optional[AblyException] = None + + class ProtocolMessageAction(IntEnum): HEARTBEAT = 0 CONNECTED = 4 @@ -51,9 +60,9 @@ async def close(self): async def ping(self): return await self.__connection_manager.ping() - def on_state_update(self, state): - self.__state = state - self.__realtime.options.loop.call_soon(functools.partial(self.emit, state)) + def on_state_update(self, state_change): + self.__state = state_change.current + self.__realtime.options.loop.call_soon(functools.partial(self.emit, state_change.current, state_change)) @property def state(self): @@ -80,9 +89,10 @@ def __init__(self, realtime, initial_state): self.__ping_future = None super().__init__() - def enact_state_change(self, state): + def enact_state_change(self, state, reason=None): + current_state = self.__state self.__state = state - self.emit('connectionstate', state) + self.emit('connectionstate', ConnectionStateChange(current_state, state, reason)) async def connect(self): if self.__state == ConnectionState.CONNECTED: @@ -166,8 +176,8 @@ async def ws_read_loop(self): if action == ProtocolMessageAction.ERROR: # ERROR error = msg["error"] if error['nonfatal'] is False: - self.enact_state_change(ConnectionState.FAILED) exception = AblyAuthException(error["message"], error["statusCode"], error["code"]) + self.enact_state_change(ConnectionState.FAILED, exception) if self.__connected_future: self.__connected_future.set_exception(exception) self.__connected_future = None diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index ec6980f1..220836d3 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -82,7 +82,7 @@ async def test_connection_ping_closed(self): async def test_auto_connect(self): ably = await RestSetup.get_ably_realtime() connect_future = asyncio.Future() - ably.connection.on(ConnectionState.CONNECTED, lambda: connect_future.set_result(None)) + ably.connection.on(ConnectionState.CONNECTED, lambda change: connect_future.set_result(change)) await connect_future assert ably.connection.state == ConnectionState.CONNECTED await ably.close() From 9b4648c87a290362d3b90d88ae0118b25ddcbc13 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 12 Oct 2022 22:53:11 +0100 Subject: [PATCH 048/481] test: add tests for ConnectionStateChange --- test/ably/realtimeconnection_test.py | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 220836d3..41ab1d5d 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -86,3 +86,39 @@ async def test_auto_connect(self): await connect_future assert ably.connection.state == ConnectionState.CONNECTED await ably.close() + + async def test_connection_state_change(self): + ably = await RestSetup.get_ably_realtime() + + connected_future = asyncio.Future() + + def on_state_change(change): + connected_future.set_result(change) + + ably.connection.on(ConnectionState.CONNECTED, on_state_change) + + state_change = await connected_future + assert state_change.previous == ConnectionState.CONNECTING + assert state_change.current == ConnectionState.CONNECTED + await ably.close() + + async def test_connection_state_change_reason(self): + ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) + + failed_changes = [] + + def on_state_change(change): + failed_changes.append(change) + + ably.connection.on(ConnectionState.FAILED, on_state_change) + + with pytest.raises(AblyAuthException) as exception: + await ably.connect() + + assert len(failed_changes) == 1 + state_change = failed_changes[0] + assert state_change is not None + assert state_change.previous == ConnectionState.CONNECTING + assert state_change.current == ConnectionState.FAILED + assert state_change.reason == exception.value + await ably.close() From 0b8425880848235f69a62bcb3b375aa2c9612eb0 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 13 Oct 2022 14:28:24 +0100 Subject: [PATCH 049/481] refactor: extract is_function_or_coroutine helper --- ably/realtime/realtime_channel.py | 7 ++++--- ably/util/helper.py | 6 ++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index ed84ce32..44196484 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -1,6 +1,5 @@ import asyncio import logging -import types from ably.realtime.connection import ConnectionState, ProtocolMessageAction from ably.types.message import Message @@ -8,6 +7,8 @@ from pyee.asyncio import AsyncIOEventEmitter from enum import Enum +from ably.util.helper import is_function_or_coroutine + log = logging.getLogger(__name__) @@ -108,7 +109,7 @@ async def subscribe(self, *args): if not args[1]: raise ValueError("channel.subscribe called without listener") listener = args[1] - elif isinstance(args[0], types.FunctionType) or asyncio.iscoroutinefunction(args[0]): + elif is_function_or_coroutine(args[0]): listener = args[0] event = None else: @@ -142,7 +143,7 @@ def unsubscribe(self, *args): if not args[1]: raise ValueError("channel.unsubscribe called without listener") listener = args[1] - elif isinstance(args[0], types.FunctionType) or asyncio.iscoroutinefunction(args[0]): + elif is_function_or_coroutine(args[0]): listener = args[0] event = None else: diff --git a/ably/util/helper.py b/ably/util/helper.py index 0ca32ba1..c3b427ac 100644 --- a/ably/util/helper.py +++ b/ably/util/helper.py @@ -1,5 +1,7 @@ import random import string +import types +import asyncio def get_random_id(): @@ -7,3 +9,7 @@ def get_random_id(): source = string.ascii_letters + string.digits random_id = ''.join((random.choice(source) for i in range(8))) return random_id + + +def is_function_or_coroutine(value): + return isinstance(value, types.FunctionType) or asyncio.iscoroutinefunction(value) From 9f3dda5308601f6f4fed7244570ce7c890c7205d Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 13 Oct 2022 14:42:53 +0100 Subject: [PATCH 050/481] refactor: validate subscribe/unsubscribe listener args --- ably/realtime/realtime_channel.py | 4 ++++ test/ably/realtimechannel_test.py | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 44196484..9d949d97 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -108,6 +108,8 @@ async def subscribe(self, *args): event = args[0] if not args[1]: raise ValueError("channel.subscribe called without listener") + if not is_function_or_coroutine(args[1]): + raise ValueError("subscribe listener must be function or coroutine function") listener = args[1] elif is_function_or_coroutine(args[0]): listener = args[0] @@ -142,6 +144,8 @@ def unsubscribe(self, *args): event = args[0] if not args[1]: raise ValueError("channel.unsubscribe called without listener") + if not is_function_or_coroutine(args[1]): + raise ValueError("unsubscribe listener must be a function or coroutine function") listener = args[1] elif is_function_or_coroutine(args[0]): listener = args[0] diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py index 4ee30357..d7acb215 100644 --- a/test/ably/realtimechannel_test.py +++ b/test/ably/realtimechannel_test.py @@ -137,7 +137,7 @@ async def test_subscribe_auto_attach(self): channel = ably.channels.get('my_channel') assert channel.state == ChannelState.INITIALIZED - listener = Mock() + listener = Mock(spec=types.FunctionType) await channel.subscribe('event', listener) assert channel.state == ChannelState.ATTACHED @@ -152,7 +152,7 @@ async def test_unsubscribe(self): await channel.attach() message_future = asyncio.Future() - listener = Mock(side_effect=lambda msg: message_future.set_result(msg)) + listener = Mock(spec=types.FunctionType, side_effect=lambda msg: message_future.set_result(msg)) await channel.subscribe('event', listener) # publish a message using rest client @@ -180,7 +180,7 @@ async def test_unsubscribe_all(self): channel = ably.channels.get('my_channel') await channel.attach() message_future = asyncio.Future() - listener = Mock(side_effect=lambda msg: message_future.set_result(msg)) + listener = Mock(spec=types.FunctionType, side_effect=lambda msg: message_future.set_result(msg)) await channel.subscribe('event', listener) # publish a message using rest client From 3411dc4de57857e3afa23ffdd5d6c62d49225f5a Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 18 Oct 2022 10:48:33 +0100 Subject: [PATCH 051/481] refine public api --- ably/realtime/connection.py | 154 +++++++++++++++++++++++++++++- ably/realtime/realtime.py | 127 ++++++++++++++++++++++-- ably/realtime/realtime_channel.py | 112 +++++++++++++++++++++- 3 files changed, 377 insertions(+), 16 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index fae9e200..aef43646 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -45,7 +45,40 @@ class ProtocolMessageAction(IntEnum): class Connection(AsyncIOEventEmitter): + """Ably Realtime Connection + + Enables the management of a connection to Ably + + Attributes + ---------- + realtime: any + Realtime client + state: str + Connection state + connection_manager: ConnectionManager + Connection manager + + + Methods + ------- + connect() + Establishes a realtime connection + close() + Closes a realtime connection + ping() + Pings a realtime connection + on_state_update(state_change) + Update and emit current state + """ + def __init__(self, realtime): + """Constructs a Connection object. + + Parameters + ---------- + realtime: any + Ably realtime client + """ self.__realtime = realtime self.__state = ConnectionState.CONNECTING if realtime.options.auto_connect else ConnectionState.INITIALIZED self.__connection_manager = ConnectionManager(realtime, self.state) @@ -53,33 +86,95 @@ def __init__(self, realtime): super().__init__() async def connect(self): + """Establishes a realtime connection. + + Causes the connection to open, entering the connecting state + """ await self.__connection_manager.connect() async def close(self): + """Causes the connection to close, entering the closing state. + + Once closed, the library will not attempt to re-establish the + connection without an explicit call to connect() + """ await self.__connection_manager.close() async def ping(self): + """ + Send a ping to the realtime connection + """ return await self.__connection_manager.ping() def on_state_update(self, state_change): + """Update and emit the connection state + """ self.__state = state_change.current self.__realtime.options.loop.call_soon(functools.partial(self.emit, state_change.current, state_change)) @property def state(self): + """Returns connection state""" return self.__state @state.setter def state(self, value): + """Sets connection state""" self.__state = value @property def connection_manager(self): + """Returns connection manager""" return self.__connection_manager class ConnectionManager(AsyncIOEventEmitter): + """Ably Realtime Connection + + Attributes + ---------- + realtime: any + Ably realtime client + initial_state: str + Initial connection state + ably: any + Ably object + state: str + Connection state + + + Methods + ------- + enact_state_change(state, reason=None) + Set new state + connect() + Establishes a realtime connection + close() + Closes a realtime connection + ping() + Pings a realtime connection + connect_impl() + Send a connection to ably websocket + send_close_message() + Send a close protocol message to ably + send_protocol_message(protocol_message) + Send protocol message to ably + setup_ws() + Set up ably websocket connection + ws_read_loop() + Handle response from ably websocket + """ + def __init__(self, realtime, initial_state): + """Constructs a Connection object. + + Parameters + ---------- + realtime: any + Ably realtime client + initial_state: any + Initial connection state + """ self.options = realtime.options self.__ably = realtime self.__state = initial_state @@ -91,11 +186,26 @@ def __init__(self, realtime, initial_state): super().__init__() def enact_state_change(self, state, reason=None): + """Sets new connection state + + Parameters + ---------- + state: any + The current connection state + reason: AblyException, optional + Error object describing the last error received if a connection failure occurs + """ current_state = self.__state self.__state = state self.emit('connectionstate', ConnectionStateChange(current_state, state, reason)) async def connect(self): + """Establishes a realtime connection. + + Explicitly calling connect() is unnecessary unless the autoConnect attribute of the ClientOptions object + is false. Unless already connected or connecting, this method causes the connection to open, entering the + CONNECTING state. + """ if self.__state == ConnectionState.CONNECTED: return @@ -111,6 +221,11 @@ async def connect(self): await self.connect_impl() async def close(self): + """Causes the connection to close, entering the closing state. + + Once closed, the library will not attempt to re-establish the + connection without an explicit call to connect() + """ if self.__state != ConnectionState.CONNECTED: log.warn('Connection.closed called while connection state not connected') self.enact_state_change(ConnectionState.CLOSING) @@ -125,17 +240,27 @@ async def close(self): await self.setup_ws_task async def connect_impl(self): + """Send a connection to ably websocket """ self.setup_ws_task = self.ably.options.loop.create_task(self.setup_ws()) await self.__connected_future self.enact_state_change(ConnectionState.CONNECTED) async def send_close_message(self): - await self.sendProtocolMessage({"action": ProtocolMessageAction.CLOSE}) + """Send a close protocol message to ably""" + await self.send_protocol_message({"action": ProtocolMessageAction.CLOSE}) - async def sendProtocolMessage(self, protocolMessage): - await self.__websocket.send(json.dumps(protocolMessage)) + async def send_protocol_message(self, protocol_message): + """Send protocol message to ably""" + await self.__websocket.send(json.dumps(protocol_message)) async def setup_ws(self): + """Set up ably websocket connection + + Raises + ------ + AblyAuthException + If connection cannot be established + """ headers = HttpUtils.default_headers() async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}', extra_headers=headers) as websocket: @@ -147,6 +272,22 @@ async def setup_ws(self): return async def ping(self): + """Send a ping to the realtime connection + + When connected, sends a heartbeat ping to the Ably server and executes + the callback with any error and the response time in milliseconds when + a heartbeat ping request is echoed from the server. + + Raises + ------ + AblyException + If ping request cannot be sent due to invalid state + + Returns + ------- + float + The response time in milliseconds + """ if self.__ping_future: response = await self.__ping_future return response @@ -155,8 +296,8 @@ async def ping(self): if self.__state in [ConnectionState.CONNECTED, ConnectionState.CONNECTING]: self.__ping_id = helper.get_random_id() ping_start_time = datetime.now().timestamp() - await self.sendProtocolMessage({"action": ProtocolMessageAction.HEARTBEAT, - "id": self.__ping_id}) + await self.send_protocol_message({"action": ProtocolMessageAction.HEARTBEAT, + "id": self.__ping_id}) else: raise AblyException("Cannot send ping request. Calling ping in invalid state", 40000, 400) ping_end_time = datetime.now().timestamp() @@ -164,6 +305,7 @@ async def ping(self): return round(response_time_ms, 2) async def ws_read_loop(self): + """Handle response from ably websocket""" while True: raw = await self.__websocket.recv() msg = json.loads(raw) @@ -205,8 +347,10 @@ async def ws_read_loop(self): @property def ably(self): + """Returns ably client""" return self.__ably @property def state(self): + """Returns channel state""" return self.__state diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 35f711c0..10bdf518 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -10,16 +10,49 @@ class AblyRealtime: - """Ably Realtime Client""" + """ + Ably Realtime Client + + Attributes + ---------- + key: str + A valid ably key string + loop: AbstractEventLoop + asyncio running event loop + auth: Auth + authentication object + options: Options + auth options + connection: Connection + realtime connection object + channels: Channels + realtime channel object + + Methods + ------- + connect() + Establishes a realtime connection + close() + Closes a realtime connection + ping() + Pings a realtime connection + """ def __init__(self, key=None, loop=None, **kwargs): - """Create an AblyRealtime instance. - - :Parameters: - **Credentials** - - `key`: a valid ably key string + """Constructs a RealtimeClient object using an Ably API key or token string. + + Parameters + ---------- + key: str + A valid ably key string + loop: AbstractEventLoop, optional + asyncio running event loop + + Raises + ------ + ValueError + If no authentication key is not provided """ - if loop is None: try: loop = asyncio.get_running_loop() @@ -41,49 +74,125 @@ def __init__(self, key=None, loop=None, **kwargs): asyncio.ensure_future(self.connection.connection_manager.connect_impl()) async def connect(self): + """Establishes a realtime connection. + + Explicitly calling connect() is unnecessary unless the autoConnect attribute of the ClientOptions object + is false. Unless already connected or connecting, this method causes the connection to open, entering the + CONNECTING state. + """ await self.connection.connect() async def close(self): + """Causes the connection to close, entering the closing state. + + Once closed, the library will not attempt to re-establish the + connection without an explicit call to connect() + """ await self.connection.close() async def ping(self): + """Send a ping to the realtime connection + + When connected, sends a heartbeat ping to the Ably server and executes + the callback with any error and the response time in milliseconds when + a heartbeat ping request is echoed from the server. + + Returns + ------- + float + The response time in milliseconds + """ return await self.connection.ping() @property def auth(self): + """Returns the auth object""" return self.__auth @property def options(self): + """Returns the auth options object""" return self.__options @property def connection(self): - """Establish realtime connection""" + """Returns the realtime connection object""" return self.__connection @property def channels(self): + """Returns the realtime channel object""" return self.__channels class Channels: + """ + Establish ably realtime channel + + Attributes + ---------- + realtime: any + Ably realtime client object + + Methods + ------- + get(name) + Gets a channel + release(name) + Releases a channel + on_channel_message(msg) + Receives message on a channel + """ + def __init__(self, realtime): + """Initial a realtime channel using the realtime object + + Parameters + ---------- + realtime: any + Ably realtime client object + """ self.all = {} self.__realtime = realtime def get(self, name): + """Creates a new RealtimeChannel object, or returns the existing channel object. + + Parameters + ---------- + + name: str + Channel name + """ if not self.all.get(name): self.all[name] = RealtimeChannel(self.__realtime, name) return self.all[name] def release(self, name): + """Releases a RealtimeChannel object, deleting it, and enabling it to be garbage collected + + It also removes any listeners associated with the channel. + To release a channel, the channel state must be INITIALIZED, DETACHED, or FAILED. + + + Parameters + ---------- + name: str + Channel name + """ if not self.all.get(name): return del self.all[name] def on_channel_message(self, msg): + """Receives message on a realtime channel + + Parameters + ---------- + msg: str + Channel message to receive + """ channel = self.all.get(msg.get('channel')) if not channel: - log.warning('Channel message recieved but no channel instance found') + log.warning('Channel message received but no channel instance found') channel.on_message(msg) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 9d949d97..07ba9611 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -21,7 +21,46 @@ class ChannelState(Enum): class RealtimeChannel(AsyncIOEventEmitter): + """ + Ably Realtime Channel + + Attributes + ---------- + realtime: any + Ably realtime client + name: str + Channel name + state: str + Channel state + + Methods + ------- + attach() + Attach to channel + detach() + Detach from channel + subscribe(*args) + Subscribe to a channel + unsubscribe() + Unsubscribe from a channel + on_message(msg) + Emit channel message + set_state(state) + Set channel state + """ + def __init__(self, realtime, name): + """Constructs a Realtime channel object. + + Parameters + ---------- + realtime: any + Ably realtime client + name: str + Channel name + state: str + Channel state + """ self.__name = name self.__attach_future = None self.__detach_future = None @@ -32,6 +71,16 @@ def __init__(self, realtime, name): super().__init__() async def attach(self): + """Attach to channel + + Attach to this channel ensuring the channel is created in the Ably system and all messages published + on the channel are received by any channel listeners registered using subscribe + + Raises + ------ + AblyException + If unable to attach channel + """ # RTL4a - if channel is attached do nothing if self.state == ChannelState.ATTACHED: return @@ -58,7 +107,7 @@ async def attach(self): await self.__realtime.connect() self.__attach_future = asyncio.Future() - await self.__realtime.connection.connection_manager.sendProtocolMessage( + await self.__realtime.connection.connection_manager.send_protocol_message( { "action": ProtocolMessageAction.ATTACH, "channel": self.name, @@ -68,6 +117,17 @@ async def attach(self): self.set_state(ChannelState.ATTACHED) async def detach(self): + """Detach from channel + + Any resulting channel state change is emitted to any listeners registered + Once all clients globally have detached from the channel, the channel will be released + in the Ably service within two minutes. + + Raises + ------ + AblyException + If unable to detach channel + """ # RTL5g - raise exception if state invalid if self.__realtime.connection.state in [ConnectionState.CLOSING, ConnectionState.FAILED]: raise AblyException( @@ -94,7 +154,7 @@ async def detach(self): await self.__realtime.connect() self.__detach_future = asyncio.Future() - await self.__realtime.connection.connection_manager.sendProtocolMessage( + await self.__realtime.connection.connection_manager.send_protocol_message( { "action": ProtocolMessageAction.DETACH, "channel": self.name, @@ -104,6 +164,22 @@ async def detach(self): self.set_state(ChannelState.DETACHED) async def subscribe(self, *args): + """Subscribe to a channel + + Registers a listener for messages on the channel. + + Parameters + ---------- + *args: event, listener, optional + Subscribe event and listener + + Raises + ------ + AblyException + If unable to subscribe to a channel due to invalid connection state + ValueError + If no valid subscribe arguments are passed + """ if isinstance(args[0], str): event = args[0] if not args[1]: @@ -137,6 +213,22 @@ async def subscribe(self, *args): await self.attach() def unsubscribe(self, *args): + """Unsubscribe from a channel + + Deregister the given listener for (for any/all event names). + This removes an earlier event-specific subscription. + + Parameters + ---------- + *args: event, listener, optional + Subscribe event and listener + + Raises + ------ + ValueError + If no valid unsubscribe arguments are passed, no listener or listener is not a function + or coroutine + """ if len(args) == 0: event = None listener = None @@ -162,6 +254,13 @@ def unsubscribe(self, *args): self.__all_messages_emitter.remove_listener('message', listener) def on_message(self, msg): + """Emit channel message + + Parameters + ---------- + msg: str + Channel message + """ action = msg.get('action') if action == ProtocolMessageAction.ATTACHED: if self.__attach_future: @@ -178,13 +277,22 @@ def on_message(self, msg): self.__all_messages_emitter.emit('message', message) def set_state(self, state): + """Set channel state + + Parameters + ---------- + state: str + New channel state + """ self.__state = state self.emit(state) @property def name(self): + """Returns channel name""" return self.__name @property def state(self): + """Returns channel state""" return self.__state From f97078ee6b58eb6969eca76f225cd21e0ac9c261 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 18 Oct 2022 14:08:31 +0100 Subject: [PATCH 052/481] undocument internal apis --- ably/realtime/connection.py | 105 ++---------------------------------- ably/realtime/realtime.py | 5 ++ 2 files changed, 8 insertions(+), 102 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index aef43646..f985ce30 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -67,8 +67,6 @@ class Connection(AsyncIOEventEmitter): Closes a realtime connection ping() Pings a realtime connection - on_state_update(state_change) - Update and emit current state """ def __init__(self, realtime): @@ -82,7 +80,7 @@ def __init__(self, realtime): self.__realtime = realtime self.__state = ConnectionState.CONNECTING if realtime.options.auto_connect else ConnectionState.INITIALIZED self.__connection_manager = ConnectionManager(realtime, self.state) - self.__connection_manager.on('connectionstate', self.on_state_update) + self.__connection_manager.on('connectionstate', self.__on_state_update) super().__init__() async def connect(self): @@ -106,15 +104,13 @@ async def ping(self): """ return await self.__connection_manager.ping() - def on_state_update(self, state_change): - """Update and emit the connection state - """ + def __on_state_update(self, state_change): self.__state = state_change.current self.__realtime.options.loop.call_soon(functools.partial(self.emit, state_change.current, state_change)) @property def state(self): - """Returns connection state""" + """The current Channel state of the channel""" return self.__state @state.setter @@ -124,57 +120,11 @@ def state(self, value): @property def connection_manager(self): - """Returns connection manager""" return self.__connection_manager class ConnectionManager(AsyncIOEventEmitter): - """Ably Realtime Connection - - Attributes - ---------- - realtime: any - Ably realtime client - initial_state: str - Initial connection state - ably: any - Ably object - state: str - Connection state - - - Methods - ------- - enact_state_change(state, reason=None) - Set new state - connect() - Establishes a realtime connection - close() - Closes a realtime connection - ping() - Pings a realtime connection - connect_impl() - Send a connection to ably websocket - send_close_message() - Send a close protocol message to ably - send_protocol_message(protocol_message) - Send protocol message to ably - setup_ws() - Set up ably websocket connection - ws_read_loop() - Handle response from ably websocket - """ - def __init__(self, realtime, initial_state): - """Constructs a Connection object. - - Parameters - ---------- - realtime: any - Ably realtime client - initial_state: any - Initial connection state - """ self.options = realtime.options self.__ably = realtime self.__state = initial_state @@ -186,26 +136,11 @@ def __init__(self, realtime, initial_state): super().__init__() def enact_state_change(self, state, reason=None): - """Sets new connection state - - Parameters - ---------- - state: any - The current connection state - reason: AblyException, optional - Error object describing the last error received if a connection failure occurs - """ current_state = self.__state self.__state = state self.emit('connectionstate', ConnectionStateChange(current_state, state, reason)) async def connect(self): - """Establishes a realtime connection. - - Explicitly calling connect() is unnecessary unless the autoConnect attribute of the ClientOptions object - is false. Unless already connected or connecting, this method causes the connection to open, entering the - CONNECTING state. - """ if self.__state == ConnectionState.CONNECTED: return @@ -221,11 +156,6 @@ async def connect(self): await self.connect_impl() async def close(self): - """Causes the connection to close, entering the closing state. - - Once closed, the library will not attempt to re-establish the - connection without an explicit call to connect() - """ if self.__state != ConnectionState.CONNECTED: log.warn('Connection.closed called while connection state not connected') self.enact_state_change(ConnectionState.CLOSING) @@ -240,27 +170,17 @@ async def close(self): await self.setup_ws_task async def connect_impl(self): - """Send a connection to ably websocket """ self.setup_ws_task = self.ably.options.loop.create_task(self.setup_ws()) await self.__connected_future self.enact_state_change(ConnectionState.CONNECTED) async def send_close_message(self): - """Send a close protocol message to ably""" await self.send_protocol_message({"action": ProtocolMessageAction.CLOSE}) async def send_protocol_message(self, protocol_message): - """Send protocol message to ably""" await self.__websocket.send(json.dumps(protocol_message)) async def setup_ws(self): - """Set up ably websocket connection - - Raises - ------ - AblyAuthException - If connection cannot be established - """ headers = HttpUtils.default_headers() async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}', extra_headers=headers) as websocket: @@ -272,22 +192,6 @@ async def setup_ws(self): return async def ping(self): - """Send a ping to the realtime connection - - When connected, sends a heartbeat ping to the Ably server and executes - the callback with any error and the response time in milliseconds when - a heartbeat ping request is echoed from the server. - - Raises - ------ - AblyException - If ping request cannot be sent due to invalid state - - Returns - ------- - float - The response time in milliseconds - """ if self.__ping_future: response = await self.__ping_future return response @@ -305,7 +209,6 @@ async def ping(self): return round(response_time_ms, 2) async def ws_read_loop(self): - """Handle response from ably websocket""" while True: raw = await self.__websocket.recv() msg = json.loads(raw) @@ -347,10 +250,8 @@ async def ws_read_loop(self): @property def ably(self): - """Returns ably client""" return self.__ably @property def state(self): - """Returns channel state""" return self.__state diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 10bdf518..16b8dd17 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -97,6 +97,11 @@ async def ping(self): the callback with any error and the response time in milliseconds when a heartbeat ping request is echoed from the server. + Raises + ------ + AblyException + If ping request cannot be sent due to invalid state + Returns ------- float From 7c3af2dad2ea15cc7612bc080c0dc64b83509776 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Thu, 20 Oct 2022 10:07:50 +0100 Subject: [PATCH 053/481] remove documentation on private methods --- ably/realtime/connection.py | 16 ++-------- ably/realtime/realtime.py | 30 +++--------------- ably/realtime/realtime_channel.py | 51 +++++++++++-------------------- 3 files changed, 24 insertions(+), 73 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index f985ce30..b820ed5f 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -51,12 +51,8 @@ class Connection(AsyncIOEventEmitter): Attributes ---------- - realtime: any - Realtime client state: str Connection state - connection_manager: ConnectionManager - Connection manager Methods @@ -70,13 +66,6 @@ class Connection(AsyncIOEventEmitter): """ def __init__(self, realtime): - """Constructs a Connection object. - - Parameters - ---------- - realtime: any - Ably realtime client - """ self.__realtime = realtime self.__state = ConnectionState.CONNECTING if realtime.options.auto_connect else ConnectionState.INITIALIZED self.__connection_manager = ConnectionManager(realtime, self.state) @@ -110,12 +99,11 @@ def __on_state_update(self, state_change): @property def state(self): - """The current Channel state of the channel""" + """The current connection state of the connection""" return self.__state @state.setter def state(self, value): - """Sets connection state""" self.__state = value @property @@ -246,7 +234,7 @@ async def ws_read_loop(self): ProtocolMessageAction.DETACHED, ProtocolMessageAction.MESSAGE ): - self.ably.channels.on_channel_message(msg) + self.ably.channels._on_channel_message(msg) @property def ably(self): diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 16b8dd17..db77222f 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -39,7 +39,7 @@ class AblyRealtime: """ def __init__(self, key=None, loop=None, **kwargs): - """Constructs a RealtimeClient object using an Ably API key or token string. + """Constructs a RealtimeClient object using an Ably API key. Parameters ---------- @@ -131,13 +131,7 @@ def channels(self): class Channels: - """ - Establish ably realtime channel - - Attributes - ---------- - realtime: any - Ably realtime client object + """Creates and destroys RealtimeChannel objects. Methods ------- @@ -145,18 +139,9 @@ class Channels: Gets a channel release(name) Releases a channel - on_channel_message(msg) - Receives message on a channel """ def __init__(self, realtime): - """Initial a realtime channel using the realtime object - - Parameters - ---------- - realtime: any - Ably realtime client object - """ self.all = {} self.__realtime = realtime @@ -189,15 +174,8 @@ def release(self, name): return del self.all[name] - def on_channel_message(self, msg): - """Receives message on a realtime channel - - Parameters - ---------- - msg: str - Channel message to receive - """ + def _on_channel_message(self, msg): channel = self.all.get(msg.get('channel')) if not channel: log.warning('Channel message received but no channel instance found') - channel.on_message(msg) + channel._on_message(msg) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 07ba9611..8d40eed2 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -26,8 +26,6 @@ class RealtimeChannel(AsyncIOEventEmitter): Attributes ---------- - realtime: any - Ably realtime client name: str Channel name state: str @@ -43,24 +41,9 @@ class RealtimeChannel(AsyncIOEventEmitter): Subscribe to a channel unsubscribe() Unsubscribe from a channel - on_message(msg) - Emit channel message - set_state(state) - Set channel state """ def __init__(self, realtime, name): - """Constructs a Realtime channel object. - - Parameters - ---------- - realtime: any - Ably realtime client - name: str - Channel name - state: str - Channel state - """ self.__name = name self.__attach_future = None self.__detach_future = None @@ -167,12 +150,22 @@ async def subscribe(self, *args): """Subscribe to a channel Registers a listener for messages on the channel. + The caller supplies a listener function, which is called + each time one or more messages arrives on the channel. + + The function resolves once the channel is attached. Parameters ---------- *args: event, listener, optional Subscribe event and listener + arg1(event): str + Subscribe to messages with the given event name + + arg2(listener): any + Subscribe to all messages on the channel + Raises ------ AblyException @@ -221,7 +214,13 @@ def unsubscribe(self, *args): Parameters ---------- *args: event, listener, optional - Subscribe event and listener + Unsubscribe event and listener + + arg1(event): str + Unsubscribe to messages with the given event name + + arg2(listener): any + Unsubscribe to all messages on the channel Raises ------ @@ -253,14 +252,7 @@ def unsubscribe(self, *args): else: self.__all_messages_emitter.remove_listener('message', listener) - def on_message(self, msg): - """Emit channel message - - Parameters - ---------- - msg: str - Channel message - """ + def _on_message(self, msg): action = msg.get('action') if action == ProtocolMessageAction.ATTACHED: if self.__attach_future: @@ -277,13 +269,6 @@ def on_message(self, msg): self.__all_messages_emitter.emit('message', message) def set_state(self, state): - """Set channel state - - Parameters - ---------- - state: str - New channel state - """ self.__state = state self.emit(state) From 7af4ac24ac6fe29559039755ff0dd4678c17eafd Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Thu, 20 Oct 2022 13:39:24 +0100 Subject: [PATCH 054/481] review: update docstring documentation --- ably/realtime/connection.py | 31 ++++++++++++++++++++-------- ably/realtime/realtime.py | 31 ++++------------------------ ably/realtime/realtime_channel.py | 22 ++++++++++++-------- test/ably/realtimeconnection_test.py | 8 +++---- 4 files changed, 43 insertions(+), 49 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index b820ed5f..bf3ffe22 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -68,8 +68,8 @@ class Connection(AsyncIOEventEmitter): def __init__(self, realtime): self.__realtime = realtime self.__state = ConnectionState.CONNECTING if realtime.options.auto_connect else ConnectionState.INITIALIZED - self.__connection_manager = ConnectionManager(realtime, self.state) - self.__connection_manager.on('connectionstate', self.__on_state_update) + self.__connection_manager = ConnectionManager(self.__realtime, self.state) + self.__connection_manager.on('connectionstate', self._on_state_update) super().__init__() async def connect(self): @@ -88,12 +88,25 @@ async def close(self): await self.__connection_manager.close() async def ping(self): - """ - Send a ping to the realtime connection + """Send a ping to the realtime connection + + When connected, sends a heartbeat ping to the Ably server and executes + the callback with any error and the response time in milliseconds when + a heartbeat ping request is echoed from the server. + + Raises + ------ + AblyException + If ping request cannot be sent due to invalid state + + Returns + ------- + float + The response time in milliseconds """ return await self.__connection_manager.ping() - def __on_state_update(self, state_change): + def _on_state_update(self, state_change): self.__state = state_change.current self.__realtime.options.loop.call_soon(functools.partial(self.emit, state_change.current, state_change)) @@ -158,7 +171,7 @@ async def close(self): await self.setup_ws_task async def connect_impl(self): - self.setup_ws_task = self.ably.options.loop.create_task(self.setup_ws()) + self.setup_ws_task = self.__ably.options.loop.create_task(self.setup_ws()) await self.__connected_future self.enact_state_change(ConnectionState.CONNECTED) @@ -170,10 +183,10 @@ async def send_protocol_message(self, protocol_message): async def setup_ws(self): headers = HttpUtils.default_headers() - async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}', + async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.__ably.key}', extra_headers=headers) as websocket: self.__websocket = websocket - task = self.ably.options.loop.create_task(self.ws_read_loop()) + task = self.__ably.options.loop.create_task(self.ws_read_loop()) try: await task except AblyAuthException: @@ -234,7 +247,7 @@ async def ws_read_loop(self): ProtocolMessageAction.DETACHED, ProtocolMessageAction.MESSAGE ): - self.ably.channels._on_channel_message(msg) + self.__ably.channels._on_channel_message(msg) @property def ably(self): diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index db77222f..5ddc2e1e 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -15,14 +15,12 @@ class AblyRealtime: Attributes ---------- - key: str - A valid ably key string loop: AbstractEventLoop asyncio running event loop auth: Auth authentication object options: Options - auth options + auth options object connection: Connection realtime connection object channels: Channels @@ -31,11 +29,9 @@ class AblyRealtime: Methods ------- connect() - Establishes a realtime connection + Establishes the realtime connection close() - Closes a realtime connection - ping() - Pings a realtime connection + Closes the realtime connection """ def __init__(self, key=None, loop=None, **kwargs): @@ -44,7 +40,7 @@ def __init__(self, key=None, loop=None, **kwargs): Parameters ---------- key: str - A valid ably key string + A valid ably API key string loop: AbstractEventLoop, optional asyncio running event loop @@ -90,25 +86,6 @@ async def close(self): """ await self.connection.close() - async def ping(self): - """Send a ping to the realtime connection - - When connected, sends a heartbeat ping to the Ably server and executes - the callback with any error and the response time in milliseconds when - a heartbeat ping request is echoed from the server. - - Raises - ------ - AblyException - If ping request cannot be sent due to invalid state - - Returns - ------- - float - The response time in milliseconds - """ - return await self.connection.ping() - @property def auth(self): """Returns the auth object""" diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 8d40eed2..34c01770 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -38,9 +38,9 @@ class RealtimeChannel(AsyncIOEventEmitter): detach() Detach from channel subscribe(*args) - Subscribe to a channel - unsubscribe() - Unsubscribe from a channel + Subscribe to messages on a channel + unsubscribe(*args) + Unsubscribe to messages from a channel """ def __init__(self, realtime, name): @@ -157,15 +157,17 @@ async def subscribe(self, *args): Parameters ---------- - *args: event, listener, optional + *args: event, listener Subscribe event and listener - arg1(event): str + arg1(event): str, optional Subscribe to messages with the given event name - arg2(listener): any + arg2(listener): callable Subscribe to all messages on the channel + When no event is provided, arg1 is used as the listener. + Raises ------ AblyException @@ -213,15 +215,17 @@ def unsubscribe(self, *args): Parameters ---------- - *args: event, listener, optional + *args: event, listener Unsubscribe event and listener - arg1(event): str + arg1(event): str, optional Unsubscribe to messages with the given event name - arg2(listener): any + arg2(listener): callable Unsubscribe to all messages on the channel + When no event is provided, arg1 is used as the listener. + Raises ------ ValueError diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 41ab1d5d..72647a31 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -45,7 +45,7 @@ async def test_auth_invalid_key(self): async def test_connection_ping_connected(self): ably = await RestSetup.get_ably_realtime() await ably.connect() - response_time_ms = await ably.ping() + response_time_ms = await ably.connection.ping() assert response_time_ms is not None assert type(response_time_ms) is float await ably.close() @@ -54,7 +54,7 @@ async def test_connection_ping_initialized(self): ably = await RestSetup.get_ably_realtime(auto_connect=False) assert ably.connection.state == ConnectionState.INITIALIZED with pytest.raises(AblyException) as exception: - await ably.ping() + await ably.connection.ping() assert exception.value.code == 400 assert exception.value.status_code == 40000 @@ -64,7 +64,7 @@ async def test_connection_ping_failed(self): await ably.connect() assert ably.connection.state == ConnectionState.FAILED with pytest.raises(AblyException) as exception: - await ably.ping() + await ably.connection.ping() assert exception.value.code == 400 assert exception.value.status_code == 40000 await ably.close() @@ -75,7 +75,7 @@ async def test_connection_ping_closed(self): assert ably.connection.state == ConnectionState.CONNECTED await ably.close() with pytest.raises(AblyException) as exception: - await ably.ping() + await ably.connection.ping() assert exception.value.code == 400 assert exception.value.status_code == 40000 From 3e75eebb91d296c3571ad925693b451a325fde58 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 20 Oct 2022 18:07:55 +0100 Subject: [PATCH 055/481] docs: add param description for auto_connect --- ably/realtime/realtime.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 5ddc2e1e..87276053 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -43,6 +43,10 @@ def __init__(self, key=None, loop=None, **kwargs): A valid ably API key string loop: AbstractEventLoop, optional asyncio running event loop + auto_connect: bool + When true, the client connects to Ably as soon as it is instantiated. + You can set this to false and explicitly connect to Ably using the + connect() method. The default is true. Raises ------ From 9c12034ab4753081a266f0d144561f101b0688f5 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 24 Oct 2022 09:19:02 +0100 Subject: [PATCH 056/481] update readme with realtime doc --- README.md | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 35830cc3..ee5ae041 100644 --- a/README.md +++ b/README.md @@ -54,10 +54,11 @@ introduced by version 1.2.0. ## Usage +### Using the Rest API + All examples assume a client and/or channel has been created in one of the following ways: With closing the client manually: - ```python from ably import AblyRest @@ -196,6 +197,60 @@ await client.time() await client.close() ``` +### Using the Realtime API +The python realtime API currently only supports authentication with ably API key. +#### Creating a client +```python +from ably import AblyRealtime + +async def main(): + client = AblyRealtime('api:key') + channel = client.channels.get('channel_name) +``` + +#### Subscribing to a channel for event +```python +message_future = asyncio.Future() + +def listener(message): + message_future.set_result(message) + +channel.subscribe('event', listener) + +# Subscribe using only listener +await channel.subscribe(listener) +``` + +#### Unsubscribing from a channel for event +```python +# unsubscribe the listener from the channel +channel.unsubscribe('event', listener) + +# unsubscribe all listeners from the channel +channel.unsubscribe() +``` + +#### Attach a channel +```python +await channel.attach() +``` +#### Detach from a channel +```python +await channel.detach() +``` + +#### Managing a connection +```python +# Establish a realtime connection. +# Explicitly calling connect() is unnecessary unless the autoConnect attribute of the ClientOptions object is false +await client.connect() + +# Close a connection +await client.close() + +# Ping a connection +await client.connection.ping() +``` ## Resources Visit https://ably.com/docs for a complete API reference and more examples. @@ -210,7 +265,7 @@ for the set of versions that currently undergo CI testing. ## Known Limitations -Currently, this SDK only supports [Ably REST](https://ably.com/docs/rest). +Currently, this SDK only supports [Ably REST](https://ably.com/docs/rest) and subscribe/unsubscribe functionality of [Ably Realtime](https://ably.com/docs/realtime) as documented above. However, you can use the [MQTT adapter](https://ably.com/docs/mqtt) to implement [Ably's Realtime](https://ably.com/docs/realtime) features using Python. See [our roadmap for this SDK](roadmap.md) for more information. From 0e9513146fcdfdfa5327c9c4c3be13ad0f30c9f8 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 20 Oct 2022 10:14:09 +0100 Subject: [PATCH 057/481] chore: improve logging in realtime.py --- ably/realtime/realtime.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 87276053..1b9bfe4f 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -64,6 +64,8 @@ def __init__(self, key=None, loop=None, **kwargs): else: raise ValueError("Key is missing. Provide an API key.") + log.info(f'Realtime client initialised with options: {vars(options)}') + self.__auth = Auth(self, options) self.__options = options self.key = key @@ -80,14 +82,15 @@ async def connect(self): is false. Unless already connected or connecting, this method causes the connection to open, entering the CONNECTING state. """ + log.info('Realtime.connect() called') await self.connection.connect() async def close(self): """Causes the connection to close, entering the closing state. - Once closed, the library will not attempt to re-establish the connection without an explicit call to connect() """ + log.info('Realtime.close() called') await self.connection.close() @property @@ -156,7 +159,20 @@ def release(self, name): del self.all[name] def _on_channel_message(self, msg): + channel_name = msg.get('channel') + if not channel_name: + log.error( + 'Channels.on_channel_message()', + f'received event without channel, action = {msg.get("action")}' + ) + return + channel = self.all.get(msg.get('channel')) if not channel: - log.warning('Channel message received but no channel instance found') + log.warning( + 'Channels.on_channel_message()', + f'receieved event for non-existent channel: {channel_name}' + ) + return + channel._on_message(msg) From be5be65f7e9caa2ec38305dfe7cbdc45d4aaad95 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 20 Oct 2022 10:21:21 +0100 Subject: [PATCH 058/481] chore: add detailed logging to connection.py --- ably/realtime/connection.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index bf3ffe22..6ea4db8d 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -107,6 +107,7 @@ async def ping(self): return await self.__connection_manager.ping() def _on_state_update(self, state_change): + log.info(f'Connection state changing from {self.state} to {state_change.current}') self.__state = state_change.current self.__realtime.options.loop.call_soon(functools.partial(self.emit, state_change.current, state_change)) @@ -158,14 +159,14 @@ async def connect(self): async def close(self): if self.__state != ConnectionState.CONNECTED: - log.warn('Connection.closed called while connection state not connected') + log.warning('Connection.closed called while connection state not connected') self.enact_state_change(ConnectionState.CLOSING) self.__closed_future = asyncio.Future() if self.__websocket and self.__state != ConnectionState.FAILED: await self.send_close_message() await self.__closed_future else: - log.warn('Connection.closed called while connection already closed or not established') + log.warning('Connection.closed called while connection already closed or not established') self.enact_state_change(ConnectionState.CLOSED) if self.setup_ws_task: await self.setup_ws_task @@ -178,13 +179,17 @@ async def connect_impl(self): async def send_close_message(self): await self.send_protocol_message({"action": ProtocolMessageAction.CLOSE}) - async def send_protocol_message(self, protocol_message): - await self.__websocket.send(json.dumps(protocol_message)) + async def send_protocol_message(self, protocolMessage): + raw_msg = json.dumps(protocolMessage) + log.info('send_protocol_message(): sending {raw_msg}') + await self.__websocket.send(raw_msg) async def setup_ws(self): headers = HttpUtils.default_headers() - async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.__ably.key}', - extra_headers=headers) as websocket: + ws_url = f'wss://{self.options.realtime_host}?key={self.__ably.key}' + log.info(f'setup_ws(): attempting to connect to {ws_url}') + async with websockets.connect(ws_url, extra_headers=headers) as websocket: + log.info(f'setup_ws(): connection established to {ws_url}') self.__websocket = websocket task = self.__ably.options.loop.create_task(self.ws_read_loop()) try: @@ -213,6 +218,7 @@ async def ws_read_loop(self): while True: raw = await self.__websocket.recv() msg = json.loads(raw) + log.info(f'ws_read_loop(): receieved protocol message: {msg}') action = msg['action'] if action == ProtocolMessageAction.CONNECTED: # CONNECTED if self.__connected_future: From ce86eb2f567487a1f72195b864482fd9c7a14255 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 20 Oct 2022 10:26:03 +0100 Subject: [PATCH 059/481] chore: add detailed logging to realtime_channel.py --- ably/realtime/realtime_channel.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 34c01770..12d2bc95 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -64,6 +64,9 @@ async def attach(self): AblyException If unable to attach channel """ + + log.info(f'RealtimeChannel.attach() called, channel = {self.name}') + # RTL4a - if channel is attached do nothing if self.state == ChannelState.ATTACHED: return @@ -111,6 +114,9 @@ async def detach(self): AblyException If unable to detach channel """ + + log.info(f'RealtimeChannel.detach() called, channel = {self.name}') + # RTL5g - raise exception if state invalid if self.__realtime.connection.state in [ConnectionState.CLOSING, ConnectionState.FAILED]: raise AblyException( @@ -188,6 +194,8 @@ async def subscribe(self, *args): else: raise ValueError('invalid subscribe arguments') + log.info(f'RealtimeChannel.subscribe called, channel = {self.name}, event = {event}') + if self.__realtime.connection.state == ConnectionState.CONNECTING: await self.__realtime.connection.connect() elif self.__realtime.connection.state != ConnectionState.CONNECTED: @@ -248,6 +256,8 @@ def unsubscribe(self, *args): else: raise ValueError('invalid unsubscribe arguments') + log.info(f'RealtimeChannel.unsubscribe called, channel = {self.name}, event = {event}') + if listener is None: self.__message_emitter.remove_all_listeners() self.__all_messages_emitter.remove_all_listeners() From 4b1f07d2da4726f0c62928b73354aec9cffb9181 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 25 Oct 2022 10:52:51 +0100 Subject: [PATCH 060/481] review: update readme --- README.md | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index ee5ae041..df8ee190 100644 --- a/README.md +++ b/README.md @@ -205,23 +205,27 @@ from ably import AblyRealtime async def main(): client = AblyRealtime('api:key') - channel = client.channels.get('channel_name) ``` -#### Subscribing to a channel for event +#### Connecting to a channel +```python +channel = client.channels.get('channel_name) +``` +#### Subscribing to messages on a channel ```python -message_future = asyncio.Future() def listener(message): - message_future.set_result(message) + print(message.data) -channel.subscribe('event', listener) +# Subscribe to messages with the 'event' name +await channel.subscribe('event', listener) -# Subscribe using only listener +# Subscribe to all messages on a channel await channel.subscribe(listener) ``` +Note that `channel.subscribe` is a coroutine function and will resolve when the channel is attached -#### Unsubscribing from a channel for event +#### Unsubscribing from messages on a channel ```python # unsubscribe the listener from the channel channel.unsubscribe('event', listener) @@ -230,7 +234,7 @@ channel.unsubscribe('event', listener) channel.unsubscribe() ``` -#### Attach a channel +#### Attach to a channel ```python await channel.attach() ``` @@ -248,8 +252,8 @@ await client.connect() # Close a connection await client.close() -# Ping a connection -await client.connection.ping() +# Send a ping +time_in_ms = await client.connection.ping() ``` ## Resources @@ -265,7 +269,7 @@ for the set of versions that currently undergo CI testing. ## Known Limitations -Currently, this SDK only supports [Ably REST](https://ably.com/docs/rest) and subscribe/unsubscribe functionality of [Ably Realtime](https://ably.com/docs/realtime) as documented above. +Currently, this SDK only supports [Ably REST](https://ably.com/docs/rest) and realtime message subscription as documented above. However, you can use the [MQTT adapter](https://ably.com/docs/mqtt) to implement [Ably's Realtime](https://ably.com/docs/realtime) features using Python. See [our roadmap for this SDK](roadmap.md) for more information. From a059ca92098990ee7bb0238bc81fdf950bfaa216 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 31 Oct 2022 12:00:58 +0000 Subject: [PATCH 061/481] add info on connection state --- README.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index df8ee190..60e5aa32 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,7 @@ await client.close() ``` ### Using the Realtime API -The python realtime API currently only supports authentication with ably API key. +The python realtime client currently only supports basic authentication. #### Creating a client ```python from ably import AblyRealtime @@ -207,9 +207,9 @@ async def main(): client = AblyRealtime('api:key') ``` -#### Connecting to a channel +#### Get a realtime channel instance ```python -channel = client.channels.get('channel_name) +channel = client.channels.get('channel_name') ``` #### Subscribing to messages on a channel ```python @@ -234,6 +234,16 @@ channel.unsubscribe('event', listener) channel.unsubscribe() ``` +#### Subscribe to connection state change +```python +from ably.realtime.connection import ConnectionState +# subscribe to failed connection state +client.connection.on(ConnectionState.FAILED, listener) + +# subscribe to connected connection state +client.connection.on(ConnectionState.CONNECTED, listener) +``` + #### Attach to a channel ```python await channel.attach() From ffcaf056a0508a94822c71687fa9220316a5b917 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 24 Oct 2022 14:31:04 +0100 Subject: [PATCH 062/481] add environment client option --- test/ably/restsetup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ably/restsetup.py b/test/ably/restsetup.py index efab592d..5cd73c1d 100644 --- a/test/ably/restsetup.py +++ b/test/ably/restsetup.py @@ -15,7 +15,7 @@ tls = (os.environ.get('ABLY_TLS') or "true").lower() == "true" host = os.environ.get('ABLY_HOST', 'sandbox-rest.ably.io') -realtime_host = 'sandbox-realtime.ably.io' +realtime_host = os.environ.get('ABLY_HOST', 'sandbox-realtime.ably.io') environment = os.environ.get('ABLY_ENV') port = 80 From fd07f93a5b36127a23d15de5e91c0aca8534c8d1 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 25 Oct 2022 14:16:54 +0100 Subject: [PATCH 063/481] add environment option --- ably/types/options.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ably/types/options.py b/ably/types/options.py index 6d254440..00ea4de3 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -30,8 +30,10 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, from ably import api_version idempotent_rest_publishing = api_version >= '1.2' + if environment is None: + environment = Defaults.environment if realtime_host is None: - realtime_host = Defaults.realtime_host + realtime_host = f'{environment}-{Defaults.realtime_host}' self.__client_id = client_id self.__log_level = log_level From 8d3d2cabe68d15c37ad7395ac98c593f877dc5d3 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 1 Nov 2022 12:38:39 +0000 Subject: [PATCH 064/481] refactor: use string-based enums --- ably/realtime/connection.py | 2 +- ably/realtime/realtime_channel.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 6ea4db8d..2c923439 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -15,7 +15,7 @@ log = logging.getLogger(__name__) -class ConnectionState(Enum): +class ConnectionState(str, Enum): INITIALIZED = 'initialized' CONNECTING = 'connecting' CONNECTED = 'connected' diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 12d2bc95..c60fb6fd 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -12,7 +12,7 @@ log = logging.getLogger(__name__) -class ChannelState(Enum): +class ChannelState(str, Enum): INITIALIZED = 'initialized' ATTACHING = 'attaching' ATTACHED = 'attached' From 09a3bb7db2e2c353d1d3d4379cda02cbdab7ee05 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 1 Nov 2022 12:39:08 +0000 Subject: [PATCH 065/481] doc: use string-based enums in usage examples --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 60e5aa32..7479203a 100644 --- a/README.md +++ b/README.md @@ -236,12 +236,11 @@ channel.unsubscribe() #### Subscribe to connection state change ```python -from ably.realtime.connection import ConnectionState -# subscribe to failed connection state -client.connection.on(ConnectionState.FAILED, listener) +# subscribe to 'failed' connection state +client.connection.on('failed', listener) -# subscribe to connected connection state -client.connection.on(ConnectionState.CONNECTED, listener) +# subscribe to 'connected' connection state +client.connection.on('connected', listener) ``` #### Attach to a channel From dbd00356a35c0ec5783afc6c7a71c95cb45d35b5 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 1 Nov 2022 13:56:17 +0000 Subject: [PATCH 066/481] refactor realtime host option --- ably/types/options.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/ably/types/options.py b/ably/types/options.py index 00ea4de3..861833ba 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -30,11 +30,6 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, from ably import api_version idempotent_rest_publishing = api_version >= '1.2' - if environment is None: - environment = Defaults.environment - if realtime_host is None: - realtime_host = f'{environment}-{Defaults.realtime_host}' - self.__client_id = client_id self.__log_level = log_level self.__tls = tls @@ -58,6 +53,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, self.__auto_connect = auto_connect self.__rest_hosts = self.__get_rest_hosts() + self.__realtime_hosts = self.__get_realtime_hosts() @property def client_id(self): @@ -255,11 +251,22 @@ def __get_rest_hosts(self): hosts = hosts[:http_max_retry_count] return hosts + def __get_realtime_hosts(self): + if self.realtime_host is not None: + return self.realtime_host + elif self.environment is not None: + return f'{self.environment}-{Defaults.realtime_host}' + else: + return Defaults.realtime_host + def get_rest_hosts(self): return self.__rest_hosts def get_rest_host(self): return self.__rest_hosts[0] + def get_realtime_host(self): + return self.__realtime_hosts + def get_fallback_rest_hosts(self): return self.__rest_hosts[1:] From 4cfc21254f41f86bffdaa482b4c2a32b558705cc Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 1 Nov 2022 19:10:22 +0000 Subject: [PATCH 067/481] update API documentation with client option --- ably/realtime/realtime.py | 5 +++++ ably/types/options.py | 3 +++ 2 files changed, 8 insertions(+) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 1b9bfe4f..5563a317 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -47,6 +47,11 @@ def __init__(self, key=None, loop=None, **kwargs): When true, the client connects to Ably as soon as it is instantiated. You can set this to false and explicitly connect to Ably using the connect() method. The default is true. + **kwargs: client options + realtime_host: str + The host to connect to. Defaults to `realtime.ably.io` + environment: str + The environment to use. Defaults to `production` Raises ------ diff --git a/ably/types/options.py b/ably/types/options.py index 861833ba..d5f8cc4f 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -26,6 +26,9 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, if environment is not None and rest_host is not None: raise ValueError('specify rest_host or environment, not both') + if environment is not None and realtime_host is not None: + raise ValueError('specify realtime_host or environment, not both') + if idempotent_rest_publishing is None: from ably import api_version idempotent_rest_publishing = api_version >= '1.2' From 761e6bdbf19f507d6fa74bf31576c93d9200fb32 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 2 Nov 2022 10:45:56 +0000 Subject: [PATCH 068/481] update realtime API docstring --- ably/realtime/realtime.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 5563a317..7b46ec67 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -49,9 +49,10 @@ def __init__(self, key=None, loop=None, **kwargs): connect() method. The default is true. **kwargs: client options realtime_host: str - The host to connect to. Defaults to `realtime.ably.io` + Enables a non-default Ably host to be specified for realtime connections. + For development environments only. The default value is realtime.ably.io. environment: str - The environment to use. Defaults to `production` + Enables a custom environment to be used with the Ably service. Defaults to `production` Raises ------ From 707190e5b55b5594d652b4f14f9a55c7bbaa42e6 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 1 Nov 2022 12:18:59 +0000 Subject: [PATCH 069/481] feat: EventEmitter methods with no event argument --- ably/realtime/connection.py | 10 +++--- ably/realtime/realtime_channel.py | 31 ++++++++---------- ably/util/eventemitter.py | 54 +++++++++++++++++++++++++++++++ ably/util/helper.py | 6 ++-- test/ably/eventemitter_test.py | 42 ++++++++++++++++-------- test/ably/realtimechannel_test.py | 37 ++++++++++++++------- 6 files changed, 130 insertions(+), 50 deletions(-) create mode 100644 ably/util/eventemitter.py diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 2c923439..1e194c89 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -5,8 +5,8 @@ import json from ably.http.httputils import HttpUtils from ably.util.exceptions import AblyAuthException, AblyException +from ably.util.eventemitter import EventEmitter from enum import Enum, IntEnum -from pyee.asyncio import AsyncIOEventEmitter from datetime import datetime from ably.util import helper from dataclasses import dataclass @@ -44,7 +44,7 @@ class ProtocolMessageAction(IntEnum): MESSAGE = 15 -class Connection(AsyncIOEventEmitter): +class Connection(EventEmitter): """Ably Realtime Connection Enables the management of a connection to Ably @@ -109,7 +109,7 @@ async def ping(self): def _on_state_update(self, state_change): log.info(f'Connection state changing from {self.state} to {state_change.current}') self.__state = state_change.current - self.__realtime.options.loop.call_soon(functools.partial(self.emit, state_change.current, state_change)) + self.__realtime.options.loop.call_soon(functools.partial(self._emit, state_change.current, state_change)) @property def state(self): @@ -125,7 +125,7 @@ def connection_manager(self): return self.__connection_manager -class ConnectionManager(AsyncIOEventEmitter): +class ConnectionManager(EventEmitter): def __init__(self, realtime, initial_state): self.options = realtime.options self.__ably = realtime @@ -140,7 +140,7 @@ def __init__(self, realtime, initial_state): def enact_state_change(self, state, reason=None): current_state = self.__state self.__state = state - self.emit('connectionstate', ConnectionStateChange(current_state, state, reason)) + self._emit('connectionstate', ConnectionStateChange(current_state, state, reason)) async def connect(self): if self.__state == ConnectionState.CONNECTED: diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index c60fb6fd..9a431be8 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -3,11 +3,11 @@ from ably.realtime.connection import ConnectionState, ProtocolMessageAction from ably.types.message import Message +from ably.util.eventemitter import EventEmitter from ably.util.exceptions import AblyException -from pyee.asyncio import AsyncIOEventEmitter from enum import Enum -from ably.util.helper import is_function_or_coroutine +from ably.util.helper import is_callable_or_coroutine log = logging.getLogger(__name__) @@ -20,7 +20,7 @@ class ChannelState(str, Enum): DETACHED = 'detached' -class RealtimeChannel(AsyncIOEventEmitter): +class RealtimeChannel(EventEmitter): """ Ably Realtime Channel @@ -49,8 +49,7 @@ def __init__(self, realtime, name): self.__detach_future = None self.__realtime = realtime self.__state = ChannelState.INITIALIZED - self.__message_emitter = AsyncIOEventEmitter() - self.__all_messages_emitter = AsyncIOEventEmitter() + self.__message_emitter = EventEmitter() super().__init__() async def attach(self): @@ -185,10 +184,10 @@ async def subscribe(self, *args): event = args[0] if not args[1]: raise ValueError("channel.subscribe called without listener") - if not is_function_or_coroutine(args[1]): + if not is_callable_or_coroutine(args[1]): raise ValueError("subscribe listener must be function or coroutine function") listener = args[1] - elif is_function_or_coroutine(args[0]): + elif is_callable_or_coroutine(args[0]): listener = args[0] event = None else: @@ -211,7 +210,7 @@ async def subscribe(self, *args): if event is not None: self.__message_emitter.on(event, listener) else: - self.__all_messages_emitter.on('message', listener) + self.__message_emitter.on(listener) await self.attach() @@ -247,10 +246,10 @@ def unsubscribe(self, *args): event = args[0] if not args[1]: raise ValueError("channel.unsubscribe called without listener") - if not is_function_or_coroutine(args[1]): + if not is_callable_or_coroutine(args[1]): raise ValueError("unsubscribe listener must be a function or coroutine function") listener = args[1] - elif is_function_or_coroutine(args[0]): + elif is_callable_or_coroutine(args[0]): listener = args[0] event = None else: @@ -259,12 +258,11 @@ def unsubscribe(self, *args): log.info(f'RealtimeChannel.unsubscribe called, channel = {self.name}, event = {event}') if listener is None: - self.__message_emitter.remove_all_listeners() - self.__all_messages_emitter.remove_all_listeners() + self.__message_emitter.off() elif event is not None: - self.__message_emitter.remove_listener(event, listener) + self.__message_emitter.off(event, listener) else: - self.__all_messages_emitter.remove_listener('message', listener) + self.__message_emitter.off(listener) def _on_message(self, msg): action = msg.get('action') @@ -279,12 +277,11 @@ def _on_message(self, msg): elif action == ProtocolMessageAction.MESSAGE: messages = Message.from_encoded_array(msg.get('messages')) for message in messages: - self.__message_emitter.emit(message.name, message) - self.__all_messages_emitter.emit('message', message) + self.__message_emitter._emit(message.name, message) def set_state(self, state): self.__state = state - self.emit(state) + self._emit(state) @property def name(self): diff --git a/ably/util/eventemitter.py b/ably/util/eventemitter.py new file mode 100644 index 00000000..f688ef71 --- /dev/null +++ b/ably/util/eventemitter.py @@ -0,0 +1,54 @@ +from pyee.asyncio import AsyncIOEventEmitter + +from ably.util.helper import is_callable_or_coroutine + +# pyee's event emitter doesn't support attaching a listener to all events +# so to patch it, we create a wrapper which uses two event emitters, one +# is used to listen to all events and this arbitrary string is the event name +# used to emit all events on that listener +_all_event = 'all' + + +def _is_named_event_args(*args): + return len(args) == 2 and is_callable_or_coroutine(args[1]) + + +def _is_all_event_args(*args): + return len(args) == 1 and is_callable_or_coroutine(args[0]) + + +class EventEmitter: + def __init__(self): + self.__named_event_emitter = AsyncIOEventEmitter() + self.__all_event_emitter = AsyncIOEventEmitter() + + def on(self, *args): + if _is_all_event_args(*args): + self.__all_event_emitter.add_listener(_all_event, args[0]) + elif _is_named_event_args(*args): + self.__named_event_emitter.add_listener(args[0], args[1]) + else: + raise ValueError("EventEmitter.on(): invalid args") + + def once(self, *args): + if _is_all_event_args(*args): + self.__all_event_emitter.once(_all_event, args[0]) + elif _is_named_event_args(*args): + self.__named_event_emitter.once(args[0], args[1]) + else: + raise ValueError("EventEmitter.once(): invalid args") + + def off(self, *args): + if len(args) == 0: + self.__all_event_emitter.remove_all_listeners() + self.__named_event_emitter.remove_all_listeners() + elif _is_all_event_args(*args): + self.__all_event_emitter.remove_listener(_all_event, args[0]) + elif _is_named_event_args(*args): + self.__named_event_emitter.remove_listener(args[0], args[1]) + else: + raise ValueError("EventEmitter.once(): invalid args") + + def _emit(self, *args): + self.__named_event_emitter.emit(*args) + self.__all_event_emitter.emit(_all_event, *args[1:]) diff --git a/ably/util/helper.py b/ably/util/helper.py index c3b427ac..cead99d9 100644 --- a/ably/util/helper.py +++ b/ably/util/helper.py @@ -1,6 +1,6 @@ +import inspect import random import string -import types import asyncio @@ -11,5 +11,5 @@ def get_random_id(): return random_id -def is_function_or_coroutine(value): - return isinstance(value, types.FunctionType) or asyncio.iscoroutinefunction(value) +def is_callable_or_coroutine(value): + return asyncio.iscoroutinefunction(value) or inspect.isfunction(value) or inspect.ismethod(value) diff --git a/test/ably/eventemitter_test.py b/test/ably/eventemitter_test.py index d57f046a..deda7626 100644 --- a/test/ably/eventemitter_test.py +++ b/test/ably/eventemitter_test.py @@ -1,6 +1,5 @@ import asyncio from ably.realtime.connection import ConnectionState -from unittest.mock import Mock from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase @@ -11,41 +10,56 @@ async def setUp(self): async def test_connection_events(self): realtime = await RestSetup.get_ably_realtime() - listener = Mock() - realtime.connection.add_listener(ConnectionState.CONNECTED, listener) + call_count = 0 + + def listener(_): + nonlocal call_count + call_count += 1 + + realtime.connection.on(ConnectionState.CONNECTED, listener) await realtime.connect() # Listener is only called once event loop is free - listener.assert_not_called() + assert call_count == 0 await asyncio.sleep(0) - listener.assert_called_once() + assert call_count == 1 await realtime.close() async def test_event_listener_error(self): realtime = await RestSetup.get_ably_realtime() - listener = Mock() + call_count = 0 + + def listener(_): + nonlocal call_count + call_count += 1 + raise Exception() # If a listener throws an exception it should not propagate (#RTE6) listener.side_effect = Exception() - realtime.connection.add_listener(ConnectionState.CONNECTED, listener) + realtime.connection.on(ConnectionState.CONNECTED, listener) await realtime.connect() - listener.assert_not_called() + assert call_count == 0 await asyncio.sleep(0) - listener.assert_called_once() + assert call_count == 1 await realtime.close() async def test_event_emitter_off(self): realtime = await RestSetup.get_ably_realtime() - listener = Mock() - realtime.connection.add_listener(ConnectionState.CONNECTED, listener) - realtime.connection.remove_listener(ConnectionState.CONNECTED, listener) + call_count = 0 + + def listener(_): + nonlocal call_count + call_count += 1 + + realtime.connection.on(ConnectionState.CONNECTED, listener) + realtime.connection.off(ConnectionState.CONNECTED, listener) await realtime.connect() - listener.assert_not_called() + assert call_count == 0 await asyncio.sleep(0) - listener.assert_not_called() + assert call_count == 0 await realtime.close() diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py index d7acb215..90072e9d 100644 --- a/test/ably/realtimechannel_test.py +++ b/test/ably/realtimechannel_test.py @@ -1,6 +1,4 @@ import asyncio -from unittest.mock import Mock -import types from ably.realtime.realtime_channel import ChannelState from ably.types.message import Message from test.ably.restsetup import RestSetup @@ -113,7 +111,10 @@ async def test_subscribe_all_events(self): await channel.attach() message_future = asyncio.Future() - listener = Mock(spec=types.FunctionType, side_effect=lambda msg: message_future.set_result(msg)) + + def listener(msg): + message_future.set_result(msg) + await channel.subscribe(listener) # publish a message using rest client @@ -122,7 +123,6 @@ async def test_subscribe_all_events(self): await rest_channel.publish('event', 'data') message = await message_future - listener.assert_called_once() assert isinstance(message, Message) assert message.name == 'event' assert message.data == 'data' @@ -137,7 +137,9 @@ async def test_subscribe_auto_attach(self): channel = ably.channels.get('my_channel') assert channel.state == ChannelState.INITIALIZED - listener = Mock(spec=types.FunctionType) + def listener(_): + pass + await channel.subscribe('event', listener) assert channel.state == ChannelState.ATTACHED @@ -152,7 +154,13 @@ async def test_unsubscribe(self): await channel.attach() message_future = asyncio.Future() - listener = Mock(spec=types.FunctionType, side_effect=lambda msg: message_future.set_result(msg)) + call_count = 0 + + def listener(msg): + nonlocal call_count + call_count += 1 + message_future.set_result(msg) + await channel.subscribe('event', listener) # publish a message using rest client @@ -160,7 +168,7 @@ async def test_unsubscribe(self): rest_channel = rest.channels.get('my_channel') await rest_channel.publish('event', 'data') await message_future - listener.assert_called_once() + assert call_count == 1 # unsubscribe the listener from the channel channel.unsubscribe('event', listener) @@ -168,7 +176,7 @@ async def test_unsubscribe(self): # test that the listener is not called again for further publishes await rest_channel.publish('event', 'data') await asyncio.sleep(1) - assert listener.call_count == 1 + assert call_count == 1 await ably.close() await rest.close() @@ -179,8 +187,15 @@ async def test_unsubscribe_all(self): await ably.connect() channel = ably.channels.get('my_channel') await channel.attach() + message_future = asyncio.Future() - listener = Mock(spec=types.FunctionType, side_effect=lambda msg: message_future.set_result(msg)) + call_count = 0 + + def listener(msg): + nonlocal call_count + call_count += 1 + message_future.set_result(msg) + await channel.subscribe('event', listener) # publish a message using rest client @@ -188,7 +203,7 @@ async def test_unsubscribe_all(self): rest_channel = rest.channels.get('my_channel') await rest_channel.publish('event', 'data') await message_future - listener.assert_called_once() + assert call_count == 1 # unsubscribe all listeners from the channel channel.unsubscribe() @@ -196,7 +211,7 @@ async def test_unsubscribe_all(self): # test that the listener is not called again for further publishes await rest_channel.publish('event', 'data') await asyncio.sleep(1) - assert listener.call_count == 1 + assert call_count == 1 await ably.close() await rest.close() From 169d7989224624e9ac8c22b49588056a3b127a50 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 1 Nov 2022 12:32:12 +0000 Subject: [PATCH 070/481] doc: add docstrings for patched EventEmitter --- ably/util/eventemitter.py | 51 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/ably/util/eventemitter.py b/ably/util/eventemitter.py index f688ef71..6e737719 100644 --- a/ably/util/eventemitter.py +++ b/ably/util/eventemitter.py @@ -18,11 +18,37 @@ def _is_all_event_args(*args): class EventEmitter: + """ + A generic interface for event registration and delivery used in a number of the types in the Realtime client + library. For example, the Connection object emits events for connection state using the EventEmitter pattern. + + Methods + ------- + on(*args) + Attach to channel + once(*args) + Detach from channel + off() + Subscribe to messages on a channel + """ def __init__(self): self.__named_event_emitter = AsyncIOEventEmitter() self.__all_event_emitter = AsyncIOEventEmitter() def on(self, *args): + """ + Registers the provided listener for the specified event, if provided, and otherwise for all events. + If on() is called more than once with the same listener and event, the listener is added multiple times to + its listener registry. Therefore, as an example, assuming the same listener is registered twice using + on(), and an event is emitted once, the listener would be invoked twice. + + Parameters + ---------- + name : str + The named event to listen for. + listener : callable + The event listener. + """ if _is_all_event_args(*args): self.__all_event_emitter.add_listener(_all_event, args[0]) elif _is_named_event_args(*args): @@ -31,6 +57,20 @@ def on(self, *args): raise ValueError("EventEmitter.on(): invalid args") def once(self, *args): + """ + Registers the provided listener for the first event that is emitted. If once() is called more than once + with the same listener, the listener is added multiple times to its listener registry. Therefore, as an + example, assuming the same listener is registered twice using once(), and an event is emitted once, the + listener would be invoked twice. However, all subsequent events emitted would not invoke the listener as + once() ensures that each registration is only invoked once. + + Parameters + ---------- + name : str + The named event to listen for. + listener : callable + The event listener. + """ if _is_all_event_args(*args): self.__all_event_emitter.once(_all_event, args[0]) elif _is_named_event_args(*args): @@ -39,6 +79,17 @@ def once(self, *args): raise ValueError("EventEmitter.once(): invalid args") def off(self, *args): + """ + Removes all registrations that match both the specified listener and, if provided, the specified event. + If called with no arguments, deregisters all registrations, for all events and listeners. + + Parameters + ---------- + name : str + The named event to listen for. + listener : callable + The event listener. + """ if len(args) == 0: self.__all_event_emitter.remove_all_listeners() self.__named_event_emitter.remove_all_listeners() From 4d44e22e3c74982832c759eea5fbd7054c3da011 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 1 Nov 2022 12:47:00 +0000 Subject: [PATCH 071/481] doc: add usage example for listening to all connection state changes --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 7479203a..919b3331 100644 --- a/README.md +++ b/README.md @@ -241,6 +241,9 @@ client.connection.on('failed', listener) # subscribe to 'connected' connection state client.connection.on('connected', listener) + +# subscribe to all connection state changes +client.connection.on(listener) ``` #### Attach to a channel From 16795eb7cf09f27d5e8c6edb910f1821f75066a4 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 2 Nov 2022 14:33:01 +0000 Subject: [PATCH 072/481] chore: bump version number for 2.0.0-beta.1 release --- ably/__init__.py | 2 +- pyproject.toml | 2 +- test/ably/resthttp_test.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ably/__init__.py b/ably/__init__.py index 128e3d08..ed9c6e09 100644 --- a/ably/__init__.py +++ b/ably/__init__.py @@ -15,4 +15,4 @@ logger.addHandler(logging.NullHandler()) api_version = '1.2' -lib_version = '1.2.1' +lib_version = '2.0.0-beta.1' diff --git a/pyproject.toml b/pyproject.toml index e977e457..3c59ae41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "ably" -version = "1.2.1" +version = "2.0.0-beta.1" description = "Python REST client library SDK for Ably realtime messaging service" license = "Apache-2.0" authors = ["Ably <support@ably.com>"] diff --git a/test/ably/resthttp_test.py b/test/ably/resthttp_test.py index e809a877..43507403 100644 --- a/test/ably/resthttp_test.py +++ b/test/ably/resthttp_test.py @@ -202,7 +202,7 @@ async def test_request_headers(self): # Agent assert 'Ably-Agent' in r.request.headers - expr = r"^ably-python\/\d.\d.\d python\/\d.\d+.\d+$" + expr = r"^ably-python\/\d.\d.\d(-beta\.\d)? python\/\d.\d+.\d+$" assert re.search(expr, r.request.headers['Ably-Agent']) await ably.close() From 6548721848b260eac9ce72491f1646093f4ad6a4 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 2 Nov 2022 14:40:27 +0000 Subject: [PATCH 073/481] chore: update CHANGELOG for 2.0.0-beta.1 release --- CHANGELOG.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20531a76..f96f4fa0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Change Log +## [v2.0.0-beta.1](https://github.com/ably/ably-python/tree/v2.0.0-beta.1) + +[Full Changelog](https://github.com/ably/ably-python/compare/v1.2.1...2.0.0-beta.1) + +- Create Basic Api Key connection [\#311](https://github.com/ably/ably-python/pull/311) +- Send Ably-Agent header in realtime connection [\#314](https://github.com/ably/ably-python/pull/314) +- Close client service [\#315](https://github.com/ably/ably-python/pull/315) +- Implement EventEmitter interface on Connection [\#316](https://github.com/ably/ably-python/pull/316) +- Finish tasks gracefully on failed connection [\#317](https://github.com/ably/ably-python/pull/317) +- Implement realtime ping [\#318](https://github.com/ably/ably-python/pull/318) +- Realtime channel attach/detach [\#319](https://github.com/ably/ably-python/pull/319) +- Add `auto_connect` implementation and client option [\#325](https://github.com/ably/ably-python/pull/325) +- RealtimeChannel subscribe/unsubscribe [\#326](https://github.com/ably/ably-python/pull/326) +- ConnectionStateChange [\#327](https://github.com/ably/ably-python/pull/327) +- Improve realtime logging [\#330](https://github.com/ably/ably-python/pull/330) +- Update readme with realtime documentation [\#334](334](https://github.com/ably/ably-python/pull/334) +- Use string-based enums [\#351](https://github.com/ably/ably-python/pull/351) +- Add environment client option for realtime [\#335](https://github.com/ably/ably-python/pull/335) +- EventEmitter: allow signatures with no event arg [\#350](https://github.com/ably/ably-python/pull/350) + ## [v1.2.1](https://github.com/ably/ably-python/tree/v1.2.1) [Full Changelog](https://github.com/ably/ably-python/compare/v1.2.0...v1.2.1) From ebfb769d2265e5e92e1cfdbf73f97f6dfa9546ae Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 2 Nov 2022 14:55:43 +0000 Subject: [PATCH 074/481] chore: add a blurb to 2.0.0-beta.1 changelog notes --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f96f4fa0..9a6a2dcf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## [v2.0.0-beta.1](https://github.com/ably/ably-python/tree/v2.0.0-beta.1) +**New ably-python realtime client**: This beta release features our first ever python realtime client! Currently the realtime client only supports basic authentication and realtime message subscription. Check out the README for usage examples. + [Full Changelog](https://github.com/ably/ably-python/compare/v1.2.1...2.0.0-beta.1) - Create Basic Api Key connection [\#311](https://github.com/ably/ably-python/pull/311) From efcddcc54f7ea5178eef6cadeff4ffd051e3fefa Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 2 Nov 2022 14:31:19 +0000 Subject: [PATCH 075/481] add connection error reason field --- ably/realtime/connection.py | 9 +++++++++ test/ably/realtimeconnection_test.py | 7 +++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 1e194c89..5bd0a4d4 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -53,6 +53,8 @@ class Connection(EventEmitter): ---------- state: str Connection state + errorReason: error + An ErrorInfo object describing the last error which occurred on the channel, if any. Methods @@ -67,6 +69,7 @@ class Connection(EventEmitter): def __init__(self, realtime): self.__realtime = realtime + self.__error_reason = None self.__state = ConnectionState.CONNECTING if realtime.options.auto_connect else ConnectionState.INITIALIZED self.__connection_manager = ConnectionManager(self.__realtime, self.state) self.__connection_manager.on('connectionstate', self._on_state_update) @@ -109,6 +112,7 @@ async def ping(self): def _on_state_update(self, state_change): log.info(f'Connection state changing from {self.state} to {state_change.current}') self.__state = state_change.current + self.__error_reason = state_change.reason self.__realtime.options.loop.call_soon(functools.partial(self._emit, state_change.current, state_change)) @property @@ -116,6 +120,11 @@ def state(self): """The current connection state of the connection""" return self.__state + @property + def error_reason(self): + """An object describing the last error which occurred on the channel, if any.""" + return self.__error_reason + @state.setter def state(self, value): self.__state = value diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 72647a31..303c1883 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -37,9 +37,10 @@ async def test_closing_state(self): async def test_auth_invalid_key(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) - with pytest.raises(AblyAuthException): + with pytest.raises(AblyAuthException) as exception: await ably.connect() assert ably.connection.state == ConnectionState.FAILED + assert ably.connection.error_reason == exception.value await ably.close() async def test_connection_ping_connected(self): @@ -60,9 +61,10 @@ async def test_connection_ping_initialized(self): async def test_connection_ping_failed(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) - with pytest.raises(AblyAuthException): + with pytest.raises(AblyAuthException) as exception: await ably.connect() assert ably.connection.state == ConnectionState.FAILED + assert ably.connection.error_reason == exception.value with pytest.raises(AblyException) as exception: await ably.connection.ping() assert exception.value.code == 400 @@ -121,4 +123,5 @@ def on_state_change(change): assert state_change.previous == ConnectionState.CONNECTING assert state_change.current == ConnectionState.FAILED assert state_change.reason == exception.value + assert ably.connection.error_reason == exception.value await ably.close() From 2d11c950fd7d37afc10d3bfa6a1df3541303bbbb Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 2 Nov 2022 16:50:42 +0000 Subject: [PATCH 076/481] update error_reason docstring --- ably/realtime/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 5bd0a4d4..f698e18d 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -53,7 +53,7 @@ class Connection(EventEmitter): ---------- state: str Connection state - errorReason: error + error_reason: ErrorInfo An ErrorInfo object describing the last error which occurred on the channel, if any. From e7c1b01a9ef5c9d28d8be0cdabe9dd2b5f44b731 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 3 Nov 2022 13:56:28 +0000 Subject: [PATCH 077/481] chore: update pyproject description for realtime client --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3c59ae41..d18ac3f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "ably" version = "2.0.0-beta.1" -description = "Python REST client library SDK for Ably realtime messaging service" +description = "Python REST and Realtime client library SDK for Ably realtime messaging service" license = "Apache-2.0" authors = ["Ably <support@ably.com>"] readme = "LONG_DESCRIPTION.rst" From 22739ec62e0020ee27fc1877415de030bc9f41ab Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 7 Nov 2022 09:57:00 +0000 Subject: [PATCH 078/481] fix realtime host url --- ably/realtime/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index f698e18d..01f6ea75 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -195,7 +195,7 @@ async def send_protocol_message(self, protocolMessage): async def setup_ws(self): headers = HttpUtils.default_headers() - ws_url = f'wss://{self.options.realtime_host}?key={self.__ably.key}' + ws_url = f'wss://{self.options.get_realtime_host()}?key={self.__ably.key}' log.info(f'setup_ws(): attempting to connect to {ws_url}') async with websockets.connect(ws_url, extra_headers=headers) as websocket: log.info(f'setup_ws(): connection established to {ws_url}') From 6fe7b2c73884af799b7db7b1f84960ecaf80fdee Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 7 Nov 2022 16:45:36 +0000 Subject: [PATCH 079/481] chore: bump version for 2.0.0-beta.2 release --- ably/__init__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ably/__init__.py b/ably/__init__.py index ed9c6e09..1d0d927c 100644 --- a/ably/__init__.py +++ b/ably/__init__.py @@ -15,4 +15,4 @@ logger.addHandler(logging.NullHandler()) api_version = '1.2' -lib_version = '2.0.0-beta.1' +lib_version = '2.0.0-beta.2' diff --git a/pyproject.toml b/pyproject.toml index d18ac3f7..8fc5b277 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "ably" -version = "2.0.0-beta.1" +version = "2.0.0-beta.2" description = "Python REST and Realtime client library SDK for Ably realtime messaging service" license = "Apache-2.0" authors = ["Ably <support@ably.com>"] From 7b27256bdde5acd066984f858cb8ac25e21abe11 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 7 Nov 2022 16:48:19 +0000 Subject: [PATCH 080/481] chore: update changelog for 2.0.0-beta.2 release --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a6a2dcf..6913dac0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Change Log +## [v2.0.0-beta.2](https://github.com/ably/ably-python/tree/v2.0.0-beta.2) + +[Full Changelog](https://github.com/ably/ably-python/compare/v2.0.0-beta.1...v2.0.0-beta.2) +- Fix a bug with realtime_host configuration [\#358](https://github.com/ably/ably-python/pull/358) + ## [v2.0.0-beta.1](https://github.com/ably/ably-python/tree/v2.0.0-beta.1) **New ably-python realtime client**: This beta release features our first ever python realtime client! Currently the realtime client only supports basic authentication and realtime message subscription. Check out the README for usage examples. From a9df1e9a296bcee9a44ca949ffbd93e574752aec Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 7 Nov 2022 11:33:28 +0000 Subject: [PATCH 081/481] add realtime request timeout --- ably/realtime/connection.py | 7 ++++++- ably/transport/defaults.py | 1 + ably/types/options.py | 10 +++++++++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 01f6ea75..0f631735 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -225,7 +225,12 @@ async def ping(self): async def ws_read_loop(self): while True: - raw = await self.__websocket.recv() + try: + raw = await asyncio.wait_for(self.__websocket.recv(), self.options.realtime_request_timeout) + except asyncio.TimeoutError: + exception = AblyException("Realtime request timeout", 504, 50003) + self.enact_state_change(ConnectionState.FAILED, exception) + raise exception msg = json.loads(raw) log.info(f'ws_read_loop(): receieved protocol message: {msg}') action = msg['action'] diff --git a/ably/transport/defaults.py b/ably/transport/defaults.py index c5fa1d04..3612501f 100644 --- a/ably/transport/defaults.py +++ b/ably/transport/defaults.py @@ -19,6 +19,7 @@ class Defaults: suspended_timeout = 60000 comet_recv_timeout = 90000 comet_send_timeout = 10000 + realtime_request_timeout = 10 transports = [] # ["web_socket", "comet"] diff --git a/ably/types/options.py b/ably/types/options.py index d5f8cc4f..da07a495 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -12,7 +12,7 @@ class Options(AuthOptions): def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realtime_host=None, port=0, tls_port=0, use_binary_protocol=True, queue_messages=False, recover=False, environment=None, - http_open_timeout=None, http_request_timeout=None, + http_open_timeout=None, http_request_timeout=None, realtime_request_timeout=None, http_max_retry_count=None, http_max_retry_duration=None, fallback_hosts=None, fallback_hosts_use_default=None, fallback_retry_timeout=None, idempotent_rest_publishing=None, loop=None, auto_connect=True, @@ -23,6 +23,9 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, if fallback_retry_timeout is None: fallback_retry_timeout = Defaults.fallback_retry_timeout + if realtime_request_timeout is None: + realtime_request_timeout = Defaults.realtime_request_timeout + if environment is not None and rest_host is not None: raise ValueError('specify rest_host or environment, not both') @@ -46,6 +49,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, self.__environment = environment self.__http_open_timeout = http_open_timeout self.__http_request_timeout = http_request_timeout + self.__realtime_request_timeout = realtime_request_timeout self.__http_max_retry_count = http_max_retry_count self.__http_max_retry_duration = http_max_retry_duration self.__fallback_hosts = fallback_hosts @@ -154,6 +158,10 @@ def http_open_timeout(self, value): def http_request_timeout(self): return self.__http_request_timeout + @property + def realtime_request_timeout(self): + return self.__realtime_request_timeout + @http_request_timeout.setter def http_request_timeout(self, value): self.__http_request_timeout = value From dc0f9242b329e9b99784a22406011ba79477408a Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Fri, 11 Nov 2022 09:45:13 +0000 Subject: [PATCH 082/481] change request timeout implementation --- ably/realtime/connection.py | 28 ++++++++++++++++++---------- ably/realtime/realtime_channel.py | 22 ++++++++++++++++------ test/ably/realtimeconnection_test.py | 7 +++++++ 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 0f631735..ad4043eb 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -159,7 +159,10 @@ async def connect(self): if self.__connected_future is None: log.fatal('Connection state is CONNECTING but connected_future does not exist') return - await self.__connected_future + try: + await asyncio.wait_for(self.__connected_future, self.options.realtime_request_timeout) + except asyncio.TimeoutError: + raise AblyException("Realtime request timeout", 504, 50003) self.enact_state_change(ConnectionState.CONNECTED) else: self.enact_state_change(ConnectionState.CONNECTING) @@ -173,7 +176,10 @@ async def close(self): self.__closed_future = asyncio.Future() if self.__websocket and self.__state != ConnectionState.FAILED: await self.send_close_message() - await self.__closed_future + try: + await asyncio.wait_for(self.__closed_future, self.options.realtime_request_timeout) + except asyncio.TimeoutError: + raise AblyException("Realtime request timeout", 504, 50003) else: log.warning('Connection.closed called while connection already closed or not established') self.enact_state_change(ConnectionState.CLOSED) @@ -219,24 +225,25 @@ async def ping(self): "id": self.__ping_id}) else: raise AblyException("Cannot send ping request. Calling ping in invalid state", 40000, 400) + try: + await asyncio.wait_for(self.__ping_future, self.options.realtime_request_timeout) + except asyncio.TimeoutError: + raise AblyException("Realtime request timeout", 504, 50003) + ping_end_time = datetime.now().timestamp() response_time_ms = (ping_end_time - ping_start_time) * 1000 return round(response_time_ms, 2) async def ws_read_loop(self): while True: - try: - raw = await asyncio.wait_for(self.__websocket.recv(), self.options.realtime_request_timeout) - except asyncio.TimeoutError: - exception = AblyException("Realtime request timeout", 504, 50003) - self.enact_state_change(ConnectionState.FAILED, exception) - raise exception + raw = await self.__websocket.recv() msg = json.loads(raw) log.info(f'ws_read_loop(): receieved protocol message: {msg}') action = msg['action'] if action == ProtocolMessageAction.CONNECTED: # CONNECTED if self.__connected_future: - self.__connected_future.set_result(None) + if not self.__connected_future.cancelled(): + self.__connected_future.set_result(None) self.__connected_future = None else: log.warn('CONNECTED message received but connected_future not set') @@ -260,7 +267,8 @@ async def ws_read_loop(self): # Resolve on heartbeat from ping request. # TODO: Handle Normal heartbeat if required if self.__ping_id == msg.get("id"): - self.__ping_future.set_result(None) + if not self.__ping_future.cancelled(): + self.__ping_future.set_result(None) self.__ping_future = None if action in ( ProtocolMessageAction.ATTACHED, diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 9a431be8..fda64c2d 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -80,10 +80,12 @@ async def attach(self): # RTL4h - wait for pending attach/detach if self.state == ChannelState.ATTACHING: - await self.__attach_future + if self.__attach_future and not self.__attach_future.cancelled(): + await self.__attach_future return elif self.state == ChannelState.DETACHING: - await self.__detach_future + if self.__detach_future and not self.__detach_future.cancelled(): + await self.__detach_future self.set_state(ChannelState.ATTACHING) @@ -98,7 +100,10 @@ async def attach(self): "channel": self.name, } ) - await self.__attach_future + try: + await asyncio.wait_for(self.__attach_future, self.__realtime.options.realtime_request_timeout) + except asyncio.TimeoutError: + raise AblyException("Realtime request timeout", 504, 50003) self.set_state(ChannelState.ATTACHED) async def detach(self): @@ -130,10 +135,12 @@ async def detach(self): # RTL5i - wait for pending attach/detach if self.state == ChannelState.DETACHING: - await self.__detach_future + if self.__attach_future and not self.__detach_future.cancelled(): + await self.__detach_future return elif self.state == ChannelState.ATTACHING: - await self.__attach_future + if self.__attach_future and not self.__attach_future.cancelled(): + await self.__attach_future self.set_state(ChannelState.DETACHING) @@ -148,7 +155,10 @@ async def detach(self): "channel": self.name, } ) - await self.__detach_future + try: + await asyncio.wait_for(self.__detach_future, self.__realtime.options.realtime_request_timeout) + except asyncio.TimeoutError: + raise AblyException("Realtime request timeout", 504, 50003) self.set_state(ChannelState.DETACHED) async def subscribe(self, *args): diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 303c1883..dbb7dfd5 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -125,3 +125,10 @@ def on_state_change(change): assert state_change.reason == exception.value assert ably.connection.error_reason == exception.value await ably.close() + + async def test_realtime_request_timeout_connect(self): + ably = await RestSetup.get_ably_realtime(realtime_request_timeout=0.000001) + with pytest.raises(AblyException) as exception: + await ably.connect() + assert exception.value.code == 50003 + assert exception.value.status_code == 504 From c058a0362c3bde73d694bc71cfb0b49e79813667 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Fri, 11 Nov 2022 10:45:19 +0000 Subject: [PATCH 083/481] update with disconnected state --- ably/realtime/connection.py | 5 ++++- test/ably/realtimeconnection_test.py | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index ad4043eb..c5cfff7f 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -19,6 +19,7 @@ class ConnectionState(str, Enum): INITIALIZED = 'initialized' CONNECTING = 'connecting' CONNECTED = 'connected' + DISCONNECTED = 'disconnected' CLOSING = 'closing' CLOSED = 'closed' FAILED = 'failed' @@ -162,7 +163,9 @@ async def connect(self): try: await asyncio.wait_for(self.__connected_future, self.options.realtime_request_timeout) except asyncio.TimeoutError: - raise AblyException("Realtime request timeout", 504, 50003) + exception = AblyException("Realtime request timeout", 504, 50003) + self.enact_state_change(ConnectionState.DISCONNECTED, exception) + raise exception self.enact_state_change(ConnectionState.CONNECTED) else: self.enact_state_change(ConnectionState.CONNECTING) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index dbb7dfd5..5a6557ed 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -132,3 +132,6 @@ async def test_realtime_request_timeout_connect(self): await ably.connect() assert exception.value.code == 50003 assert exception.value.status_code == 504 + assert ably.connection.state == ConnectionState.DISCONNECTED + assert ably.connection.error_reason == exception.value + ably.close() From 52c5d81c8cadd15bbbf6ae864175363a576dc430 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Fri, 11 Nov 2022 15:06:46 +0000 Subject: [PATCH 084/481] refactor connect timeout --- ably/realtime/connection.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index c5cfff7f..a0f46b87 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -161,9 +161,9 @@ async def connect(self): log.fatal('Connection state is CONNECTING but connected_future does not exist') return try: - await asyncio.wait_for(self.__connected_future, self.options.realtime_request_timeout) - except asyncio.TimeoutError: - exception = AblyException("Realtime request timeout", 504, 50003) + await self.__connected_future + except asyncio.CancelledError: + exception = AblyException("Connection cancelled due to request timeout", 504, 50003) self.enact_state_change(ConnectionState.DISCONNECTED, exception) raise exception self.enact_state_change(ConnectionState.CONNECTED) @@ -191,7 +191,12 @@ async def close(self): async def connect_impl(self): self.setup_ws_task = self.__ably.options.loop.create_task(self.setup_ws()) - await self.__connected_future + try: + await asyncio.wait_for(self.__connected_future, self.options.realtime_request_timeout) + except asyncio.TimeoutError: + exception = AblyException("Realtime request timeout", 504, 50003) + self.enact_state_change(ConnectionState.DISCONNECTED, exception) + raise exception self.enact_state_change(ConnectionState.CONNECTED) async def send_close_message(self): From 321f20fe76038eed298e5c212265591e05ecacd6 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 14 Nov 2022 14:30:49 +0000 Subject: [PATCH 085/481] review: rename and document realtime timeout --- ably/realtime/connection.py | 6 +++--- ably/realtime/realtime.py | 4 ++++ ably/realtime/realtime_channel.py | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index a0f46b87..44f1a6f5 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -182,7 +182,7 @@ async def close(self): try: await asyncio.wait_for(self.__closed_future, self.options.realtime_request_timeout) except asyncio.TimeoutError: - raise AblyException("Realtime request timeout", 504, 50003) + raise AblyException("Timeout waiting for connection close response", 504, 50003) else: log.warning('Connection.closed called while connection already closed or not established') self.enact_state_change(ConnectionState.CLOSED) @@ -194,7 +194,7 @@ async def connect_impl(self): try: await asyncio.wait_for(self.__connected_future, self.options.realtime_request_timeout) except asyncio.TimeoutError: - exception = AblyException("Realtime request timeout", 504, 50003) + exception = AblyException("Timeout waiting for realtime connection", 504, 50003) self.enact_state_change(ConnectionState.DISCONNECTED, exception) raise exception self.enact_state_change(ConnectionState.CONNECTED) @@ -236,7 +236,7 @@ async def ping(self): try: await asyncio.wait_for(self.__ping_future, self.options.realtime_request_timeout) except asyncio.TimeoutError: - raise AblyException("Realtime request timeout", 504, 50003) + raise AblyException("Timeout waiting for ping response", 504, 50003) ping_end_time = datetime.now().timestamp() response_time_ms = (ping_end_time - ping_start_time) * 1000 diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 7b46ec67..08ffb01c 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -53,6 +53,10 @@ def __init__(self, key=None, loop=None, **kwargs): For development environments only. The default value is realtime.ably.io. environment: str Enables a custom environment to be used with the Ably service. Defaults to `production` + realtime_request_timeout: float + Timeout (in seconds) for the wait of acknowledgement for operations performed via a realtime + connection. Operations include establishing a connection with Ably, or sending a HEARTBEAT, + CONNECT, ATTACH, DETACH or CLOSE request. The default is 10 seconds. Raises ------ diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index fda64c2d..4a4aa4c6 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -103,7 +103,7 @@ async def attach(self): try: await asyncio.wait_for(self.__attach_future, self.__realtime.options.realtime_request_timeout) except asyncio.TimeoutError: - raise AblyException("Realtime request timeout", 504, 50003) + raise AblyException("Timeout waiting for channel attach", 504, 50003) self.set_state(ChannelState.ATTACHED) async def detach(self): @@ -158,7 +158,7 @@ async def detach(self): try: await asyncio.wait_for(self.__detach_future, self.__realtime.options.realtime_request_timeout) except asyncio.TimeoutError: - raise AblyException("Realtime request timeout", 504, 50003) + raise AblyException("Timeout waiting for channel detach", 504, 50003) self.set_state(ChannelState.DETACHED) async def subscribe(self, *args): From a942851c4603643865d79607d608f0a30df47e7d Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 15 Nov 2022 15:26:22 +0000 Subject: [PATCH 086/481] add more timeout test --- ably/realtime/connection.py | 12 ++++++--- ably/realtime/realtime.py | 4 +-- ably/realtime/realtime_channel.py | 22 ++++++++++----- ably/transport/defaults.py | 2 +- test/ably/realtimechannel_test.py | 40 ++++++++++++++++++++++++++++ test/ably/realtimeconnection_test.py | 19 ++++++++++++- 6 files changed, 85 insertions(+), 14 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 44f1a6f5..12b213c1 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -145,6 +145,7 @@ def __init__(self, realtime, initial_state): self.__websocket = None self.setup_ws_task = None self.__ping_future = None + self.timeout_in_secs = self.options.realtime_request_timeout / 1000 super().__init__() def enact_state_change(self, state, reason=None): @@ -180,7 +181,7 @@ async def close(self): if self.__websocket and self.__state != ConnectionState.FAILED: await self.send_close_message() try: - await asyncio.wait_for(self.__closed_future, self.options.realtime_request_timeout) + await asyncio.wait_for(self.__closed_future, self.timeout_in_secs) except asyncio.TimeoutError: raise AblyException("Timeout waiting for connection close response", 504, 50003) else: @@ -192,7 +193,7 @@ async def close(self): async def connect_impl(self): self.setup_ws_task = self.__ably.options.loop.create_task(self.setup_ws()) try: - await asyncio.wait_for(self.__connected_future, self.options.realtime_request_timeout) + await asyncio.wait_for(self.__connected_future, self.timeout_in_secs) except asyncio.TimeoutError: exception = AblyException("Timeout waiting for realtime connection", 504, 50003) self.enact_state_change(ConnectionState.DISCONNECTED, exception) @@ -222,7 +223,10 @@ async def setup_ws(self): async def ping(self): if self.__ping_future: - response = await self.__ping_future + try: + response = await self.__ping_future + except asyncio.CancelledError: + raise AblyException("Ping request cancelled due to request timeout", 504, 50003) return response self.__ping_future = asyncio.Future() @@ -234,7 +238,7 @@ async def ping(self): else: raise AblyException("Cannot send ping request. Calling ping in invalid state", 40000, 400) try: - await asyncio.wait_for(self.__ping_future, self.options.realtime_request_timeout) + await asyncio.wait_for(self.__ping_future, self.timeout_in_secs) except asyncio.TimeoutError: raise AblyException("Timeout waiting for ping response", 504, 50003) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 08ffb01c..5373e331 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -54,9 +54,9 @@ def __init__(self, key=None, loop=None, **kwargs): environment: str Enables a custom environment to be used with the Ably service. Defaults to `production` realtime_request_timeout: float - Timeout (in seconds) for the wait of acknowledgement for operations performed via a realtime + Timeout (in milliseconds) for the wait of acknowledgement for operations performed via a realtime connection. Operations include establishing a connection with Ably, or sending a HEARTBEAT, - CONNECT, ATTACH, DETACH or CLOSE request. The default is 10 seconds. + CONNECT, ATTACH, DETACH or CLOSE request. The default is 10 seconds(10000 milliseconds). Raises ------ diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 4a4aa4c6..67a37b05 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -50,6 +50,7 @@ def __init__(self, realtime, name): self.__realtime = realtime self.__state = ChannelState.INITIALIZED self.__message_emitter = EventEmitter() + self.timeout_in_secs = self.__realtime.options.realtime_request_timeout / 1000 super().__init__() async def attach(self): @@ -80,12 +81,17 @@ async def attach(self): # RTL4h - wait for pending attach/detach if self.state == ChannelState.ATTACHING: - if self.__attach_future and not self.__attach_future.cancelled(): + try: await self.__attach_future + except asyncio.CancelledError: + raise AblyException("Unable to attach channel due to request timeout", 504, 50003) return elif self.state == ChannelState.DETACHING: - if self.__detach_future and not self.__detach_future.cancelled(): + try: await self.__detach_future + except asyncio.CancelledError: + raise AblyException("Unable to detach channel due to request timeout", 504, 50003) + return self.set_state(ChannelState.ATTACHING) @@ -101,7 +107,7 @@ async def attach(self): } ) try: - await asyncio.wait_for(self.__attach_future, self.__realtime.options.realtime_request_timeout) + await asyncio.wait_for(self.__attach_future, self.timeout_in_secs) except asyncio.TimeoutError: raise AblyException("Timeout waiting for channel attach", 504, 50003) self.set_state(ChannelState.ATTACHED) @@ -135,12 +141,16 @@ async def detach(self): # RTL5i - wait for pending attach/detach if self.state == ChannelState.DETACHING: - if self.__attach_future and not self.__detach_future.cancelled(): + try: await self.__detach_future + except asyncio.CancelledError: + raise AblyException("Unable to detach channel due to request timeout", 504, 50003) return elif self.state == ChannelState.ATTACHING: - if self.__attach_future and not self.__attach_future.cancelled(): + try: await self.__attach_future + except asyncio.CancelledError: + raise AblyException("Unable to attach channel due to request timeout", 504, 50003) self.set_state(ChannelState.DETACHING) @@ -156,7 +166,7 @@ async def detach(self): } ) try: - await asyncio.wait_for(self.__detach_future, self.__realtime.options.realtime_request_timeout) + await asyncio.wait_for(self.__detach_future, self.timeout_in_secs) except asyncio.TimeoutError: raise AblyException("Timeout waiting for channel detach", 504, 50003) self.set_state(ChannelState.DETACHED) diff --git a/ably/transport/defaults.py b/ably/transport/defaults.py index 3612501f..cc67fed0 100644 --- a/ably/transport/defaults.py +++ b/ably/transport/defaults.py @@ -19,7 +19,7 @@ class Defaults: suspended_timeout = 60000 comet_recv_timeout = 90000 comet_send_timeout = 10000 - realtime_request_timeout = 10 + realtime_request_timeout = 10000 transports = [] # ["web_socket", "comet"] diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py index 90072e9d..2b6f6667 100644 --- a/test/ably/realtimechannel_test.py +++ b/test/ably/realtimechannel_test.py @@ -1,8 +1,11 @@ import asyncio +import pytest from ably.realtime.realtime_channel import ChannelState from ably.types.message import Message from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase +from ably.realtime.connection import ProtocolMessageAction +from ably.util.exceptions import AblyException class TestRealtimeChannel(BaseAsyncTestCase): @@ -215,3 +218,40 @@ def listener(msg): await ably.close() await rest.close() + + async def test_realtime_request_timeout_attach(self): + ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) + await ably.connect() + original_send_protocol_message = ably.connection.connection_manager.send_protocol_message + + async def new_send_protocol_message(msg): + if msg.get('action') == ProtocolMessageAction.ATTACH: + return + await original_send_protocol_message(msg) + ably.connection.connection_manager.send_protocol_message = new_send_protocol_message + + channel = ably.channels.get('channel_name') + with pytest.raises(AblyException) as exception: + await channel.attach() + assert exception.value.code == 50003 + assert exception.value.status_code == 504 + await ably.close() + + async def test_realtime_request_timeout_detach(self): + ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) + await ably.connect() + original_send_protocol_message = ably.connection.connection_manager.send_protocol_message + + async def new_send_protocol_message(msg): + if msg.get('action') == ProtocolMessageAction.DETACH: + return + await original_send_protocol_message(msg) + ably.connection.connection_manager.send_protocol_message = new_send_protocol_message + + channel = ably.channels.get('channel_name') + await channel.attach() + with pytest.raises(AblyException) as exception: + await channel.detach() + assert exception.value.code == 50003 + assert exception.value.status_code == 504 + await ably.close() diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 5a6557ed..d5266eca 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -1,5 +1,5 @@ import asyncio -from ably.realtime.connection import ConnectionState +from ably.realtime.connection import ConnectionState, ProtocolMessageAction import pytest from ably.util.exceptions import AblyAuthException, AblyException from test.ably.restsetup import RestSetup @@ -135,3 +135,20 @@ async def test_realtime_request_timeout_connect(self): assert ably.connection.state == ConnectionState.DISCONNECTED assert ably.connection.error_reason == exception.value ably.close() + + async def test_realtime_request_timeout_ping(self): + ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) + await ably.connect() + original_send_protocol_message = ably.connection.connection_manager.send_protocol_message + + async def new_send_protocol_message(msg): + if msg.get('action') == ProtocolMessageAction.HEARTBEAT: + return + await original_send_protocol_message(msg) + ably.connection.connection_manager.send_protocol_message = new_send_protocol_message + + with pytest.raises(AblyException) as exception: + await ably.connection.ping() + assert exception.value.code == 50003 + assert exception.value.status_code == 504 + await ably.close() From 99072b9fee49843fe3c534e987f2ca3a646816e6 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 15 Nov 2022 17:03:26 +0000 Subject: [PATCH 087/481] add test for close request timeout --- test/ably/realtimeconnection_test.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index d5266eca..5ec9a0b7 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -152,3 +152,19 @@ async def new_send_protocol_message(msg): assert exception.value.code == 50003 assert exception.value.status_code == 504 await ably.close() + + async def test_realtime_request_timeout_close(self): + ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) + await ably.connect() + original_send_protocol_message = ably.connection.connection_manager.send_protocol_message + + async def new_send_protocol_message(msg): + if msg.get('action') == ProtocolMessageAction.CLOSE: + return + await original_send_protocol_message(msg) + ably.connection.connection_manager.send_protocol_message = new_send_protocol_message + + with pytest.raises(AblyException) as exception: + await ably.close() + assert exception.value.code == 50003 + assert exception.value.status_code == 504 From bbdc571fecfa015fbb54743b792d6b63cb5f8f14 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 16 Nov 2022 10:41:45 +0000 Subject: [PATCH 088/481] make timeout internal --- ably/realtime/connection.py | 8 ++++---- ably/realtime/realtime_channel.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 12b213c1..ad3e777b 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -145,7 +145,7 @@ def __init__(self, realtime, initial_state): self.__websocket = None self.setup_ws_task = None self.__ping_future = None - self.timeout_in_secs = self.options.realtime_request_timeout / 1000 + self.__timeout_in_secs = self.options.realtime_request_timeout / 1000 super().__init__() def enact_state_change(self, state, reason=None): @@ -181,7 +181,7 @@ async def close(self): if self.__websocket and self.__state != ConnectionState.FAILED: await self.send_close_message() try: - await asyncio.wait_for(self.__closed_future, self.timeout_in_secs) + await asyncio.wait_for(self.__closed_future, self.__timeout_in_secs) except asyncio.TimeoutError: raise AblyException("Timeout waiting for connection close response", 504, 50003) else: @@ -193,7 +193,7 @@ async def close(self): async def connect_impl(self): self.setup_ws_task = self.__ably.options.loop.create_task(self.setup_ws()) try: - await asyncio.wait_for(self.__connected_future, self.timeout_in_secs) + await asyncio.wait_for(self.__connected_future, self.__timeout_in_secs) except asyncio.TimeoutError: exception = AblyException("Timeout waiting for realtime connection", 504, 50003) self.enact_state_change(ConnectionState.DISCONNECTED, exception) @@ -238,7 +238,7 @@ async def ping(self): else: raise AblyException("Cannot send ping request. Calling ping in invalid state", 40000, 400) try: - await asyncio.wait_for(self.__ping_future, self.timeout_in_secs) + await asyncio.wait_for(self.__ping_future, self.__timeout_in_secs) except asyncio.TimeoutError: raise AblyException("Timeout waiting for ping response", 504, 50003) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 67a37b05..75e3f5e1 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -50,7 +50,7 @@ def __init__(self, realtime, name): self.__realtime = realtime self.__state = ChannelState.INITIALIZED self.__message_emitter = EventEmitter() - self.timeout_in_secs = self.__realtime.options.realtime_request_timeout / 1000 + self.__timeout_in_secs = self.__realtime.options.realtime_request_timeout / 1000 super().__init__() async def attach(self): @@ -107,7 +107,7 @@ async def attach(self): } ) try: - await asyncio.wait_for(self.__attach_future, self.timeout_in_secs) + await asyncio.wait_for(self.__attach_future, self.__timeout_in_secs) except asyncio.TimeoutError: raise AblyException("Timeout waiting for channel attach", 504, 50003) self.set_state(ChannelState.ATTACHED) @@ -166,7 +166,7 @@ async def detach(self): } ) try: - await asyncio.wait_for(self.__detach_future, self.timeout_in_secs) + await asyncio.wait_for(self.__detach_future, self.__timeout_in_secs) except asyncio.TimeoutError: raise AblyException("Timeout waiting for channel detach", 504, 50003) self.set_state(ChannelState.DETACHED) From 4709ba8cec39553b5c2fb9d480571001573c566b Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 15 Nov 2022 14:12:34 +0000 Subject: [PATCH 089/481] refactor: Realtime extends Rest --- ably/realtime/realtime.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 5373e331..7417d113 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -2,6 +2,7 @@ import asyncio from ably.realtime.connection import Connection from ably.rest.auth import Auth +from ably.rest.rest import AblyRest from ably.types.options import Options from ably.realtime.realtime_channel import RealtimeChannel @@ -9,7 +10,7 @@ log = logging.getLogger(__name__) -class AblyRealtime: +class AblyRealtime(AblyRest): """ Ably Realtime Client @@ -63,6 +64,8 @@ def __init__(self, key=None, loop=None, **kwargs): ValueError If no authentication key is not provided """ + super().__init__(key, **kwargs) + if loop is None: try: loop = asyncio.get_running_loop() @@ -102,6 +105,7 @@ async def close(self): """ log.info('Realtime.close() called') await self.connection.close() + await super().close() @property def auth(self): From 5bad3b89a335dce08c8b6f4cc098345f1495a6c0 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 16 Nov 2022 00:44:01 +0000 Subject: [PATCH 090/481] refactor: RealtimeChannel extends Channel --- ably/realtime/realtime_channel.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 75e3f5e1..36cc6703 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -2,6 +2,7 @@ import logging from ably.realtime.connection import ConnectionState, ProtocolMessageAction +from ably.rest.channel import Channel from ably.types.message import Message from ably.util.eventemitter import EventEmitter from ably.util.exceptions import AblyException @@ -20,7 +21,7 @@ class ChannelState(str, Enum): DETACHED = 'detached' -class RealtimeChannel(EventEmitter): +class RealtimeChannel(EventEmitter, Channel): """ Ably Realtime Channel @@ -44,6 +45,7 @@ class RealtimeChannel(EventEmitter): """ def __init__(self, realtime, name): + EventEmitter.__init__(self) self.__name = name self.__attach_future = None self.__detach_future = None @@ -51,7 +53,7 @@ def __init__(self, realtime, name): self.__state = ChannelState.INITIALIZED self.__message_emitter = EventEmitter() self.__timeout_in_secs = self.__realtime.options.realtime_request_timeout / 1000 - super().__init__() + Channel.__init__(self, realtime, name, {}) async def attach(self): """Attach to channel From 5f277ad6b6946fb7039f1b842b55d0e10a2dbeb8 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 16 Nov 2022 00:44:17 +0000 Subject: [PATCH 091/481] test: use Rest methods on Realtime for publishing --- test/ably/realtimechannel_test.py | 7 ++----- test/ably/restsetup.py | 3 ++- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py index 2b6f6667..c95488cf 100644 --- a/test/ably/realtimechannel_test.py +++ b/test/ably/realtimechannel_test.py @@ -63,9 +63,7 @@ def listener(message): await channel.subscribe('event', listener) # publish a message using rest client - rest = await RestSetup.get_ably_rest() - rest_channel = rest.channels.get('my_channel') - await rest_channel.publish('event', 'data') + await channel.publish('event', 'data') message = await first_message_future assert isinstance(message, Message) @@ -73,11 +71,10 @@ def listener(message): assert message.data == 'data' # test that the listener is called again for further publishes - await rest_channel.publish('event', 'data') + await channel.publish('event', 'data') await second_message_future await ably.close() - await rest.close() async def test_subscribe_coroutine(self): ably = await RestSetup.get_ably_realtime() diff --git a/test/ably/restsetup.py b/test/ably/restsetup.py index 5cd73c1d..32097567 100644 --- a/test/ably/restsetup.py +++ b/test/ably/restsetup.py @@ -87,7 +87,8 @@ async def get_ably_realtime(cls, **kw): test_vars = await RestSetup.get_test_vars() options = { 'key': test_vars["keys"][0]["key_str"], - 'realtime_host': realtime_host, + 'realtime_host': test_vars["realtime_host"], + 'rest_host': test_vars["host"], 'port': test_vars["port"], 'tls_port': test_vars["tls_port"], 'tls': test_vars["tls"], From 947d3b0b8c550ccefa4088286811ef10aef8ddb4 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 28 Nov 2022 13:59:18 +0000 Subject: [PATCH 092/481] send api protocol version --- ably/realtime/connection.py | 3 ++- ably/transport/defaults.py | 2 +- ably/types/options.py | 5 +++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index ad3e777b..9f362864 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -210,7 +210,8 @@ async def send_protocol_message(self, protocolMessage): async def setup_ws(self): headers = HttpUtils.default_headers() - ws_url = f'wss://{self.options.get_realtime_host()}?key={self.__ably.key}' + ws_url = (f'wss://{self.options.get_realtime_host()}?key={self.__ably.key}' + f'&v={self.options.protocol_version}') log.info(f'setup_ws(): attempting to connect to {ws_url}') async with websockets.connect(ws_url, extra_headers=headers) as websocket: log.info(f'setup_ws(): connection established to {ws_url}') diff --git a/ably/transport/defaults.py b/ably/transport/defaults.py index cc67fed0..60303ef5 100644 --- a/ably/transport/defaults.py +++ b/ably/transport/defaults.py @@ -1,5 +1,5 @@ class Defaults: - protocol_version = 1 + protocol_version = "2" fallback_hosts = [ "a.ably-realtime.com", "b.ably-realtime.com", diff --git a/ably/types/options.py b/ably/types/options.py index da07a495..403620d6 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -61,6 +61,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, self.__rest_hosts = self.__get_rest_hosts() self.__realtime_hosts = self.__get_realtime_hosts() + self.__protocol_version = Defaults.protocol_version @property def client_id(self): @@ -206,6 +207,10 @@ def loop(self): def auto_connect(self): return self.__auto_connect + @property + def protocol_version(self): + return self.__protocol_version + def __get_rest_hosts(self): """ Return the list of hosts as they should be tried. First comes the main From 24aa7e66ab7e65cc57c2ef2a7acfc85729781814 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 28 Nov 2022 11:46:13 +0000 Subject: [PATCH 093/481] clear connection error reason connect is called --- ably/realtime/connection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index ad3e777b..372f4f2d 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -81,6 +81,7 @@ async def connect(self): Causes the connection to open, entering the connecting state """ + self.__error_reason = None await self.__connection_manager.connect() async def close(self): From 69273e5336799ef6c5f439aaa58693b1d9246e49 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 30 Nov 2022 15:20:24 +0000 Subject: [PATCH 094/481] review: encode url params --- ably/realtime/connection.py | 8 ++++++-- ably/types/options.py | 5 ----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 9f362864..575d5cca 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -3,7 +3,9 @@ import asyncio import websockets import json +import urllib.parse from ably.http.httputils import HttpUtils +from ably.transport.defaults import Defaults from ably.util.exceptions import AblyAuthException, AblyException from ably.util.eventemitter import EventEmitter from enum import Enum, IntEnum @@ -210,8 +212,10 @@ async def send_protocol_message(self, protocolMessage): async def setup_ws(self): headers = HttpUtils.default_headers() - ws_url = (f'wss://{self.options.get_realtime_host()}?key={self.__ably.key}' - f'&v={self.options.protocol_version}') + protocol_version = Defaults.protocol_version + params = {"key": self.__ably.key, "v": protocol_version} + query_params = urllib.parse.urlencode(params) + ws_url = (f'wss://{self.options.get_realtime_host()}?{query_params}') log.info(f'setup_ws(): attempting to connect to {ws_url}') async with websockets.connect(ws_url, extra_headers=headers) as websocket: log.info(f'setup_ws(): connection established to {ws_url}') diff --git a/ably/types/options.py b/ably/types/options.py index 403620d6..da07a495 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -61,7 +61,6 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, self.__rest_hosts = self.__get_rest_hosts() self.__realtime_hosts = self.__get_realtime_hosts() - self.__protocol_version = Defaults.protocol_version @property def client_id(self): @@ -207,10 +206,6 @@ def loop(self): def auto_connect(self): return self.__auto_connect - @property - def protocol_version(self): - return self.__protocol_version - def __get_rest_hosts(self): """ Return the list of hosts as they should be tried. First comes the main From 655b3132b966003766b7b55157ff4b39ada78648 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 21 Nov 2022 17:15:21 +0000 Subject: [PATCH 095/481] implement disconnected retry timeout --- ably/realtime/connection.py | 16 ++++++++++++++-- ably/realtime/realtime.py | 4 +++- ably/transport/defaults.py | 1 + ably/types/options.py | 12 ++++++++++-- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 9a3fe37e..ea5ba381 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -167,8 +167,10 @@ async def connect(self): try: await self.__connected_future except asyncio.CancelledError: - exception = AblyException("Connection cancelled due to request timeout", 504, 50003) + exception = AblyException( + "Connection cancelled due to request timeout. Attempting reconnection...", 504, 50003) self.enact_state_change(ConnectionState.DISCONNECTED, exception) + log.info('Connection cancelled due to request timeout. Attempting reconnection...') raise exception self.enact_state_change(ConnectionState.CONNECTED) else: @@ -193,14 +195,24 @@ async def close(self): if self.setup_ws_task: await self.setup_ws_task + def on_setup_ws_done(self, task): + exception = task.exception() + if exception is not None: + if self.__connected_future and not self.__connected_future.cancelled(): + self.__connected_future.set_exception(exception) + self.enact_state_change(ConnectionState.DISCONNECTED, exception) + async def connect_impl(self): self.setup_ws_task = self.__ably.options.loop.create_task(self.setup_ws()) + self.setup_ws_task.add_done_callback(self.on_setup_ws_done) try: await asyncio.wait_for(self.__connected_future, self.__timeout_in_secs) except asyncio.TimeoutError: exception = AblyException("Timeout waiting for realtime connection", 504, 50003) self.enact_state_change(ConnectionState.DISCONNECTED, exception) - raise exception + await asyncio.sleep(self.options.disconnected_retry_timeout / 1000) + log.info('Attempting reconnection') + await self.connect() self.enact_state_change(ConnectionState.CONNECTED) async def send_close_message(self): diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 7417d113..4539f460 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -58,7 +58,9 @@ def __init__(self, key=None, loop=None, **kwargs): Timeout (in milliseconds) for the wait of acknowledgement for operations performed via a realtime connection. Operations include establishing a connection with Ably, or sending a HEARTBEAT, CONNECT, ATTACH, DETACH or CLOSE request. The default is 10 seconds(10000 milliseconds). - + disconnected_retry_timeout: float + If the connection is still in the DISCONNECTED state after this delay, the client library will + attempt to reconnect automatically. The default is 15 seconds. Raises ------ ValueError diff --git a/ably/transport/defaults.py b/ably/transport/defaults.py index 60303ef5..79f72ca9 100644 --- a/ably/transport/defaults.py +++ b/ably/transport/defaults.py @@ -20,6 +20,7 @@ class Defaults: comet_recv_timeout = 90000 comet_send_timeout = 10000 realtime_request_timeout = 10000 + disconnected_retry_timeout = 15000 transports = [] # ["web_socket", "comet"] diff --git a/ably/types/options.py b/ably/types/options.py index da07a495..0a926992 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -13,8 +13,8 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realtime_host=None, port=0, tls_port=0, use_binary_protocol=True, queue_messages=False, recover=False, environment=None, http_open_timeout=None, http_request_timeout=None, realtime_request_timeout=None, - http_max_retry_count=None, http_max_retry_duration=None, - fallback_hosts=None, fallback_hosts_use_default=None, fallback_retry_timeout=None, + http_max_retry_count=None, http_max_retry_duration=None, fallback_hosts=None, + fallback_hosts_use_default=None, fallback_retry_timeout=None, disconnected_retry_timeout=None, idempotent_rest_publishing=None, loop=None, auto_connect=True, **kwargs): super().__init__(**kwargs) @@ -26,6 +26,9 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, if realtime_request_timeout is None: realtime_request_timeout = Defaults.realtime_request_timeout + if disconnected_retry_timeout is None: + disconnected_retry_timeout = Defaults.disconnected_retry_timeout + if environment is not None and rest_host is not None: raise ValueError('specify rest_host or environment, not both') @@ -55,6 +58,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, self.__fallback_hosts = fallback_hosts self.__fallback_hosts_use_default = fallback_hosts_use_default self.__fallback_retry_timeout = fallback_retry_timeout + self.__disconnected_retry_timeout = disconnected_retry_timeout self.__idempotent_rest_publishing = idempotent_rest_publishing self.__loop = loop self.__auto_connect = auto_connect @@ -194,6 +198,10 @@ def fallback_hosts_use_default(self): def fallback_retry_timeout(self): return self.__fallback_retry_timeout + @property + def disconnected_retry_timeout(self): + return self.__disconnected_retry_timeout + @property def idempotent_rest_publishing(self): return self.__idempotent_rest_publishing From dc73a52b43907e78b796576104073e27efb9371c Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 23 Nov 2022 16:12:31 +0000 Subject: [PATCH 096/481] add test for disconnected retry --- test/ably/realtimeconnection_test.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 5ec9a0b7..73c38a82 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -168,3 +168,26 @@ async def new_send_protocol_message(msg): await ably.close() assert exception.value.code == 50003 assert exception.value.status_code == 504 + + async def test_disconnected_retry_timeout(self): + ably = await RestSetup.get_ably_realtime(realtime_request_timeout=0.001, + disconnected_retry_timeout=2000) + state_changes = [] + + def on_state_change(state_change): + state_changes.append(state_change) + + ably.connection.on(on_state_change) + + with pytest.raises(AblyException) as exception: + await ably.connect() + assert exception.value.code == 50003 + assert exception.value.status_code == 504 + assert ably.connection.state == ConnectionState.DISCONNECTED + # 2 state changes happens per retry. + # Retry timeout of 2 secs, will retry connection twice in 3 and/or 4 seconds, resulting in 4 state changes + await asyncio.sleep(4) + assert len(state_changes) == 4 + assert state_changes[0].previous == ConnectionState.CONNECTING + assert state_changes[0].current == ConnectionState.DISCONNECTED + ably.close() From 297061ba80f4ef489b587af7fa3173fbf2372f5b Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Fri, 25 Nov 2022 12:15:57 +0000 Subject: [PATCH 097/481] change retry implementation --- ably/realtime/connection.py | 10 ++++++++-- ably/realtime/realtime.py | 2 +- ably/transport/defaults.py | 2 +- test/ably/realtimeconnection_test.py | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index ea5ba381..805f11c2 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -165,12 +165,14 @@ async def connect(self): log.fatal('Connection state is CONNECTING but connected_future does not exist') return try: + print("toh") await self.__connected_future except asyncio.CancelledError: exception = AblyException( "Connection cancelled due to request timeout. Attempting reconnection...", 504, 50003) self.enact_state_change(ConnectionState.DISCONNECTED, exception) log.info('Connection cancelled due to request timeout. Attempting reconnection...') + print("cancelled error") raise exception self.enact_state_change(ConnectionState.CONNECTED) else: @@ -209,10 +211,14 @@ async def connect_impl(self): await asyncio.wait_for(self.__connected_future, self.__timeout_in_secs) except asyncio.TimeoutError: exception = AblyException("Timeout waiting for realtime connection", 504, 50003) - self.enact_state_change(ConnectionState.DISCONNECTED, exception) + self.enact_state_change(ConnectionState.CONNECTING, exception) await asyncio.sleep(self.options.disconnected_retry_timeout / 1000) log.info('Attempting reconnection') - await self.connect() + self.__connected_future = asyncio.Future() + print("timeout error") + # task.add_done_callback(self.on_setup_ws_done) + # task = self.__ably.options.loop.create_task(self.connect()) + self.__ably.options.loop.create_task(self.connect()) self.enact_state_change(ConnectionState.CONNECTED) async def send_close_message(self): diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 4539f460..4bc0aaa9 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -86,7 +86,7 @@ def __init__(self, key=None, loop=None, **kwargs): self.key = key self.__connection = Connection(self) self.__channels = Channels(self) - + print(options.auto_connect, "+++") if options.auto_connect: asyncio.ensure_future(self.connection.connection_manager.connect_impl()) diff --git a/ably/transport/defaults.py b/ably/transport/defaults.py index 79f72ca9..6b0fec88 100644 --- a/ably/transport/defaults.py +++ b/ably/transport/defaults.py @@ -20,7 +20,7 @@ class Defaults: comet_recv_timeout = 90000 comet_send_timeout = 10000 realtime_request_timeout = 10000 - disconnected_retry_timeout = 15000 + disconnected_retry_timeout = 1500 transports = [] # ["web_socket", "comet"] diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 73c38a82..6d8b25c2 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -171,7 +171,7 @@ async def new_send_protocol_message(msg): async def test_disconnected_retry_timeout(self): ably = await RestSetup.get_ably_realtime(realtime_request_timeout=0.001, - disconnected_retry_timeout=2000) + disconnected_retry_timeout=2000, auto_connect=False) state_changes = [] def on_state_change(state_change): From f3cb7e3b2b8b9db4119806bcb168ba6f6d9b7f0d Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 29 Nov 2022 11:57:17 +0000 Subject: [PATCH 098/481] refactor: create WebSocketTransport class --- ably/realtime/websockettransport.py | 107 ++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 ably/realtime/websockettransport.py diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py new file mode 100644 index 00000000..1409e5bd --- /dev/null +++ b/ably/realtime/websockettransport.py @@ -0,0 +1,107 @@ +from __future__ import annotations +from typing import TYPE_CHECKING +import asyncio +from enum import IntEnum +import json +import logging +from ably.http.httputils import HttpUtils +from websockets.client import WebSocketClientProtocol, connect as ws_connect +from websockets.exceptions import ConnectionClosedOK + +if TYPE_CHECKING: + from ably.realtime.connection import ConnectionManager + +log = logging.getLogger(__name__) + + +class ProtocolMessageAction(IntEnum): + HEARTBEAT = 0 + CONNECTED = 4 + ERROR = 9 + CLOSE = 7 + CLOSED = 8 + ATTACH = 10 + ATTACHED = 11 + DETACH = 12 + DETACHED = 13 + MESSAGE = 15 + + +class WebSocketTransport: + def __init__(self, connection_manager: ConnectionManager): + self.websocket: WebSocketClientProtocol | None = None + self.read_loop: asyncio.Task | None = None + self.connect_task: asyncio.Task | None = None + self.ws_connect_task: asyncio.Task | None = None + self.connection_manager = connection_manager + self.is_connected = False + + async def connect(self): + headers = HttpUtils.default_headers() + host = self.connection_manager.options.get_realtime_host() + key = self.connection_manager.ably.key + ws_url = f'wss://{host}?key={key}' + log.info(f'connect(): attempting to connect to {ws_url}') + self.ws_connect_task = asyncio.create_task(self.ws_connect(ws_url, headers)) + self.ws_connect_task.add_done_callback(self.on_ws_connect_done) + + def on_ws_connect_done(self, task: asyncio.Task): + try: + exception = task.exception() + except asyncio.CancelledError as e: + exception = e + if isinstance(exception, ConnectionClosedOK): + return + + async def ws_connect(self, ws_url, headers): + async with ws_connect(ws_url, extra_headers=headers) as websocket: + log.info(f'ws_connect(): connection established to {ws_url}') + self.websocket = websocket + self.read_loop = self.connection_manager.options.loop.create_task(self.ws_read_loop()) + self.read_loop.add_done_callback(self.on_read_loop_done) + await self.read_loop + + async def ws_read_loop(self): + while True: + if self.websocket is not None: + try: + raw = await self.websocket.recv() + except ConnectionClosedOK: + break + msg = json.loads(raw) + log.info(f'ws_read_loop(): receieved protocol message: {msg}') + if msg['action'] == ProtocolMessageAction.CLOSED: + if self.ws_connect_task: + self.ws_connect_task.cancel() + await self.connection_manager.on_protocol_message(msg) + else: + raise Exception() + + def on_read_loop_done(self, task: asyncio.Task): + try: + exception = task.exception() + except asyncio.CancelledError as e: + exception = e + if isinstance(exception, ConnectionClosedOK): + return + + async def dispose(self): + if self.read_loop: + self.read_loop.cancel() + if self.ws_connect_task: + self.ws_connect_task.cancel() + if self.websocket: + try: + await self.websocket.close() + except asyncio.CancelledError: + return + + async def close(self): + await self.send({'action': ProtocolMessageAction.CLOSE}) + + async def send(self, message: dict): + if self.websocket is None: + raise Exception() + raw_msg = json.dumps(message) + log.info(f'WebSocketTransport.send(): sending {raw_msg}') + await self.websocket.send(raw_msg) From e10c4ced60ccfdc17c2bb5903bb1413e827b60ed Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 29 Nov 2022 11:58:13 +0000 Subject: [PATCH 099/481] refactor: use ProtocolMessageAction from websockettransport module --- ably/realtime/connection.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 805f11c2..ee5d731e 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -1,6 +1,7 @@ import functools import logging import asyncio +from ably.realtime.connectionmanager import ProtocolMessageAction import websockets import json import urllib.parse @@ -8,7 +9,7 @@ from ably.transport.defaults import Defaults from ably.util.exceptions import AblyAuthException, AblyException from ably.util.eventemitter import EventEmitter -from enum import Enum, IntEnum +from enum import Enum from datetime import datetime from ably.util import helper from dataclasses import dataclass @@ -34,19 +35,6 @@ class ConnectionStateChange: reason: Optional[AblyException] = None -class ProtocolMessageAction(IntEnum): - HEARTBEAT = 0 - CONNECTED = 4 - ERROR = 9 - CLOSE = 7 - CLOSED = 8 - ATTACH = 10 - ATTACHED = 11 - DETACH = 12 - DETACHED = 13 - MESSAGE = 15 - - class Connection(EventEmitter): """Ably Realtime Connection From 16cc130878f13c268dc1f8ef650264b870789ca9 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 29 Nov 2022 11:58:50 +0000 Subject: [PATCH 100/481] chore: fix styling of protocol_message var --- ably/realtime/connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index ee5d731e..2c515a98 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -212,8 +212,8 @@ async def connect_impl(self): async def send_close_message(self): await self.send_protocol_message({"action": ProtocolMessageAction.CLOSE}) - async def send_protocol_message(self, protocolMessage): - raw_msg = json.dumps(protocolMessage) + async def send_protocol_message(self, protocol_message): + raw_msg = json.dumps(protocol_message) log.info('send_protocol_message(): sending {raw_msg}') await self.__websocket.send(raw_msg) From cdba5d0aa0da8a1712691b7359112c65b784e3e5 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 29 Nov 2022 12:11:51 +0000 Subject: [PATCH 101/481] refactor: use WebSocketTransport in ConnectionManager --- ably/realtime/connection.py | 200 +++++++++++++-------------- ably/realtime/realtime.py | 1 - ably/realtime/websockettransport.py | 8 ++ test/ably/realtimeconnection_test.py | 10 +- 4 files changed, 110 insertions(+), 109 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 2c515a98..f1e7aa13 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -1,12 +1,7 @@ import functools import logging import asyncio -from ably.realtime.connectionmanager import ProtocolMessageAction -import websockets -import json -import urllib.parse -from ably.http.httputils import HttpUtils -from ably.transport.defaults import Defaults +from ably.realtime.websockettransport import WebSocketTransport, ProtocolMessageAction from ably.util.exceptions import AblyAuthException, AblyException from ably.util.eventemitter import EventEmitter from enum import Enum @@ -133,10 +128,9 @@ def __init__(self, realtime, initial_state): self.__state = initial_state self.__connected_future = asyncio.Future() if initial_state == ConnectionState.CONNECTING else None self.__closed_future = None - self.__websocket = None - self.setup_ws_task = None self.__ping_future = None self.__timeout_in_secs = self.options.realtime_request_timeout / 1000 + self.transport: WebSocketTransport | None = None super().__init__() def enact_state_change(self, state, reason=None): @@ -145,93 +139,96 @@ def enact_state_change(self, state, reason=None): self._emit('connectionstate', ConnectionStateChange(current_state, state, reason)) async def connect(self): + if not self.__connected_future: + self.__connected_future = asyncio.Future() + self.try_connect() + await self.__connected_future + + def try_connect(self): + task = asyncio.create_task(self._connect()) + task.add_done_callback(self.on_connection_attempt_done) + + async def _connect(self): if self.__state == ConnectionState.CONNECTED: return if self.__state == ConnectionState.CONNECTING: - if self.__connected_future is None: - log.fatal('Connection state is CONNECTING but connected_future does not exist') - return try: - print("toh") + if not self.__connected_future: + self.__connected_future = asyncio.Future() await self.__connected_future except asyncio.CancelledError: exception = AblyException( "Connection cancelled due to request timeout. Attempting reconnection...", 504, 50003) - self.enact_state_change(ConnectionState.DISCONNECTED, exception) log.info('Connection cancelled due to request timeout. Attempting reconnection...') - print("cancelled error") raise exception - self.enact_state_change(ConnectionState.CONNECTED) else: self.enact_state_change(ConnectionState.CONNECTING) - self.__connected_future = asyncio.Future() await self.connect_impl() + def on_connection_attempt_done(self, task): + try: + exception = task.exception() + except asyncio.CancelledError: + exception = AblyException( + "Connection cancelled due to request timeout. Attempting reconnection...", 504, 50003) + if exception is None: + return + if self.__state in (ConnectionState.CLOSED, ConnectionState.FAILED): + return + if self.__state != ConnectionState.DISCONNECTED: + if self.__connected_future: + self.__connected_future.set_exception(exception) + self.__connected_future = None + self.enact_state_change(ConnectionState.DISCONNECTED, exception) + async def close(self): + if self.__state in (ConnectionState.CLOSED, ConnectionState.INITIALIZED, ConnectionState.FAILED): + self.enact_state_change(ConnectionState.CLOSED) + return + if self.__state is ConnectionState.DISCONNECTED: + if self.transport: + await self.transport.dispose() + self.transport = None + self.enact_state_change(ConnectionState.CLOSED) + return if self.__state != ConnectionState.CONNECTED: log.warning('Connection.closed called while connection state not connected') + if self.__state == ConnectionState.CONNECTING: + await self.__connected_future self.enact_state_change(ConnectionState.CLOSING) self.__closed_future = asyncio.Future() - if self.__websocket and self.__state != ConnectionState.FAILED: - await self.send_close_message() + if self.transport and self.transport.is_connected: + await self.transport.close() try: await asyncio.wait_for(self.__closed_future, self.__timeout_in_secs) except asyncio.TimeoutError: raise AblyException("Timeout waiting for connection close response", 504, 50003) else: - log.warning('Connection.closed called while connection already closed or not established') + log.warning('ConnectionManager: called close with no connected transport') self.enact_state_change(ConnectionState.CLOSED) - if self.setup_ws_task: - await self.setup_ws_task - - def on_setup_ws_done(self, task): - exception = task.exception() - if exception is not None: - if self.__connected_future and not self.__connected_future.cancelled(): - self.__connected_future.set_exception(exception) - self.enact_state_change(ConnectionState.DISCONNECTED, exception) + if self.transport and self.transport.ws_connect_task is not None: + await self.transport.ws_connect_task async def connect_impl(self): - self.setup_ws_task = self.__ably.options.loop.create_task(self.setup_ws()) - self.setup_ws_task.add_done_callback(self.on_setup_ws_done) + self.transport = WebSocketTransport(self) + await self.transport.connect() try: - await asyncio.wait_for(self.__connected_future, self.__timeout_in_secs) + await asyncio.wait_for(asyncio.shield(self.__connected_future), self.__timeout_in_secs) except asyncio.TimeoutError: exception = AblyException("Timeout waiting for realtime connection", 504, 50003) - self.enact_state_change(ConnectionState.CONNECTING, exception) - await asyncio.sleep(self.options.disconnected_retry_timeout / 1000) - log.info('Attempting reconnection') - self.__connected_future = asyncio.Future() - print("timeout error") - # task.add_done_callback(self.on_setup_ws_done) - # task = self.__ably.options.loop.create_task(self.connect()) - self.__ably.options.loop.create_task(self.connect()) - self.enact_state_change(ConnectionState.CONNECTED) - - async def send_close_message(self): - await self.send_protocol_message({"action": ProtocolMessageAction.CLOSE}) + self.enact_state_change(ConnectionState.DISCONNECTED, exception) + if self.transport: + await self.transport.dispose() + self.tranpsort = None + self.__connected_future.set_exception(exception) + raise exception async def send_protocol_message(self, protocol_message): - raw_msg = json.dumps(protocol_message) - log.info('send_protocol_message(): sending {raw_msg}') - await self.__websocket.send(raw_msg) - - async def setup_ws(self): - headers = HttpUtils.default_headers() - protocol_version = Defaults.protocol_version - params = {"key": self.__ably.key, "v": protocol_version} - query_params = urllib.parse.urlencode(params) - ws_url = (f'wss://{self.options.get_realtime_host()}?{query_params}') - log.info(f'setup_ws(): attempting to connect to {ws_url}') - async with websockets.connect(ws_url, extra_headers=headers) as websocket: - log.info(f'setup_ws(): connection established to {ws_url}') - self.__websocket = websocket - task = self.__ably.options.loop.create_task(self.ws_read_loop()) - try: - await task - except AblyAuthException: - return + if self.transport is not None: + await self.transport.send(protocol_message) + else: + raise Exception() async def ping(self): if self.__ping_future: @@ -258,48 +255,47 @@ async def ping(self): response_time_ms = (ping_end_time - ping_start_time) * 1000 return round(response_time_ms, 2) - async def ws_read_loop(self): - while True: - raw = await self.__websocket.recv() - msg = json.loads(raw) - log.info(f'ws_read_loop(): receieved protocol message: {msg}') - action = msg['action'] - if action == ProtocolMessageAction.CONNECTED: # CONNECTED + async def on_protocol_message(self, msg): + action = msg['action'] + if action == ProtocolMessageAction.CONNECTED: # CONNECTED + if self.transport: + self.transport.is_connected = True + if self.__connected_future: + if not self.__connected_future.cancelled(): + self.__connected_future.set_result(None) + self.__connected_future = None + else: + log.warn('CONNECTED message received but connected_future not set') + self.enact_state_change(ConnectionState.CONNECTED) + if action == ProtocolMessageAction.ERROR: # ERROR + error = msg["error"] + if error['nonfatal'] is False: + exception = AblyAuthException(error["message"], error["statusCode"], error["code"]) + self.enact_state_change(ConnectionState.FAILED, exception) if self.__connected_future: - if not self.__connected_future.cancelled(): - self.__connected_future.set_result(None) + self.__connected_future.set_exception(exception) self.__connected_future = None - else: - log.warn('CONNECTED message received but connected_future not set') - if action == ProtocolMessageAction.ERROR: # ERROR - error = msg["error"] - if error['nonfatal'] is False: - exception = AblyAuthException(error["message"], error["statusCode"], error["code"]) - self.enact_state_change(ConnectionState.FAILED, exception) - if self.__connected_future: - self.__connected_future.set_exception(exception) - self.__connected_future = None - self.__websocket = None - raise exception - if action == ProtocolMessageAction.CLOSED: - await self.__websocket.close() - self.__websocket = None - self.__closed_future.set_result(None) - break - if action == ProtocolMessageAction.HEARTBEAT: - if self.__ping_future: - # Resolve on heartbeat from ping request. - # TODO: Handle Normal heartbeat if required - if self.__ping_id == msg.get("id"): - if not self.__ping_future.cancelled(): - self.__ping_future.set_result(None) - self.__ping_future = None - if action in ( - ProtocolMessageAction.ATTACHED, - ProtocolMessageAction.DETACHED, - ProtocolMessageAction.MESSAGE - ): - self.__ably.channels._on_channel_message(msg) + if self.transport: + await self.transport.dispose() + raise exception + if action == ProtocolMessageAction.CLOSED: + if self.transport: + await self.transport.dispose() + self.__closed_future.set_result(None) + if action == ProtocolMessageAction.HEARTBEAT: + if self.__ping_future: + # Resolve on heartbeat from ping request. + # TODO: Handle Normal heartbeat if required + if self.__ping_id == msg.get("id"): + if not self.__ping_future.cancelled(): + self.__ping_future.set_result(None) + self.__ping_future = None + if action in ( + ProtocolMessageAction.ATTACHED, + ProtocolMessageAction.DETACHED, + ProtocolMessageAction.MESSAGE + ): + self.__ably.channels._on_channel_message(msg) @property def ably(self): diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 4bc0aaa9..c9c73dd4 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -86,7 +86,6 @@ def __init__(self, key=None, loop=None, **kwargs): self.key = key self.__connection = Connection(self) self.__channels = Channels(self) - print(options.auto_connect, "+++") if options.auto_connect: asyncio.ensure_future(self.connection.connection_manager.connect_impl()) diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index 1409e5bd..74ab0e1d 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -4,7 +4,9 @@ from enum import IntEnum import json import logging +import urllib.parse from ably.http.httputils import HttpUtils +from ably.transport.defaults import Defaults from websockets.client import WebSocketClientProtocol, connect as ws_connect from websockets.exceptions import ConnectionClosedOK @@ -37,6 +39,12 @@ def __init__(self, connection_manager: ConnectionManager): self.is_connected = False async def connect(self): + headers = HttpUtils.default_headers() + protocol_version = Defaults.protocol_version + params = {"key": self.connection_manager.ably.key, "v": protocol_version} + query_params = urllib.parse.urlencode(params) + ws_url = (f'wss://{self.connection_manager.options.get_realtime_host()}?{query_params}') + headers = HttpUtils.default_headers() host = self.connection_manager.options.get_realtime_host() key = self.connection_manager.ably.key diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 6d8b25c2..188c614a 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -156,13 +156,11 @@ async def new_send_protocol_message(msg): async def test_realtime_request_timeout_close(self): ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) await ably.connect() - original_send_protocol_message = ably.connection.connection_manager.send_protocol_message - async def new_send_protocol_message(msg): - if msg.get('action') == ProtocolMessageAction.CLOSE: - return - await original_send_protocol_message(msg) - ably.connection.connection_manager.send_protocol_message = new_send_protocol_message + async def new_close_transport(): + pass + + ably.connection.connection_manager.transport.close = new_close_transport with pytest.raises(AblyException) as exception: await ably.close() From 46ae9e5ea1ddcc9eba40646f65d0847c1e033726 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 29 Nov 2022 12:13:47 +0000 Subject: [PATCH 102/481] fix: await calls to ably.close() in tests --- test/ably/realtimeconnection_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 188c614a..2ee39b82 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -134,7 +134,7 @@ async def test_realtime_request_timeout_connect(self): assert exception.value.status_code == 504 assert ably.connection.state == ConnectionState.DISCONNECTED assert ably.connection.error_reason == exception.value - ably.close() + await ably.close() async def test_realtime_request_timeout_ping(self): ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) @@ -188,4 +188,4 @@ def on_state_change(state_change): assert len(state_changes) == 4 assert state_changes[0].previous == ConnectionState.CONNECTING assert state_changes[0].current == ConnectionState.DISCONNECTED - ably.close() + await ably.close() From 820890a552faad0b3088ab9d4ee5ebda7f72b636 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 29 Nov 2022 13:06:37 +0000 Subject: [PATCH 103/481] refactor: transition to DISCONNECTED synchronously on timeout --- ably/realtime/connection.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index f1e7aa13..4336e2aa 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -217,12 +217,13 @@ async def connect_impl(self): await asyncio.wait_for(asyncio.shield(self.__connected_future), self.__timeout_in_secs) except asyncio.TimeoutError: exception = AblyException("Timeout waiting for realtime connection", 504, 50003) - self.enact_state_change(ConnectionState.DISCONNECTED, exception) if self.transport: await self.transport.dispose() self.tranpsort = None self.__connected_future.set_exception(exception) - raise exception + connected_future = self.__connected_future + self.__connected_future = None + self.on_connection_attempt_done(connected_future) async def send_protocol_message(self, protocol_message): if self.transport is not None: From 3a9389d9bda216f6aa30c8b55a7a6b88a8bcb67d Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 29 Nov 2022 13:07:09 +0000 Subject: [PATCH 104/481] refactor: improve invalid state WebSocketTransport error --- ably/realtime/websockettransport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index 74ab0e1d..6451235f 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -83,7 +83,7 @@ async def ws_read_loop(self): self.ws_connect_task.cancel() await self.connection_manager.on_protocol_message(msg) else: - raise Exception() + raise Exception('ws_read_loop running with no websocket') def on_read_loop_done(self, task: asyncio.Task): try: From 7bb624c13470e0dbeb6f24f1c96a132acdc60fb8 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 29 Nov 2022 13:07:27 +0000 Subject: [PATCH 105/481] feat: reimplement disconnected_retry_timeout --- ably/realtime/connection.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 4336e2aa..b79898da 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -181,6 +181,11 @@ def on_connection_attempt_done(self, task): self.__connected_future.set_exception(exception) self.__connected_future = None self.enact_state_change(ConnectionState.DISCONNECTED, exception) + asyncio.create_task(self.retry_connection_attempt()) + + async def retry_connection_attempt(self): + await asyncio.sleep(self.ably.options.disconnected_retry_timeout / 1000) + self.try_connect() async def close(self): if self.__state in (ConnectionState.CLOSED, ConnectionState.INITIALIZED, ConnectionState.FAILED): From 9012175ff95b307c6b241a18ce6b939cab208ec1 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 29 Nov 2022 13:07:48 +0000 Subject: [PATCH 106/481] test: update test for disconnected_retry_timeout --- test/ably/realtimeconnection_test.py | 42 +++++++++++++++++----------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 2ee39b82..f495093a 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -168,24 +168,32 @@ async def new_close_transport(): assert exception.value.status_code == 504 async def test_disconnected_retry_timeout(self): - ably = await RestSetup.get_ably_realtime(realtime_request_timeout=0.001, - disconnected_retry_timeout=2000, auto_connect=False) - state_changes = [] + ably = await RestSetup.get_ably_realtime(disconnected_retry_timeout=2000, auto_connect=False) + original_connect = ably.connection.connection_manager._connect + call_count = 0 + test_future = asyncio.Future() + test_exception = Exception() + + # intercept the library connection mechanism to fail the first two connection attempts + async def new_connect(): + nonlocal call_count + if call_count < 2: + call_count += 1 + raise test_exception + else: + await original_connect() + test_future.set_result(None) + + ably.connection.connection_manager._connect = new_connect + + with pytest.raises(Exception) as exception: + await ably.connect() - def on_state_change(state_change): - state_changes.append(state_change) + assert ably.connection.state == ConnectionState.DISCONNECTED + assert exception.value == test_exception - ably.connection.on(on_state_change) + await test_future + + assert ably.connection.state == ConnectionState.CONNECTED - with pytest.raises(AblyException) as exception: - await ably.connect() - assert exception.value.code == 50003 - assert exception.value.status_code == 504 - assert ably.connection.state == ConnectionState.DISCONNECTED - # 2 state changes happens per retry. - # Retry timeout of 2 secs, will retry connection twice in 3 and/or 4 seconds, resulting in 4 state changes - await asyncio.sleep(4) - assert len(state_changes) == 4 - assert state_changes[0].previous == ConnectionState.CONNECTING - assert state_changes[0].current == ConnectionState.DISCONNECTED await ably.close() From 6783148406f66ff53e57351553d031b10a5a197d Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 6 Dec 2022 10:35:19 +0000 Subject: [PATCH 107/481] test: add fixture for connection to unroutable host --- test/ably/realtimeconnection_test.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index f495093a..1c8ec292 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -197,3 +197,12 @@ async def new_connect(): assert ably.connection.state == ConnectionState.CONNECTED await ably.close() + + async def test_unroutable_host(self): + ably = await RestSetup.get_ably_realtime(realtime_host="10.255.255.1") + with pytest.raises(AblyException) as exception: + await ably.connect() + assert exception.value.code == 50003 + assert exception.value.status_code == 504 + assert ably.connection.state == ConnectionState.DISCONNECTED + assert ably.connection.error_reason == exception.value From eef4b429c0c9ce4998ad49e1ae30c538774785df Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 6 Dec 2022 10:36:38 +0000 Subject: [PATCH 108/481] fix: remove errant variable shadowing for ws_url --- ably/realtime/websockettransport.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index 6451235f..96adc617 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -44,11 +44,6 @@ async def connect(self): params = {"key": self.connection_manager.ably.key, "v": protocol_version} query_params = urllib.parse.urlencode(params) ws_url = (f'wss://{self.connection_manager.options.get_realtime_host()}?{query_params}') - - headers = HttpUtils.default_headers() - host = self.connection_manager.options.get_realtime_host() - key = self.connection_manager.ably.key - ws_url = f'wss://{host}?key={key}' log.info(f'connect(): attempting to connect to {ws_url}') self.ws_connect_task = asyncio.create_task(self.ws_connect(ws_url, headers)) self.ws_connect_task.add_done_callback(self.on_ws_connect_done) From 8d6ec545494228b8dbbca301bf18d4cbac13bdc5 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 6 Dec 2022 10:38:58 +0000 Subject: [PATCH 109/481] refactor: wrap websocket opening errors in AblyExceptions --- ably/realtime/websockettransport.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index 96adc617..7832ed7d 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -7,8 +7,9 @@ import urllib.parse from ably.http.httputils import HttpUtils from ably.transport.defaults import Defaults +from ably.util.exceptions import AblyException from websockets.client import WebSocketClientProtocol, connect as ws_connect -from websockets.exceptions import ConnectionClosedOK +from websockets.exceptions import ConnectionClosedOK, WebSocketException if TYPE_CHECKING: from ably.realtime.connection import ConnectionManager @@ -57,12 +58,15 @@ def on_ws_connect_done(self, task: asyncio.Task): return async def ws_connect(self, ws_url, headers): - async with ws_connect(ws_url, extra_headers=headers) as websocket: - log.info(f'ws_connect(): connection established to {ws_url}') - self.websocket = websocket - self.read_loop = self.connection_manager.options.loop.create_task(self.ws_read_loop()) - self.read_loop.add_done_callback(self.on_read_loop_done) - await self.read_loop + try: + async with ws_connect(ws_url, extra_headers=headers) as websocket: + log.info(f'ws_connect(): connection established to {ws_url}') + self.websocket = websocket + self.read_loop = self.connection_manager.options.loop.create_task(self.ws_read_loop()) + self.read_loop.add_done_callback(self.on_read_loop_done) + await self.read_loop + except WebSocketException as e: + raise AblyException(f'Error opening websocket connection: {e.message}', 400, 40000) async def ws_read_loop(self): while True: From 29625a4889180a4bc76615d91f8cc45aab6eed27 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 6 Dec 2022 10:39:25 +0000 Subject: [PATCH 110/481] refactor: ProtocolMessageAction enum ascending order --- ably/realtime/websockettransport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index 7832ed7d..a6b33000 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -20,9 +20,9 @@ class ProtocolMessageAction(IntEnum): HEARTBEAT = 0 CONNECTED = 4 - ERROR = 9 CLOSE = 7 CLOSED = 8 + ERROR = 9 ATTACH = 10 ATTACHED = 11 DETACH = 12 From f589721a383fccfaa53eb22f74c31c250867a137 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 6 Dec 2022 10:41:02 +0000 Subject: [PATCH 111/481] refactor: handle socket.gaierror from websocket connection --- ably/realtime/websockettransport.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index a6b33000..485480b6 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -4,6 +4,7 @@ from enum import IntEnum import json import logging +import socket import urllib.parse from ably.http.httputils import HttpUtils from ably.transport.defaults import Defaults @@ -65,7 +66,7 @@ async def ws_connect(self, ws_url, headers): self.read_loop = self.connection_manager.options.loop.create_task(self.ws_read_loop()) self.read_loop.add_done_callback(self.on_read_loop_done) await self.read_loop - except WebSocketException as e: + except (WebSocketException, socket.gaierror) as e: raise AblyException(f'Error opening websocket connection: {e.message}', 400, 40000) async def ws_read_loop(self): From 7c1fc6ce212fb401b14115408db7248c84eff675 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 6 Dec 2022 10:54:41 +0000 Subject: [PATCH 112/481] refactor: finish connection attempt on ws opening failure --- ably/realtime/websockettransport.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index 485480b6..3aeafccf 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -55,8 +55,11 @@ def on_ws_connect_done(self, task: asyncio.Task): exception = task.exception() except asyncio.CancelledError as e: exception = e - if isinstance(exception, ConnectionClosedOK): + if exception is None or isinstance(exception, ConnectionClosedOK): return + connected_future = asyncio.Future() + connected_future.set_exception(exception) + self.connection_manager.on_connection_attempt_done(connected_future) async def ws_connect(self, ws_url, headers): try: @@ -67,7 +70,7 @@ async def ws_connect(self, ws_url, headers): self.read_loop.add_done_callback(self.on_read_loop_done) await self.read_loop except (WebSocketException, socket.gaierror) as e: - raise AblyException(f'Error opening websocket connection: {e.message}', 400, 40000) + raise AblyException(f'Error opening websocket connection: {e}', 400, 40000) async def ws_read_loop(self): while True: From d1aedc1d83724e00920309f5db2998f06bb55889 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 6 Dec 2022 10:55:02 +0000 Subject: [PATCH 113/481] test: add test fixture for connection with invalid host --- test/ably/realtimeconnection_test.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 1c8ec292..86883f25 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -206,3 +206,12 @@ async def test_unroutable_host(self): assert exception.value.status_code == 504 assert ably.connection.state == ConnectionState.DISCONNECTED assert ably.connection.error_reason == exception.value + + async def test_invalid_host(self): + ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost") + with pytest.raises(AblyException) as exception: + await ably.connect() + assert exception.value.code == 40000 + assert exception.value.status_code == 400 + assert ably.connection.state == ConnectionState.DISCONNECTED + assert ably.connection.error_reason == exception.value From dffacbb88ac698f311b89842909d7001d916c912 Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Fri, 16 Dec 2022 10:21:29 +0000 Subject: [PATCH 114/481] add realtime_hosts option to realtime client --- ably/realtime/realtime.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index c9c73dd4..75e3270a 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -61,6 +61,9 @@ def __init__(self, key=None, loop=None, **kwargs): disconnected_retry_timeout: float If the connection is still in the DISCONNECTED state after this delay, the client library will attempt to reconnect automatically. The default is 15 seconds. + fallback_hosts: list[str] + An array of fallback hosts to be used in the case of an error necessitating the use of an alternative host. + If you have been provided a set of custom fallback hosts by Ably, please specify them here. Raises ------ ValueError From 784c3e927208ecb7a92121294765cdc3832a614f Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Thu, 22 Dec 2022 15:27:50 +0000 Subject: [PATCH 115/481] add connection check function --- ably/realtime/connection.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index b79898da..107494cb 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -1,6 +1,7 @@ import functools import logging import asyncio +import httpx from ably.realtime.websockettransport import WebSocketTransport, ProtocolMessageAction from ably.util.exceptions import AblyAuthException, AblyException from ably.util.eventemitter import EventEmitter @@ -166,6 +167,13 @@ async def _connect(self): self.enact_state_change(ConnectionState.CONNECTING) await self.connect_impl() + def check_connection(self): + try: + response = httpx.get("https://internet-up.ably-realtime.com/is-the-internet-up.txt") + return response.status_code == 200 and response.text == "yes" + finally: + return False + def on_connection_attempt_done(self, task): try: exception = task.exception() From 352d684d4d9dfd479a3ee18c1254434009ed74b1 Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Wed, 4 Jan 2023 13:14:44 +0000 Subject: [PATCH 116/481] fix missing newline at the end of the internet up check response --- ably/realtime/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 107494cb..427adcfe 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -170,7 +170,7 @@ async def _connect(self): def check_connection(self): try: response = httpx.get("https://internet-up.ably-realtime.com/is-the-internet-up.txt") - return response.status_code == 200 and response.text == "yes" + return response.status_code == 200 and response.text == "yes\n" finally: return False From 07592772052cfb026b3d34b56ea7d22f2f1dd22a Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 6 Dec 2022 13:37:39 +0000 Subject: [PATCH 117/481] implement connection_state_ttl --- ably/realtime/connection.py | 31 +++++++++++++++++++++++++++++-- ably/transport/defaults.py | 4 +++- ably/types/options.py | 25 ++++++++++++++++++++----- 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index b79898da..4082d5e0 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -21,6 +21,7 @@ class ConnectionState(str, Enum): CLOSING = 'closing' CLOSED = 'closed' FAILED = 'failed' + SUSPENDED = "suspended" @dataclass @@ -131,13 +132,27 @@ def __init__(self, realtime, initial_state): self.__ping_future = None self.__timeout_in_secs = self.options.realtime_request_timeout / 1000 self.transport: WebSocketTransport | None = None + self.__ttl_task = None + self.__retry_task = None super().__init__() def enact_state_change(self, state, reason=None): current_state = self.__state self.__state = state + print(self.state, "enact") + if self.state == ConnectionState.DISCONNECTED: + if not self.__ttl_task or self.__ttl_task.done(): + self.__ttl_task = asyncio.create_task(self.__connection_state_ttl()) self._emit('connectionstate', ConnectionStateChange(current_state, state, reason)) + async def __connection_state_ttl(self): + await asyncio.sleep(self.ably.options.connection_state_ttl) + exception = AblyException("Exceeded connectionStateTtl while in DISCONNECTED state", 504, 50003) + self.enact_state_change(ConnectionState.SUSPENDED, exception) + if self.__retry_task: + self.__retry_task.cancel() + asyncio.create_task(self.retry_connection_attempt()) + async def connect(self): if not self.__connected_future: self.__connected_future = asyncio.Future() @@ -145,11 +160,13 @@ async def connect(self): await self.__connected_future def try_connect(self): + print("erm", self.__state) task = asyncio.create_task(self._connect()) task.add_done_callback(self.on_connection_attempt_done) async def _connect(self): if self.__state == ConnectionState.CONNECTED: + self.__ttl_task.cancel() return if self.__state == ConnectionState.CONNECTING: @@ -177,14 +194,22 @@ def on_connection_attempt_done(self, task): if self.__state in (ConnectionState.CLOSED, ConnectionState.FAILED): return if self.__state != ConnectionState.DISCONNECTED: + print("howdy") if self.__connected_future: self.__connected_future.set_exception(exception) self.__connected_future = None self.enact_state_change(ConnectionState.DISCONNECTED, exception) - asyncio.create_task(self.retry_connection_attempt()) + self.__retry_task = asyncio.create_task(self.retry_connection_attempt()) async def retry_connection_attempt(self): - await asyncio.sleep(self.ably.options.disconnected_retry_timeout / 1000) + print("retrying", self.__state) + if self.state == ConnectionState.SUSPENDED: + print("suspended") + retry_timeout = self.ably.options.suspended_retry_timeout / 1000 + else: + print("not yet") + retry_timeout = self.ably.options.disconnected_retry_timeout / 1000 + await asyncio.sleep(retry_timeout) self.try_connect() async def close(self): @@ -272,6 +297,8 @@ async def on_protocol_message(self, msg): self.__connected_future = None else: log.warn('CONNECTED message received but connected_future not set') + if self.__ttl_task: + self.__ttl_task.cancel() self.enact_state_change(ConnectionState.CONNECTED) if action == ProtocolMessageAction.ERROR: # ERROR error = msg["error"] diff --git a/ably/transport/defaults.py b/ably/transport/defaults.py index 6b0fec88..915d3ef8 100644 --- a/ably/transport/defaults.py +++ b/ably/transport/defaults.py @@ -20,7 +20,9 @@ class Defaults: comet_recv_timeout = 90000 comet_send_timeout = 10000 realtime_request_timeout = 10000 - disconnected_retry_timeout = 1500 + disconnected_retry_timeout = 15000 + connection_state_ttl = 120000 + suspended_retry_timeout = 30000 transports = [] # ["web_socket", "comet"] diff --git a/ably/types/options.py b/ably/types/options.py index 0a926992..e4d8aef1 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -9,14 +9,13 @@ class Options(AuthOptions): - def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, - realtime_host=None, port=0, tls_port=0, use_binary_protocol=True, - queue_messages=False, recover=False, environment=None, + def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realtime_host=None, port=0, + tls_port=0, use_binary_protocol=True, queue_messages=False, recover=False, environment=None, http_open_timeout=None, http_request_timeout=None, realtime_request_timeout=None, http_max_retry_count=None, http_max_retry_duration=None, fallback_hosts=None, fallback_hosts_use_default=None, fallback_retry_timeout=None, disconnected_retry_timeout=None, - idempotent_rest_publishing=None, loop=None, auto_connect=True, - **kwargs): + idempotent_rest_publishing=None, loop=None, auto_connect=True, connection_state_ttl=None, + suspended_retry_timeout=None, **kwargs): super().__init__(**kwargs) # TODO check these defaults @@ -29,6 +28,12 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, if disconnected_retry_timeout is None: disconnected_retry_timeout = Defaults.disconnected_retry_timeout + if connection_state_ttl is None: + connection_state_ttl = Defaults.connection_state_ttl + + if suspended_retry_timeout is None: + suspended_retry_timeout = Defaults.suspended_retry_timeout + if environment is not None and rest_host is not None: raise ValueError('specify rest_host or environment, not both') @@ -62,6 +67,8 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, self.__idempotent_rest_publishing = idempotent_rest_publishing self.__loop = loop self.__auto_connect = auto_connect + self.__connection_state_ttl = connection_state_ttl + self.__suspended_retry_timeout = suspended_retry_timeout self.__rest_hosts = self.__get_rest_hosts() self.__realtime_hosts = self.__get_realtime_hosts() @@ -214,6 +221,14 @@ def loop(self): def auto_connect(self): return self.__auto_connect + @property + def connection_state_ttl(self): + return self.__connection_state_ttl + + @property + def suspended_retry_timeout(self): + return self.__suspended_retry_timeout + def __get_rest_hosts(self): """ Return the list of hosts as they should be tried. First comes the main From b0ddbc234814116a0f4e6418c3e8f6c6cb51e5e2 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 7 Dec 2022 15:06:42 +0000 Subject: [PATCH 118/481] override ttl with connection details ttl --- ably/realtime/connection.py | 16 +++++++++------- ably/types/options.py | 4 ++++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 4082d5e0..a56ea69b 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -134,19 +134,21 @@ def __init__(self, realtime, initial_state): self.transport: WebSocketTransport | None = None self.__ttl_task = None self.__retry_task = None + self.__connection_details = None super().__init__() def enact_state_change(self, state, reason=None): current_state = self.__state self.__state = state - print(self.state, "enact") if self.state == ConnectionState.DISCONNECTED: if not self.__ttl_task or self.__ttl_task.done(): self.__ttl_task = asyncio.create_task(self.__connection_state_ttl()) self._emit('connectionstate', ConnectionStateChange(current_state, state, reason)) async def __connection_state_ttl(self): - await asyncio.sleep(self.ably.options.connection_state_ttl) + if self.__connection_details: + self.ably.options.connection_state_ttl = self.__connection_details["connectionStateTtl"] + await asyncio.sleep(self.ably.options.connection_state_ttl / 1000) exception = AblyException("Exceeded connectionStateTtl while in DISCONNECTED state", 504, 50003) self.enact_state_change(ConnectionState.SUSPENDED, exception) if self.__retry_task: @@ -160,7 +162,6 @@ async def connect(self): await self.__connected_future def try_connect(self): - print("erm", self.__state) task = asyncio.create_task(self._connect()) task.add_done_callback(self.on_connection_attempt_done) @@ -194,7 +195,6 @@ def on_connection_attempt_done(self, task): if self.__state in (ConnectionState.CLOSED, ConnectionState.FAILED): return if self.__state != ConnectionState.DISCONNECTED: - print("howdy") if self.__connected_future: self.__connected_future.set_exception(exception) self.__connected_future = None @@ -202,12 +202,9 @@ def on_connection_attempt_done(self, task): self.__retry_task = asyncio.create_task(self.retry_connection_attempt()) async def retry_connection_attempt(self): - print("retrying", self.__state) if self.state == ConnectionState.SUSPENDED: - print("suspended") retry_timeout = self.ably.options.suspended_retry_timeout / 1000 else: - print("not yet") retry_timeout = self.ably.options.disconnected_retry_timeout / 1000 await asyncio.sleep(retry_timeout) self.try_connect() @@ -299,6 +296,7 @@ async def on_protocol_message(self, msg): log.warn('CONNECTED message received but connected_future not set') if self.__ttl_task: self.__ttl_task.cancel() + self.__connection_details = msg['connectionDetails'] self.enact_state_change(ConnectionState.CONNECTED) if action == ProtocolMessageAction.ERROR: # ERROR error = msg["error"] @@ -337,3 +335,7 @@ def ably(self): @property def state(self): return self.__state + + @property + def connection_details(self): + return self.__connection_details diff --git a/ably/types/options.py b/ably/types/options.py index e4d8aef1..70b79b40 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -225,6 +225,10 @@ def auto_connect(self): def connection_state_ttl(self): return self.__connection_state_ttl + @connection_state_ttl.setter + def connection_state_ttl(self, value): + self.__connection_state_ttl = value + @property def suspended_retry_timeout(self): return self.__suspended_retry_timeout From 224d86c5714ecf04541f59ef5e518217990f9cad Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Thu, 8 Dec 2022 12:45:07 +0000 Subject: [PATCH 119/481] update suspended state behaviour --- ably/realtime/connection.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index a56ea69b..ec4a647b 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -135,12 +135,13 @@ def __init__(self, realtime, initial_state): self.__ttl_task = None self.__retry_task = None self.__connection_details = None + self.__in_suspended_state = False super().__init__() def enact_state_change(self, state, reason=None): current_state = self.__state self.__state = state - if self.state == ConnectionState.DISCONNECTED: + if self.__state == ConnectionState.DISCONNECTED: if not self.__ttl_task or self.__ttl_task.done(): self.__ttl_task = asyncio.create_task(self.__connection_state_ttl()) self._emit('connectionstate', ConnectionStateChange(current_state, state, reason)) @@ -151,9 +152,10 @@ async def __connection_state_ttl(self): await asyncio.sleep(self.ably.options.connection_state_ttl / 1000) exception = AblyException("Exceeded connectionStateTtl while in DISCONNECTED state", 504, 50003) self.enact_state_change(ConnectionState.SUSPENDED, exception) + self.__in_suspended_state = True if self.__retry_task: self.__retry_task.cancel() - asyncio.create_task(self.retry_connection_attempt()) + self.__retry_task = asyncio.create_task(self.retry_connection_attempt()) async def connect(self): if not self.__connected_future: @@ -167,7 +169,8 @@ def try_connect(self): async def _connect(self): if self.__state == ConnectionState.CONNECTED: - self.__ttl_task.cancel() + if self.__ttl_task: + self.__ttl_task.cancel() return if self.__state == ConnectionState.CONNECTING: @@ -198,14 +201,18 @@ def on_connection_attempt_done(self, task): if self.__connected_future: self.__connected_future.set_exception(exception) self.__connected_future = None - self.enact_state_change(ConnectionState.DISCONNECTED, exception) + if self.__in_suspended_state: + self.enact_state_change(ConnectionState.SUSPENDED, exception) + else: + self.enact_state_change(ConnectionState.DISCONNECTED, exception) self.__retry_task = asyncio.create_task(self.retry_connection_attempt()) async def retry_connection_attempt(self): - if self.state == ConnectionState.SUSPENDED: + if self.__in_suspended_state: retry_timeout = self.ably.options.suspended_retry_timeout / 1000 else: retry_timeout = self.ably.options.disconnected_retry_timeout / 1000 + await asyncio.sleep(retry_timeout) self.try_connect() @@ -294,6 +301,7 @@ async def on_protocol_message(self, msg): self.__connected_future = None else: log.warn('CONNECTED message received but connected_future not set') + self.__in_suspended_state = False if self.__ttl_task: self.__ttl_task.cancel() self.__connection_details = msg['connectionDetails'] From 32cb485ac666e71718072740d08788a6e18b656f Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Fri, 9 Dec 2022 14:15:46 +0000 Subject: [PATCH 120/481] add test for connection state ttl --- ably/realtime/connection.py | 6 ++++-- ably/realtime/realtime.py | 11 +++++++++-- test/ably/realtimeconnection_test.py | 23 +++++++++++++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index ec4a647b..613c954c 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -212,7 +212,6 @@ async def retry_connection_attempt(self): retry_timeout = self.ably.options.suspended_retry_timeout / 1000 else: retry_timeout = self.ably.options.disconnected_retry_timeout / 1000 - await asyncio.sleep(retry_timeout) self.try_connect() @@ -242,7 +241,10 @@ async def close(self): log.warning('ConnectionManager: called close with no connected transport') self.enact_state_change(ConnectionState.CLOSED) if self.transport and self.transport.ws_connect_task is not None: - await self.transport.ws_connect_task + try: + await self.transport.ws_connect_task + except AblyException as e: + log.warning(f'Connection error encountered while closing: {e}') async def connect_impl(self): self.transport = WebSocketTransport(self) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 75e3270a..9b744217 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -62,8 +62,15 @@ def __init__(self, key=None, loop=None, **kwargs): If the connection is still in the DISCONNECTED state after this delay, the client library will attempt to reconnect automatically. The default is 15 seconds. fallback_hosts: list[str] - An array of fallback hosts to be used in the case of an error necessitating the use of an alternative host. - If you have been provided a set of custom fallback hosts by Ably, please specify them here. + An array of fallback hosts to be used in the case of an error necessitating the use of an + alternative host. If you have been provided a set of custom fallback hosts by Ably, please specify + them here. + connection_state_ttl: float + The duration that Ably will persist the connection state for when a Realtime client is abruptly + disconnected. + suspended_retry_timeout: float + When the connection enters the SUSPENDED state, after this delay, if the state is still SUSPENDED, + the client library attempts to reconnect automatically. The default is 30 seconds. Raises ------ ValueError diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 86883f25..3521e6bb 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -206,6 +206,7 @@ async def test_unroutable_host(self): assert exception.value.status_code == 504 assert ably.connection.state == ConnectionState.DISCONNECTED assert ably.connection.error_reason == exception.value + await ably.close() async def test_invalid_host(self): ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost") @@ -215,3 +216,25 @@ async def test_invalid_host(self): assert exception.value.status_code == 400 assert ably.connection.state == ConnectionState.DISCONNECTED assert ably.connection.error_reason == exception.value + await ably.close() + + async def test_connection_state_ttl(self): + ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost", connection_state_ttl=2000) + changes = [] + suspended_future = asyncio.Future() + + def on_state_change(state_change): + changes.append(state_change) + if state_change.current == ConnectionState.SUSPENDED: + suspended_future.set_result(None) + with pytest.raises(AblyException) as exception: + await ably.connect() + ably.connection.on(on_state_change) + assert exception.value.code == 40000 + assert exception.value.status_code == 400 + assert ably.connection.state == ConnectionState.DISCONNECTED + await suspended_future + assert ably.connection.state == changes[-1].current + assert ably.connection.state == ConnectionState.SUSPENDED + assert ably.connection.error_reason == changes[-1].reason + await ably.close() From c5602a805f9a8bc46f82718dceb297990c2d79eb Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Thu, 5 Jan 2023 15:33:48 +0000 Subject: [PATCH 121/481] implememt review --- ably/realtime/connection.py | 11 ++++++++--- test/ably/realtimeconnection_test.py | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 613c954c..a0fc0b75 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -121,6 +121,10 @@ def state(self, value): def connection_manager(self): return self.__connection_manager + @property + def connection_details(self): + return self.__connection_manager.connection_details + class ConnectionManager(EventEmitter): def __init__(self, realtime, initial_state): @@ -143,15 +147,16 @@ def enact_state_change(self, state, reason=None): self.__state = state if self.__state == ConnectionState.DISCONNECTED: if not self.__ttl_task or self.__ttl_task.done(): - self.__ttl_task = asyncio.create_task(self.__connection_state_ttl()) + self.__ttl_task = asyncio.create_task(self.__start_suspended_timer()) self._emit('connectionstate', ConnectionStateChange(current_state, state, reason)) - async def __connection_state_ttl(self): + async def __start_suspended_timer(self): if self.__connection_details: self.ably.options.connection_state_ttl = self.__connection_details["connectionStateTtl"] await asyncio.sleep(self.ably.options.connection_state_ttl / 1000) exception = AblyException("Exceeded connectionStateTtl while in DISCONNECTED state", 504, 50003) self.enact_state_change(ConnectionState.SUSPENDED, exception) + self.__connection_details = None self.__in_suspended_state = True if self.__retry_task: self.__retry_task.cancel() @@ -306,7 +311,7 @@ async def on_protocol_message(self, msg): self.__in_suspended_state = False if self.__ttl_task: self.__ttl_task.cancel() - self.__connection_details = msg['connectionDetails'] + self.__connection_details = msg.get('connectionDetails') self.enact_state_change(ConnectionState.CONNECTED) if action == ProtocolMessageAction.ERROR: # ERROR error = msg["error"] diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 3521e6bb..806f0097 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -236,5 +236,6 @@ def on_state_change(state_change): await suspended_future assert ably.connection.state == changes[-1].current assert ably.connection.state == ConnectionState.SUSPENDED + assert ably.connection.connection_details is None assert ably.connection.error_reason == changes[-1].reason await ably.close() From 671a2481a8bb4cfa662aa50ad000bac0f4428926 Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Fri, 6 Jan 2023 10:40:54 +0000 Subject: [PATCH 122/481] change to FAILED state when unable to connect --- ably/realtime/connection.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 427adcfe..1adc7491 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -170,7 +170,7 @@ async def _connect(self): def check_connection(self): try: response = httpx.get("https://internet-up.ably-realtime.com/is-the-internet-up.txt") - return response.status_code == 200 and response.text == "yes\n" + return response.status_code == 200 and "yes" in response.text finally: return False @@ -192,8 +192,12 @@ def on_connection_attempt_done(self, task): asyncio.create_task(self.retry_connection_attempt()) async def retry_connection_attempt(self): - await asyncio.sleep(self.ably.options.disconnected_retry_timeout / 1000) - self.try_connect() + if self.check_connection(): + await asyncio.sleep(self.ably.options.disconnected_retry_timeout / 1000) + self.try_connect() + else: + exception = AblyException("Unable to connect (network unreachable)", 80003, 404) + self.enact_state_change(ConnectionState.FAILED, exception) async def close(self): if self.__state in (ConnectionState.CLOSED, ConnectionState.INITIALIZED, ConnectionState.FAILED): From ef6dcb9376293108c8c45bc6b03f3c3963356ff5 Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Fri, 6 Jan 2023 11:03:34 +0000 Subject: [PATCH 123/481] fix lint failing due to line that's too long --- ably/realtime/realtime.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 75e3270a..b1abc523 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -62,8 +62,9 @@ def __init__(self, key=None, loop=None, **kwargs): If the connection is still in the DISCONNECTED state after this delay, the client library will attempt to reconnect automatically. The default is 15 seconds. fallback_hosts: list[str] - An array of fallback hosts to be used in the case of an error necessitating the use of an alternative host. - If you have been provided a set of custom fallback hosts by Ably, please specify them here. + An array of fallback hosts to be used in the case of an error necessitating the use of + an alternative host.If you have been provided a set of custom fallback hosts by Ably, + please specify them here. Raises ------ ValueError From b309a2ddca8662508954fa03ec663414c8685b82 Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Fri, 6 Jan 2023 14:20:39 +0000 Subject: [PATCH 124/481] fix disconnected retry timeout test hanging --- ably/realtime/connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 1adc7491..57a761d8 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -171,7 +171,7 @@ def check_connection(self): try: response = httpx.get("https://internet-up.ably-realtime.com/is-the-internet-up.txt") return response.status_code == 200 and "yes" in response.text - finally: + except httpx.HTTPError: return False def on_connection_attempt_done(self, task): @@ -192,8 +192,8 @@ def on_connection_attempt_done(self, task): asyncio.create_task(self.retry_connection_attempt()) async def retry_connection_attempt(self): + await asyncio.sleep(self.ably.options.disconnected_retry_timeout / 1000) if self.check_connection(): - await asyncio.sleep(self.ably.options.disconnected_retry_timeout / 1000) self.try_connect() else: exception = AblyException("Unable to connect (network unreachable)", 80003, 404) From c484d33963906e37b4e6e52d408452c5501590ff Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 9 Jan 2023 17:08:45 +0000 Subject: [PATCH 125/481] review: refactor connection details --- ably/realtime/connection.py | 28 +++++++++++++++++++--------- ably/types/options.py | 7 +++---- test/ably/realtimeconnection_test.py | 2 +- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index a0fc0b75..bd1c1d09 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -31,6 +31,18 @@ class ConnectionStateChange: reason: Optional[AblyException] = None +@dataclass +class ConnectionDetails: + connectionStateTtl: int + + def __init__(self, connection_state_ttl: int): + self.connectionStateTtl = connection_state_ttl + + @staticmethod + def from_dict(json_dict: dict): + return ConnectionDetails(json_dict.get('connectionStateTtl')) + + class Connection(EventEmitter): """Ably Realtime Connection @@ -139,7 +151,7 @@ def __init__(self, realtime, initial_state): self.__ttl_task = None self.__retry_task = None self.__connection_details = None - self.__in_suspended_state = False + self.__fail_state = ConnectionState.DISCONNECTED super().__init__() def enact_state_change(self, state, reason=None): @@ -152,12 +164,12 @@ def enact_state_change(self, state, reason=None): async def __start_suspended_timer(self): if self.__connection_details: - self.ably.options.connection_state_ttl = self.__connection_details["connectionStateTtl"] + self.ably.options.connection_state_ttl = self.__connection_details.connectionStateTtl await asyncio.sleep(self.ably.options.connection_state_ttl / 1000) exception = AblyException("Exceeded connectionStateTtl while in DISCONNECTED state", 504, 50003) self.enact_state_change(ConnectionState.SUSPENDED, exception) self.__connection_details = None - self.__in_suspended_state = True + self.__fail_state = ConnectionState.SUSPENDED if self.__retry_task: self.__retry_task.cancel() self.__retry_task = asyncio.create_task(self.retry_connection_attempt()) @@ -174,8 +186,6 @@ def try_connect(self): async def _connect(self): if self.__state == ConnectionState.CONNECTED: - if self.__ttl_task: - self.__ttl_task.cancel() return if self.__state == ConnectionState.CONNECTING: @@ -206,14 +216,14 @@ def on_connection_attempt_done(self, task): if self.__connected_future: self.__connected_future.set_exception(exception) self.__connected_future = None - if self.__in_suspended_state: + if self.__fail_state == ConnectionState.SUSPENDED: self.enact_state_change(ConnectionState.SUSPENDED, exception) else: self.enact_state_change(ConnectionState.DISCONNECTED, exception) self.__retry_task = asyncio.create_task(self.retry_connection_attempt()) async def retry_connection_attempt(self): - if self.__in_suspended_state: + if self.__fail_state == ConnectionState.SUSPENDED: retry_timeout = self.ably.options.suspended_retry_timeout / 1000 else: retry_timeout = self.ably.options.disconnected_retry_timeout / 1000 @@ -308,10 +318,10 @@ async def on_protocol_message(self, msg): self.__connected_future = None else: log.warn('CONNECTED message received but connected_future not set') - self.__in_suspended_state = False + self.__fail_state == ConnectionState.DISCONNECTED if self.__ttl_task: self.__ttl_task.cancel() - self.__connection_details = msg.get('connectionDetails') + self.__connection_details = ConnectionDetails.from_dict(msg["connectionDetails"]) self.enact_state_change(ConnectionState.CONNECTED) if action == ProtocolMessageAction.ERROR: # ERROR error = msg["error"] diff --git a/ably/types/options.py b/ably/types/options.py index 70b79b40..c85f1c05 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -14,8 +14,8 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti http_open_timeout=None, http_request_timeout=None, realtime_request_timeout=None, http_max_retry_count=None, http_max_retry_duration=None, fallback_hosts=None, fallback_hosts_use_default=None, fallback_retry_timeout=None, disconnected_retry_timeout=None, - idempotent_rest_publishing=None, loop=None, auto_connect=True, connection_state_ttl=None, - suspended_retry_timeout=None, **kwargs): + idempotent_rest_publishing=None, loop=None, auto_connect=True, suspended_retry_timeout=None, + **kwargs): super().__init__(**kwargs) # TODO check these defaults @@ -28,8 +28,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti if disconnected_retry_timeout is None: disconnected_retry_timeout = Defaults.disconnected_retry_timeout - if connection_state_ttl is None: - connection_state_ttl = Defaults.connection_state_ttl + connection_state_ttl = Defaults.connection_state_ttl if suspended_retry_timeout is None: suspended_retry_timeout = Defaults.suspended_retry_timeout diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 806f0097..6045d7f7 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -219,7 +219,7 @@ async def test_invalid_host(self): await ably.close() async def test_connection_state_ttl(self): - ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost", connection_state_ttl=2000) + ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost") changes = [] suspended_future = asyncio.Future() From 88ebf336916fe445eaad4fa96d885c46b42fe0f1 Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Thu, 22 Dec 2022 15:27:50 +0000 Subject: [PATCH 126/481] add connection check function --- ably/realtime/connection.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index bd1c1d09..2834960b 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -1,6 +1,7 @@ import functools import logging import asyncio +import httpx from ably.realtime.websockettransport import WebSocketTransport, ProtocolMessageAction from ably.util.exceptions import AblyAuthException, AblyException from ably.util.eventemitter import EventEmitter @@ -202,6 +203,13 @@ async def _connect(self): self.enact_state_change(ConnectionState.CONNECTING) await self.connect_impl() + def check_connection(self): + try: + response = httpx.get("https://internet-up.ably-realtime.com/is-the-internet-up.txt") + return response.status_code == 200 and response.text == "yes" + finally: + return False + def on_connection_attempt_done(self, task): try: exception = task.exception() From e687c3c3b4cbaf79a0def7d5217d2c90df7cdc25 Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Wed, 4 Jan 2023 13:14:44 +0000 Subject: [PATCH 127/481] fix missing newline at the end of the internet up check response --- ably/realtime/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 2834960b..8ebae5ce 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -206,7 +206,7 @@ async def _connect(self): def check_connection(self): try: response = httpx.get("https://internet-up.ably-realtime.com/is-the-internet-up.txt") - return response.status_code == 200 and response.text == "yes" + return response.status_code == 200 and response.text == "yes\n" finally: return False From c1de730bd1d5e7350efc74875f65ee4467d8943e Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Fri, 6 Jan 2023 10:40:54 +0000 Subject: [PATCH 128/481] change to FAILED state when unable to connect --- ably/realtime/connection.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 8ebae5ce..0bcca6ca 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -206,7 +206,7 @@ async def _connect(self): def check_connection(self): try: response = httpx.get("https://internet-up.ably-realtime.com/is-the-internet-up.txt") - return response.status_code == 200 and response.text == "yes\n" + return response.status_code == 200 and "yes" in response.text finally: return False @@ -236,7 +236,11 @@ async def retry_connection_attempt(self): else: retry_timeout = self.ably.options.disconnected_retry_timeout / 1000 await asyncio.sleep(retry_timeout) - self.try_connect() + if self.check_connection(): + self.try_connect() + else: + exception = AblyException("Unable to connect (network unreachable)", 80003, 404) + self.enact_state_change(ConnectionState.FAILED, exception) async def close(self): if self.__state in (ConnectionState.CLOSED, ConnectionState.INITIALIZED, ConnectionState.FAILED): From 3a21bc1ce0cc91df8aa828e2c9bbc481ec1aaaec Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Fri, 6 Jan 2023 14:20:39 +0000 Subject: [PATCH 129/481] fix disconnected retry timeout test hanging --- ably/realtime/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 0bcca6ca..08a0c11e 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -207,7 +207,7 @@ def check_connection(self): try: response = httpx.get("https://internet-up.ably-realtime.com/is-the-internet-up.txt") return response.status_code == 200 and "yes" in response.text - finally: + except httpx.HTTPError: return False def on_connection_attempt_done(self, task): From da873bf78109632d3de18cc9bd7546f567074384 Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Wed, 11 Jan 2023 12:06:04 +0000 Subject: [PATCH 130/481] transition to fail state when network connection check fails --- ably/realtime/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 08a0c11e..25d106a8 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -240,7 +240,7 @@ async def retry_connection_attempt(self): self.try_connect() else: exception = AblyException("Unable to connect (network unreachable)", 80003, 404) - self.enact_state_change(ConnectionState.FAILED, exception) + self.enact_state_change(self.__fail_state, exception) async def close(self): if self.__state in (ConnectionState.CLOSED, ConnectionState.INITIALIZED, ConnectionState.FAILED): From be097189cfdef556dc3cbb13b8ae9069a0775296 Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Wed, 11 Jan 2023 12:48:56 +0000 Subject: [PATCH 131/481] add connectivity_check_url option and default --- ably/realtime/connection.py | 6 ++++-- ably/transport/defaults.py | 1 + ably/types/options.py | 11 ++++++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 25d106a8..a05e7425 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -3,6 +3,7 @@ import asyncio import httpx from ably.realtime.websockettransport import WebSocketTransport, ProtocolMessageAction +from ably.transport.defaults import Defaults from ably.util.exceptions import AblyAuthException, AblyException from ably.util.eventemitter import EventEmitter from enum import Enum @@ -205,8 +206,9 @@ async def _connect(self): def check_connection(self): try: - response = httpx.get("https://internet-up.ably-realtime.com/is-the-internet-up.txt") - return response.status_code == 200 and "yes" in response.text + response = httpx.get(self.options.connectivity_check_url) + return response.status_code == 200 and \ + (self.options.connectivity_check_url != Defaults.connectivity_check_url or "yes" in response.text) except httpx.HTTPError: return False diff --git a/ably/transport/defaults.py b/ably/transport/defaults.py index 915d3ef8..04c57031 100644 --- a/ably/transport/defaults.py +++ b/ably/transport/defaults.py @@ -10,6 +10,7 @@ class Defaults: rest_host = "rest.ably.io" realtime_host = "realtime.ably.io" + connectivity_check_url = "https://internet-up.ably-realtime.com/is-the-internet-up.txt" environment = 'production' port = 80 diff --git a/ably/types/options.py b/ably/types/options.py index c85f1c05..7aaab5eb 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -15,7 +15,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti http_max_retry_count=None, http_max_retry_duration=None, fallback_hosts=None, fallback_hosts_use_default=None, fallback_retry_timeout=None, disconnected_retry_timeout=None, idempotent_rest_publishing=None, loop=None, auto_connect=True, suspended_retry_timeout=None, - **kwargs): + connectivity_check_url=None, **kwargs): super().__init__(**kwargs) # TODO check these defaults @@ -28,6 +28,9 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti if disconnected_retry_timeout is None: disconnected_retry_timeout = Defaults.disconnected_retry_timeout + if connectivity_check_url is None: + connectivity_check_url = Defaults.connectivity_check_url + connection_state_ttl = Defaults.connection_state_ttl if suspended_retry_timeout is None: @@ -68,10 +71,12 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti self.__auto_connect = auto_connect self.__connection_state_ttl = connection_state_ttl self.__suspended_retry_timeout = suspended_retry_timeout + self.__connectivity_check_url = connectivity_check_url self.__rest_hosts = self.__get_rest_hosts() self.__realtime_hosts = self.__get_realtime_hosts() + @property def client_id(self): return self.__client_id @@ -232,6 +237,10 @@ def connection_state_ttl(self, value): def suspended_retry_timeout(self): return self.__suspended_retry_timeout + @property + def connectivity_check_url(self): + return self.__connectivity_check_url + def __get_rest_hosts(self): """ Return the list of hosts as they should be tried. First comes the main From 29da3958638c1fceefd2aa3c1d0ca3a3bb5846ad Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Wed, 11 Jan 2023 12:49:01 +0000 Subject: [PATCH 132/481] add retry_connection_attempt tests --- test/ably/realtimeconnection_test.py | 33 ++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 6045d7f7..a881c805 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -198,6 +198,39 @@ async def new_connect(): await ably.close() + async def test_connectivity_check_default(self): + ably = await RestSetup.get_ably_realtime() + # The default connectivity check should return True + assert ably.connection.connection_manager.check_connection() is True + + async def test_connectivity_check_non_default(self): + ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://httpbin.org/status/200") + # A non-default URL should return True with a HTTP OK despite not returning "Yes" in the body + assert ably.connection.connection_manager.check_connection() is True + + async def test_connectivity_check_bad_status(self): + ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://httpbin.org/status/400") + # Should return False when the URL returns a non-2xx response code + assert ably.connection.connection_manager.check_connection() is False + + async def test_retry_connection_attempt(self): + ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://httpbin.org/status/400", + disconnected_retry_timeout=1, auto_connect=False) + test_future = asyncio.Future() + + def on_state_change(change): + if change.current == ConnectionState.DISCONNECTED: + test_future.set_result(change) + + ably.connection.connection_manager.on('connectionstate', on_state_change) + + asyncio.create_task(ably.connection.connection_manager.retry_connection_attempt()) + + state_change = await test_future + + assert state_change.reason.status_code == 80003 + assert state_change.reason.message == "Unable to connect (network unreachable)" + async def test_unroutable_host(self): ably = await RestSetup.get_ably_realtime(realtime_host="10.255.255.1") with pytest.raises(AblyException) as exception: From 7153210059945a2453cc497297860ed14ebdfdb3 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 11 Jan 2023 12:48:27 +0000 Subject: [PATCH 133/481] refactor and update test --- ably/realtime/connection.py | 7 +++---- test/ably/realtimeconnection_test.py | 3 +++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index bd1c1d09..b19458e7 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -216,10 +216,7 @@ def on_connection_attempt_done(self, task): if self.__connected_future: self.__connected_future.set_exception(exception) self.__connected_future = None - if self.__fail_state == ConnectionState.SUSPENDED: - self.enact_state_change(ConnectionState.SUSPENDED, exception) - else: - self.enact_state_change(ConnectionState.DISCONNECTED, exception) + self.enact_state_change(self.__fail_state, exception) self.__retry_task = asyncio.create_task(self.retry_connection_attempt()) async def retry_connection_attempt(self): @@ -255,6 +252,8 @@ async def close(self): else: log.warning('ConnectionManager: called close with no connected transport') self.enact_state_change(ConnectionState.CLOSED) + if self.__ttl_task and not self.__ttl_task.done(): + self.__ttl_task.cancel() if self.transport and self.transport.ws_connect_task is not None: try: await self.transport.ws_connect_task diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 6045d7f7..f2c785b3 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -4,6 +4,7 @@ from ably.util.exceptions import AblyAuthException, AblyException from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase +from ably.transport.defaults import Defaults class TestRealtimeAuth(BaseAsyncTestCase): @@ -219,6 +220,7 @@ async def test_invalid_host(self): await ably.close() async def test_connection_state_ttl(self): + Defaults.connection_state_ttl = 100 ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost") changes = [] suspended_future = asyncio.Future() @@ -239,3 +241,4 @@ def on_state_change(state_change): assert ably.connection.connection_details is None assert ably.connection.error_reason == changes[-1].reason await ably.close() + Defaults.connection_state_ttl = 120000 From b26e5e45078b28669637af7db3cbb45e21a2bf30 Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Wed, 11 Jan 2023 13:23:51 +0000 Subject: [PATCH 134/481] check for all 2xx status codes in check_connection --- ably/realtime/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index a05e7425..b7b3d73b 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -207,7 +207,7 @@ async def _connect(self): def check_connection(self): try: response = httpx.get(self.options.connectivity_check_url) - return response.status_code == 200 and \ + return 200 <= response.status_code < 300 and \ (self.options.connectivity_check_url != Defaults.connectivity_check_url or "yes" in response.text) except httpx.HTTPError: return False From 37d7db75a9eca089060b232542487e685e9e7132 Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Wed, 11 Jan 2023 13:24:12 +0000 Subject: [PATCH 135/481] add documentation for connectivity_check_url option --- ably/realtime/realtime.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 9b744217..f3a6a71f 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -71,6 +71,11 @@ def __init__(self, key=None, loop=None, **kwargs): suspended_retry_timeout: float When the connection enters the SUSPENDED state, after this delay, if the state is still SUSPENDED, the client library attempts to reconnect automatically. The default is 30 seconds. + connectivity_check_url: string + Override the URL used by the realtime client to check if the internet is available. + In the event of a failure to connect to the primary endpoint, the client will send a + GET request to this URL to check if the internet is available. If this request returns + a success response the client will attempt to connect to a fallback host. Raises ------ ValueError From 30a00732ba33629b6d4f926f1a3f00f901dfc6aa Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Wed, 11 Jan 2023 13:24:53 +0000 Subject: [PATCH 136/481] remove errant newline --- ably/types/options.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ably/types/options.py b/ably/types/options.py index 7aaab5eb..4d7edfc4 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -76,7 +76,6 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti self.__rest_hosts = self.__get_rest_hosts() self.__realtime_hosts = self.__get_realtime_hosts() - @property def client_id(self): return self.__client_id From c62273f8615c189d0fcd4a95d4dee2e8b3c96cc2 Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Wed, 11 Jan 2023 13:26:00 +0000 Subject: [PATCH 137/481] use echo.ably.io for connectivity url tests --- test/ably/realtimeconnection_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index a881c805..29ea2cbb 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -204,17 +204,17 @@ async def test_connectivity_check_default(self): assert ably.connection.connection_manager.check_connection() is True async def test_connectivity_check_non_default(self): - ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://httpbin.org/status/200") + ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://echo.ably.io/respondWith?status=200") # A non-default URL should return True with a HTTP OK despite not returning "Yes" in the body assert ably.connection.connection_manager.check_connection() is True async def test_connectivity_check_bad_status(self): - ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://httpbin.org/status/400") + ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://echo.ably.io/respondWith?status=400") # Should return False when the URL returns a non-2xx response code assert ably.connection.connection_manager.check_connection() is False async def test_retry_connection_attempt(self): - ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://httpbin.org/status/400", + ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://echo.ably.io/respondWith?status=400", disconnected_retry_timeout=1, auto_connect=False) test_future = asyncio.Future() From d02403bc5648396827049d9be19df1da226c732d Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Wed, 11 Jan 2023 13:33:47 +0000 Subject: [PATCH 138/481] fix line too long linting on realtimeconnection_test.py --- test/ably/realtimeconnection_test.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 29ea2cbb..0501274e 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -204,18 +204,21 @@ async def test_connectivity_check_default(self): assert ably.connection.connection_manager.check_connection() is True async def test_connectivity_check_non_default(self): - ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://echo.ably.io/respondWith?status=200") + ably = await RestSetup.get_ably_realtime( + connectivity_check_url="https://echo.ably.io/respondWith?status=200") # A non-default URL should return True with a HTTP OK despite not returning "Yes" in the body assert ably.connection.connection_manager.check_connection() is True async def test_connectivity_check_bad_status(self): - ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://echo.ably.io/respondWith?status=400") + ably = await RestSetup.get_ably_realtime( + connectivity_check_url="https://echo.ably.io/respondWith?status=400") # Should return False when the URL returns a non-2xx response code assert ably.connection.connection_manager.check_connection() is False async def test_retry_connection_attempt(self): - ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://echo.ably.io/respondWith?status=400", - disconnected_retry_timeout=1, auto_connect=False) + ably = await RestSetup.get_ably_realtime( + connectivity_check_url="https://echo.ably.io/respondWith?status=400", disconnected_retry_timeout=1, + auto_connect=False) test_future = asyncio.Future() def on_state_change(change): From 5fe55475d73de8d547dddd0d4cbdf1e031e25743 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 19 Dec 2022 12:22:53 +0000 Subject: [PATCH 139/481] handle connected message --- ably/realtime/connection.py | 46 ++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index b19458e7..508f90aa 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -21,13 +21,26 @@ class ConnectionState(str, Enum): CLOSING = 'closing' CLOSED = 'closed' FAILED = 'failed' - SUSPENDED = "suspended" + SUSPENDED = 'suspended' + + +class ConnectionEvent(str): + INITIALIZED = 'initialized' + CONNECTING = 'connecting' + CONNECTED = 'connected' + DISCONNECTED = 'disconnected' + CLOSING = 'closing' + CLOSED = 'closed' + FAILED = 'failed' + SUSPENDED = 'suspended' + UPDATE = 'update' @dataclass class ConnectionStateChange: previous: ConnectionState current: ConnectionState + event: ConnectionEvent reason: Optional[AblyException] = None @@ -152,9 +165,10 @@ def __init__(self, realtime, initial_state): self.__retry_task = None self.__connection_details = None self.__fail_state = ConnectionState.DISCONNECTED + self.__fail_event = ConnectionEvent.DISCONNECTED super().__init__() - def enact_state_change(self, state, reason=None): + def enact_state_change(self, state, event, reason=None): current_state = self.__state self.__state = state if self.__state == ConnectionState.DISCONNECTED: @@ -167,9 +181,10 @@ async def __start_suspended_timer(self): self.ably.options.connection_state_ttl = self.__connection_details.connectionStateTtl await asyncio.sleep(self.ably.options.connection_state_ttl / 1000) exception = AblyException("Exceeded connectionStateTtl while in DISCONNECTED state", 504, 50003) - self.enact_state_change(ConnectionState.SUSPENDED, exception) + self.enact_state_change(ConnectionState.SUSPENDED, ConnectionEvent.SUSPENDED, exception) self.__connection_details = None self.__fail_state = ConnectionState.SUSPENDED + self.__fail_event = ConnectionEvent.SUSPENDED if self.__retry_task: self.__retry_task.cancel() self.__retry_task = asyncio.create_task(self.retry_connection_attempt()) @@ -199,7 +214,7 @@ async def _connect(self): log.info('Connection cancelled due to request timeout. Attempting reconnection...') raise exception else: - self.enact_state_change(ConnectionState.CONNECTING) + self.enact_state_change(ConnectionState.CONNECTING, ConnectionEvent.CONNECTING) await self.connect_impl() def on_connection_attempt_done(self, task): @@ -216,7 +231,11 @@ def on_connection_attempt_done(self, task): if self.__connected_future: self.__connected_future.set_exception(exception) self.__connected_future = None - self.enact_state_change(self.__fail_state, exception) + self.enact_state_change(self.__fail_state, self.__fail_event, exception) + # if self.__in_suspended_state: + # self.enact_state_change(ConnectionState.SUSPENDED, ConnectionEvent.SUSPENDED, exception) + # else: + # self.enact_state_change(ConnectionState.DISCONNECTED, ConnectionEvent.DISCONNECTED, exception) self.__retry_task = asyncio.create_task(self.retry_connection_attempt()) async def retry_connection_attempt(self): @@ -229,19 +248,19 @@ async def retry_connection_attempt(self): async def close(self): if self.__state in (ConnectionState.CLOSED, ConnectionState.INITIALIZED, ConnectionState.FAILED): - self.enact_state_change(ConnectionState.CLOSED) + self.enact_state_change(ConnectionState.CLOSED, ConnectionEvent.CLOSED) return if self.__state is ConnectionState.DISCONNECTED: if self.transport: await self.transport.dispose() self.transport = None - self.enact_state_change(ConnectionState.CLOSED) + self.enact_state_change(ConnectionState.CLOSED, ConnectionEvent.CLOSED) return if self.__state != ConnectionState.CONNECTED: log.warning('Connection.closed called while connection state not connected') if self.__state == ConnectionState.CONNECTING: await self.__connected_future - self.enact_state_change(ConnectionState.CLOSING) + self.enact_state_change(ConnectionState.CLOSING, ConnectionEvent.CLOSING) self.__closed_future = asyncio.Future() if self.transport and self.transport.is_connected: await self.transport.close() @@ -251,7 +270,7 @@ async def close(self): raise AblyException("Timeout waiting for connection close response", 504, 50003) else: log.warning('ConnectionManager: called close with no connected transport') - self.enact_state_change(ConnectionState.CLOSED) + self.enact_state_change(ConnectionState.CLOSED, ConnectionEvent.CLOSED) if self.__ttl_task and not self.__ttl_task.done(): self.__ttl_task.cancel() if self.transport and self.transport.ws_connect_task is not None: @@ -309,6 +328,7 @@ async def ping(self): async def on_protocol_message(self, msg): action = msg['action'] if action == ProtocolMessageAction.CONNECTED: # CONNECTED + msg_error = msg.get("error") if self.transport: self.transport.is_connected = True if self.__connected_future: @@ -318,15 +338,19 @@ async def on_protocol_message(self, msg): else: log.warn('CONNECTED message received but connected_future not set') self.__fail_state == ConnectionState.DISCONNECTED + self.__fail_event == ConnectionEvent.DISCONNECTED if self.__ttl_task: self.__ttl_task.cancel() self.__connection_details = ConnectionDetails.from_dict(msg["connectionDetails"]) - self.enact_state_change(ConnectionState.CONNECTED) + if self.__state == ConnectionState.CONNECTED: + self.enact_state_change(ConnectionState.CONNECTED, ConnectionEvent.UPDATE, msg_error) + else: + self.enact_state_change(ConnectionState.CONNECTED, ConnectionEvent.CONNECTED) if action == ProtocolMessageAction.ERROR: # ERROR error = msg["error"] if error['nonfatal'] is False: exception = AblyAuthException(error["message"], error["statusCode"], error["code"]) - self.enact_state_change(ConnectionState.FAILED, exception) + self.enact_state_change(ConnectionState.FAILED, ConnectionEvent.FAILED, exception) if self.__connected_future: self.__connected_future.set_exception(exception) self.__connected_future = None From e32d4bc68ed2a5f93c63fb7dd8ee31ad60331de8 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 11 Jan 2023 15:19:12 +0000 Subject: [PATCH 140/481] fix typos from rebase --- ably/realtime/connection.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 508f90aa..489682a3 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -24,7 +24,7 @@ class ConnectionState(str, Enum): SUSPENDED = 'suspended' -class ConnectionEvent(str): +class ConnectionEvent(str, Enum): INITIALIZED = 'initialized' CONNECTING = 'connecting' CONNECTED = 'connected' @@ -174,7 +174,7 @@ def enact_state_change(self, state, event, reason=None): if self.__state == ConnectionState.DISCONNECTED: if not self.__ttl_task or self.__ttl_task.done(): self.__ttl_task = asyncio.create_task(self.__start_suspended_timer()) - self._emit('connectionstate', ConnectionStateChange(current_state, state, reason)) + self._emit('connectionstate', ConnectionStateChange(current_state, state, event, reason)) async def __start_suspended_timer(self): if self.__connection_details: @@ -232,10 +232,6 @@ def on_connection_attempt_done(self, task): self.__connected_future.set_exception(exception) self.__connected_future = None self.enact_state_change(self.__fail_state, self.__fail_event, exception) - # if self.__in_suspended_state: - # self.enact_state_change(ConnectionState.SUSPENDED, ConnectionEvent.SUSPENDED, exception) - # else: - # self.enact_state_change(ConnectionState.DISCONNECTED, ConnectionEvent.DISCONNECTED, exception) self.__retry_task = asyncio.create_task(self.retry_connection_attempt()) async def retry_connection_attempt(self): From 5b5d8fd231abbb994ee8f2e53256bdf884c72f24 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 12 Sep 2022 08:34:29 +0100 Subject: [PATCH 141/481] add basic realtime auth --- ably/__init__.py | 1 + ably/realtime/__init__.py | 0 ably/realtime/realtime.py | 34 ++++++++++++++++++++++++++++++++++ test/ably/realtimeauthtest.py | 21 +++++++++++++++++++++ 4 files changed, 56 insertions(+) create mode 100644 ably/realtime/__init__.py create mode 100644 ably/realtime/realtime.py create mode 100644 test/ably/realtimeauthtest.py diff --git a/ably/__init__.py b/ably/__init__.py index 9782ea44..5e05eca1 100644 --- a/ably/__init__.py +++ b/ably/__init__.py @@ -1,4 +1,5 @@ from ably.rest.rest import AblyRest +from ably.realtime.realtime import AblyRealtime from ably.rest.auth import Auth from ably.rest.push import Push from ably.types.capability import Capability diff --git a/ably/realtime/__init__.py b/ably/realtime/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py new file mode 100644 index 00000000..d9baff7c --- /dev/null +++ b/ably/realtime/realtime.py @@ -0,0 +1,34 @@ +import logging +from ably.rest.auth import Auth +from ably.types.options import Options + + +log = logging.getLogger(__name__) + +class AblyRealtime: + """Ably Realtime Client""" + + def __init__(self, key=None, **kwargs): + """Create an AblyRealtime instance. + + :Parameters: + **Credentials** + - `key`: a valid ably key string + """ + + if key is not None: + options = Options(key=key, **kwargs) + else: + options = Options(**kwargs) + + self.__auth = Auth(self, options) + + self.__options = options + + @property + def auth(self): + return self.__auth + + @property + def options(self): + return self.__options diff --git a/test/ably/realtimeauthtest.py b/test/ably/realtimeauthtest.py new file mode 100644 index 00000000..2c759481 --- /dev/null +++ b/test/ably/realtimeauthtest.py @@ -0,0 +1,21 @@ +import pytest +from ably import Auth, AblyRealtime +from ably.util.exceptions import AblyAuthException +from test.ably.utils import BaseAsyncTestCase + + +class TestRealtimeAuth(BaseAsyncTestCase): + async def setUp(self): + self.invalid_key = "some key" + self.valid_key_format = "Vjhddw.owt:R97sjjbdERJdjwer" + + def test_auth_with_correct_key_format(self): + key = self.valid_key_format.split(":") + ably = AblyRealtime(self.valid_key_format) + assert Auth.Method.BASIC == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" + assert ably.auth.auth_options.key_name == key[0] + assert ably.auth.auth_options.key_secret == key[1] + + def test_auth_incorrect_key_format(self): + with pytest.raises(AblyAuthException): + ably = AblyRealtime(self.invalid_key) \ No newline at end of file From d2585ed70f1bc8d43d8050c9c12891358b9d6b98 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 13 Sep 2022 09:28:14 +0100 Subject: [PATCH 142/481] create connection --- ably/realtime/connection.py | 33 ++++++++++++++++++++ ably/realtime/realtime.py | 11 +++++-- poetry.lock | 60 ++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 4 files changed, 102 insertions(+), 3 deletions(-) create mode 100644 ably/realtime/connection.py diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py new file mode 100644 index 00000000..0fab035a --- /dev/null +++ b/ably/realtime/connection.py @@ -0,0 +1,33 @@ +import asyncio +import websockets +import json + + +class RealtimeConnection: + def __init__(self, realtime): + self.options = realtime.options + self.__ably = realtime + + async def connect(self): + self.connected_future = asyncio.Future() + asyncio.create_task(self.connect_impl()) + return await self.connected_future + + async def connect_impl(self): + async with websockets.connect(f'wss://realtime.ably.io?key={self.ably.key}') as websocket: + self.websocket = websocket + task = asyncio.create_task(self.ws_read_loop()) + await task + + async def ws_read_loop(self): + while True: + raw = await self.websocket.recv() + msg = json.loads(raw) + action = msg['action'] + if (action == 4): # CONNECTED + self.connected_future.set_result(msg) + return msg + + @property + def ably(self): + return self.__ably diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index d9baff7c..475a728e 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -1,4 +1,5 @@ import logging +from ably.realtime.connection import RealtimeConnection from ably.rest.auth import Auth from ably.types.options import Options @@ -8,7 +9,7 @@ class AblyRealtime: """Ably Realtime Client""" - def __init__(self, key=None, **kwargs): + def __init__(self, key=None, token=None, token_details=None, **kwargs): """Create an AblyRealtime instance. :Parameters: @@ -22,8 +23,9 @@ def __init__(self, key=None, **kwargs): options = Options(**kwargs) self.__auth = Auth(self, options) - self.__options = options + self.key = key + self.__connection = RealtimeConnection(self) @property def auth(self): @@ -32,3 +34,8 @@ def auth(self): @property def options(self): return self.__options + + @property + def connection(self): + """Returns the channels container object""" + return self.__connection diff --git a/poetry.lock b/poetry.lock index 6ba85565..68779fba 100644 --- a/poetry.lock +++ b/poetry.lock @@ -36,7 +36,7 @@ python-versions = ">=3.5" dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] -tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] +tests-no-zope = ["cloudpickle", "cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] [[package]] name = "certifi" @@ -487,6 +487,14 @@ category = "main" optional = false python-versions = ">=3.7" +[[package]] +name = "websockets" +version = "10.3" +description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" +category = "main" +optional = false +python-versions = ">=3.7" + [[package]] name = "zipp" version = "3.10.0" @@ -809,6 +817,56 @@ typing-extensions = [ {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, ] +websockets = [ + {file = "websockets-10.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:661f641b44ed315556a2fa630239adfd77bd1b11cb0b9d96ed8ad90b0b1e4978"}, + {file = "websockets-10.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b529fdfa881b69fe563dbd98acce84f3e5a67df13de415e143ef053ff006d500"}, + {file = "websockets-10.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f351c7d7d92f67c0609329ab2735eee0426a03022771b00102816a72715bb00b"}, + {file = "websockets-10.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:379e03422178436af4f3abe0aa8f401aa77ae2487843738542a75faf44a31f0c"}, + {file = "websockets-10.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e904c0381c014b914136c492c8fa711ca4cced4e9b3d110e5e7d436d0fc289e8"}, + {file = "websockets-10.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e7e6f2d6fd48422071cc8a6f8542016f350b79cc782752de531577d35e9bd677"}, + {file = "websockets-10.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b9c77f0d1436ea4b4dc089ed8335fa141e6a251a92f75f675056dac4ab47a71e"}, + {file = "websockets-10.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e6fa05a680e35d0fcc1470cb070b10e6fe247af54768f488ed93542e71339d6f"}, + {file = "websockets-10.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2f94fa3ae454a63ea3a19f73b95deeebc9f02ba2d5617ca16f0bbdae375cda47"}, + {file = "websockets-10.3-cp310-cp310-win32.whl", hash = "sha256:6ed1d6f791eabfd9808afea1e068f5e59418e55721db8b7f3bfc39dc831c42ae"}, + {file = "websockets-10.3-cp310-cp310-win_amd64.whl", hash = "sha256:347974105bbd4ea068106ec65e8e8ebd86f28c19e529d115d89bd8cc5cda3079"}, + {file = "websockets-10.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fab7c640815812ed5f10fbee7abbf58788d602046b7bb3af9b1ac753a6d5e916"}, + {file = "websockets-10.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:994cdb1942a7a4c2e10098d9162948c9e7b235df755de91ca33f6e0481366fdb"}, + {file = "websockets-10.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:aad5e300ab32036eb3fdc350ad30877210e2f51bceaca83fb7fef4d2b6c72b79"}, + {file = "websockets-10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e49ea4c1a9543d2bd8a747ff24411509c29e4bdcde05b5b0895e2120cb1a761d"}, + {file = "websockets-10.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6ea6b300a6bdd782e49922d690e11c3669828fe36fc2471408c58b93b5535a98"}, + {file = "websockets-10.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:ef5ce841e102278c1c2e98f043db99d6755b1c58bde475516aef3a008ed7f28e"}, + {file = "websockets-10.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d1655a6fc7aecd333b079d00fb3c8132d18988e47f19740c69303bf02e9883c6"}, + {file = "websockets-10.3-cp37-cp37m-win32.whl", hash = "sha256:83e5ca0d5b743cde3d29fda74ccab37bdd0911f25bd4cdf09ff8b51b7b4f2fa1"}, + {file = "websockets-10.3-cp37-cp37m-win_amd64.whl", hash = "sha256:da4377904a3379f0c1b75a965fff23b28315bcd516d27f99a803720dfebd94d4"}, + {file = "websockets-10.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a1e15b230c3613e8ea82c9fc6941b2093e8eb939dd794c02754d33980ba81e36"}, + {file = "websockets-10.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:31564a67c3e4005f27815634343df688b25705cccb22bc1db621c781ddc64c69"}, + {file = "websockets-10.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c8d1d14aa0f600b5be363077b621b1b4d1eb3fbf90af83f9281cda668e6ff7fd"}, + {file = "websockets-10.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8fbd7d77f8aba46d43245e86dd91a8970eac4fb74c473f8e30e9c07581f852b2"}, + {file = "websockets-10.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:210aad7fdd381c52e58777560860c7e6110b6174488ef1d4b681c08b68bf7f8c"}, + {file = "websockets-10.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6075fd24df23133c1b078e08a9b04a3bc40b31a8def4ee0b9f2c8865acce913e"}, + {file = "websockets-10.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7f6d96fdb0975044fdd7953b35d003b03f9e2bcf85f2d2cf86285ece53e9f991"}, + {file = "websockets-10.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c7250848ce69559756ad0086a37b82c986cd33c2d344ab87fea596c5ac6d9442"}, + {file = "websockets-10.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:28dd20b938a57c3124028680dc1600c197294da5db4292c76a0b48efb3ed7f76"}, + {file = "websockets-10.3-cp38-cp38-win32.whl", hash = "sha256:54c000abeaff6d8771a4e2cef40900919908ea7b6b6a30eae72752607c6db559"}, + {file = "websockets-10.3-cp38-cp38-win_amd64.whl", hash = "sha256:7ab36e17af592eec5747c68ef2722a74c1a4a70f3772bc661079baf4ae30e40d"}, + {file = "websockets-10.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a141de3d5a92188234afa61653ed0bbd2dde46ad47b15c3042ffb89548e77094"}, + {file = "websockets-10.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:97bc9d41e69a7521a358f9b8e44871f6cdeb42af31815c17aed36372d4eec667"}, + {file = "websockets-10.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d6353ba89cfc657a3f5beabb3b69be226adbb5c6c7a66398e17809b0ce3c4731"}, + {file = "websockets-10.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec2b0ab7edc8cd4b0eb428b38ed89079bdc20c6bdb5f889d353011038caac2f9"}, + {file = "websockets-10.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:85506b3328a9e083cc0a0fb3ba27e33c8db78341b3eb12eb72e8afd166c36680"}, + {file = "websockets-10.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8af75085b4bc0b5c40c4a3c0e113fa95e84c60f4ed6786cbb675aeb1ee128247"}, + {file = "websockets-10.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:07cdc0a5b2549bcfbadb585ad8471ebdc7bdf91e32e34ae3889001c1c106a6af"}, + {file = "websockets-10.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:5b936bf552e4f6357f5727579072ff1e1324717902127ffe60c92d29b67b7be3"}, + {file = "websockets-10.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e4e08305bfd76ba8edab08dcc6496f40674f44eb9d5e23153efa0a35750337e8"}, + {file = "websockets-10.3-cp39-cp39-win32.whl", hash = "sha256:bb621ec2dbbbe8df78a27dbd9dd7919f9b7d32a73fafcb4d9252fc4637343582"}, + {file = "websockets-10.3-cp39-cp39-win_amd64.whl", hash = "sha256:51695d3b199cd03098ae5b42833006a0f43dc5418d3102972addc593a783bc02"}, + {file = "websockets-10.3-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:907e8247480f287aa9bbc9391bd6de23c906d48af54c8c421df84655eef66af7"}, + {file = "websockets-10.3-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b1359aba0ff810d5830d5ab8e2c4a02bebf98a60aa0124fb29aa78cfdb8031f"}, + {file = "websockets-10.3-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:93d5ea0b5da8d66d868b32c614d2b52d14304444e39e13a59566d4acb8d6e2e4"}, + {file = "websockets-10.3-pp37-pypy37_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7934e055fd5cd9dee60f11d16c8d79c4567315824bacb1246d0208a47eca9755"}, + {file = "websockets-10.3-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:3eda1cb7e9da1b22588cefff09f0951771d6ee9fa8dbe66f5ae04cc5f26b2b55"}, + {file = "websockets-10.3.tar.gz", hash = "sha256:fc06cc8073c8e87072138ba1e431300e2d408f054b27047d047b549455066ff4"}, +] zipp = [ {file = "zipp-3.10.0-py3-none-any.whl", hash = "sha256:4fcb6f278987a6605757302a6e40e896257570d11c51628968ccb2a47e80c6c1"}, {file = "zipp-3.10.0.tar.gz", hash = "sha256:7a7262fd930bd3e36c50b9a64897aec3fafff3dfdeec9623ae22b40e93f99bb8"}, diff --git a/pyproject.toml b/pyproject.toml index fc106f9e..a3dd5f37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ h2 = "^4.0.0" # Optional dependencies pycrypto = { version = "^2.6.1", optional = true } pycryptodome = { version = "*", optional = true } +websockets = "^10.3" [tool.poetry.extras] oldcrypto = ["pycrypto"] From 3fd7e7d6f06929556671bb92471386838ab715a3 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 13 Sep 2022 15:27:26 +0100 Subject: [PATCH 143/481] update connection --- ably/realtime/connection.py | 9 +++++++-- ably/realtime/realtime.py | 4 +++- test/ably/realtimeauthtest.py | 29 ++++++++++++++++++++++++----- test/ably/restsetup.py | 2 ++ 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 0fab035a..c870bd15 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -1,6 +1,7 @@ import asyncio import websockets import json +from ably.util.exceptions import AblyAuthException class RealtimeConnection: @@ -13,8 +14,9 @@ async def connect(self): asyncio.create_task(self.connect_impl()) return await self.connected_future + async def connect_impl(self): - async with websockets.connect(f'wss://realtime.ably.io?key={self.ably.key}') as websocket: + async with websockets.connect(f'{self.options.realtime_host}?key={self.ably.key}') as websocket: self.websocket = websocket task = asyncio.create_task(self.ws_read_loop()) await task @@ -26,7 +28,10 @@ async def ws_read_loop(self): action = msg['action'] if (action == 4): # CONNECTED self.connected_future.set_result(msg) - return msg + if (action == 9): # ERROR + error = msg["error"] + self.connected_future.set_exception(AblyAuthException(error["message"], error["statusCode"], error["code"])) + @property def ably(self): diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 475a728e..059a43ec 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -1,4 +1,5 @@ import logging +import os from ably.realtime.connection import RealtimeConnection from ably.rest.auth import Auth from ably.types.options import Options @@ -9,7 +10,7 @@ class AblyRealtime: """Ably Realtime Client""" - def __init__(self, key=None, token=None, token_details=None, **kwargs): + def __init__(self, key=None, **kwargs): """Create an AblyRealtime instance. :Parameters: @@ -22,6 +23,7 @@ def __init__(self, key=None, token=None, token_details=None, **kwargs): else: options = Options(**kwargs) + options.realtime_host = os.environ.get('ABLY_REALTIME_HOST') self.__auth = Auth(self, options) self.__options = options self.key = key diff --git a/test/ably/realtimeauthtest.py b/test/ably/realtimeauthtest.py index 2c759481..f696569b 100644 --- a/test/ably/realtimeauthtest.py +++ b/test/ably/realtimeauthtest.py @@ -1,21 +1,40 @@ import pytest from ably import Auth, AblyRealtime from ably.util.exceptions import AblyAuthException +from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase class TestRealtimeAuth(BaseAsyncTestCase): async def setUp(self): - self.invalid_key = "some key" - self.valid_key_format = "Vjhddw.owt:R97sjjbdERJdjwer" + self.test_vars = await RestSetup.get_test_vars() + self.valid_key_format = "Vjdw.owt:R97sjjjwer" - def test_auth_with_correct_key_format(self): + async def test_auth_with_valid_key(self): + ably = AblyRealtime(self.test_vars["keys"][0]["key_str"]) + assert Auth.Method.BASIC == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" + assert ably.auth.auth_options.key_name == self.test_vars["keys"][0]['key_name'] + assert ably.auth.auth_options.key_secret == self.test_vars["keys"][0]['key_secret'] + + async def test_auth_incorrect_key(self): + with pytest.raises(AblyAuthException): + AblyRealtime("some invalid key") + + async def test_auth_with_valid_key_format(self): key = self.valid_key_format.split(":") ably = AblyRealtime(self.valid_key_format) assert Auth.Method.BASIC == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" assert ably.auth.auth_options.key_name == key[0] assert ably.auth.auth_options.key_secret == key[1] - def test_auth_incorrect_key_format(self): + # async def test_auth_connection(self): + # ably = AblyRealtime(self.test_vars["keys"][0]["key_str"]) + # conn = await ably.connection.connect() + # assert conn["action"] == 4 + # assert "connectionDetails" in conn + + async def test_auth_invalid_key(self): + ably = AblyRealtime(self.valid_key_format) with pytest.raises(AblyAuthException): - ably = AblyRealtime(self.invalid_key) \ No newline at end of file + await ably.connection.connect() + diff --git a/test/ably/restsetup.py b/test/ably/restsetup.py index 3c681005..9babdd05 100644 --- a/test/ably/restsetup.py +++ b/test/ably/restsetup.py @@ -14,6 +14,7 @@ tls = (os.environ.get('ABLY_TLS') or "true").lower() == "true" host = os.environ.get('ABLY_HOST', 'sandbox-rest.ably.io') +realtime_host = os.environ.get('ABLY_REALTIME_HOST', 'sandbox-realtime.ably.io') environment = os.environ.get('ABLY_ENV') port = 80 @@ -51,6 +52,7 @@ async def get_test_vars(sender=None): "tls_port": tls_port, "tls": tls, "environment": environment, + "realtime_host": realtime_host, "keys": [{ "key_name": "%s.%s" % (app_id, k.get("id", "")), "key_secret": k.get("value", ""), From f74d85eb6ed6f7a63896259bfb7cdc31df1e1dec Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 16:26:57 +0100 Subject: [PATCH 144/481] Add get_ably_realtime test helper --- test/ably/restsetup.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/test/ably/restsetup.py b/test/ably/restsetup.py index 9babdd05..efab592d 100644 --- a/test/ably/restsetup.py +++ b/test/ably/restsetup.py @@ -6,6 +6,7 @@ from ably.types.capability import Capability from ably.types.options import Options from ably.util.exceptions import AblyException +from ably.realtime.realtime import AblyRealtime log = logging.getLogger(__name__) @@ -14,7 +15,7 @@ tls = (os.environ.get('ABLY_TLS') or "true").lower() == "true" host = os.environ.get('ABLY_HOST', 'sandbox-rest.ably.io') -realtime_host = os.environ.get('ABLY_REALTIME_HOST', 'sandbox-realtime.ably.io') +realtime_host = 'sandbox-realtime.ably.io' environment = os.environ.get('ABLY_ENV') port = 80 @@ -81,6 +82,20 @@ async def get_ably_rest(cls, **kw): options.update(kw) return AblyRest(**options) + @classmethod + async def get_ably_realtime(cls, **kw): + test_vars = await RestSetup.get_test_vars() + options = { + 'key': test_vars["keys"][0]["key_str"], + 'realtime_host': realtime_host, + 'port': test_vars["port"], + 'tls_port': test_vars["tls_port"], + 'tls': test_vars["tls"], + 'environment': test_vars["environment"], + } + options.update(kw) + return AblyRealtime(**options) + @classmethod async def clear_test_vars(cls): test_vars = RestSetup.__test_vars From f8891c10d26f0f6c22af5bc466cae5b3dd439971 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 16:27:21 +0100 Subject: [PATCH 145/481] Use configured realtime_host for websocket connections --- ably/realtime/connection.py | 3 +-- ably/realtime/realtime.py | 2 -- ably/types/options.py | 3 +++ 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index c870bd15..bedfda18 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -14,9 +14,8 @@ async def connect(self): asyncio.create_task(self.connect_impl()) return await self.connected_future - async def connect_impl(self): - async with websockets.connect(f'{self.options.realtime_host}?key={self.ably.key}') as websocket: + async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}') as websocket: self.websocket = websocket task = asyncio.create_task(self.ws_read_loop()) await task diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 059a43ec..9f44f7ff 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -1,5 +1,4 @@ import logging -import os from ably.realtime.connection import RealtimeConnection from ably.rest.auth import Auth from ably.types.options import Options @@ -23,7 +22,6 @@ def __init__(self, key=None, **kwargs): else: options = Options(**kwargs) - options.realtime_host = os.environ.get('ABLY_REALTIME_HOST') self.__auth = Auth(self, options) self.__options = options self.key = key diff --git a/ably/types/options.py b/ably/types/options.py index 38ef8ed9..441d87b6 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -27,6 +27,9 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, from ably import api_version idempotent_rest_publishing = api_version >= '1.2' + if realtime_host is None: + realtime_host = Defaults.realtime_host + self.__client_id = client_id self.__log_level = log_level self.__tls = tls From 3a3a3294ce3098713575c8e4ad15fc0b3e000035 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 16:27:33 +0100 Subject: [PATCH 146/481] Update tests to use realtime helper method --- test/ably/realtimeauthtest.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/ably/realtimeauthtest.py b/test/ably/realtimeauthtest.py index f696569b..626eb12d 100644 --- a/test/ably/realtimeauthtest.py +++ b/test/ably/realtimeauthtest.py @@ -8,21 +8,21 @@ class TestRealtimeAuth(BaseAsyncTestCase): async def setUp(self): self.test_vars = await RestSetup.get_test_vars() - self.valid_key_format = "Vjdw.owt:R97sjjjwer" + self.valid_key_format = "api:key" async def test_auth_with_valid_key(self): - ably = AblyRealtime(self.test_vars["keys"][0]["key_str"]) + ably = await RestSetup.get_ably_realtime(key=self.test_vars["keys"][0]["key_str"]) assert Auth.Method.BASIC == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" assert ably.auth.auth_options.key_name == self.test_vars["keys"][0]['key_name'] assert ably.auth.auth_options.key_secret == self.test_vars["keys"][0]['key_secret'] async def test_auth_incorrect_key(self): with pytest.raises(AblyAuthException): - AblyRealtime("some invalid key") + await RestSetup.get_ably_realtime(key="some invalid key") async def test_auth_with_valid_key_format(self): key = self.valid_key_format.split(":") - ably = AblyRealtime(self.valid_key_format) + ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) assert Auth.Method.BASIC == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" assert ably.auth.auth_options.key_name == key[0] assert ably.auth.auth_options.key_secret == key[1] @@ -34,7 +34,6 @@ async def test_auth_with_valid_key_format(self): # assert "connectionDetails" in conn async def test_auth_invalid_key(self): - ably = AblyRealtime(self.valid_key_format) + ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) with pytest.raises(AblyAuthException): await ably.connection.connect() - From a8dc53d8b26186f0ef4dd8d7a6098acec4976ce7 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 16:29:37 +0100 Subject: [PATCH 147/481] Add Realtime.connect method --- ably/realtime/realtime.py | 8 ++++++-- test/ably/realtimeauthtest.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 9f44f7ff..71dc5b38 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -6,6 +6,7 @@ log = logging.getLogger(__name__) + class AblyRealtime: """Ably Realtime Client""" @@ -26,7 +27,10 @@ def __init__(self, key=None, **kwargs): self.__options = options self.key = key self.__connection = RealtimeConnection(self) - + + async def connect(self): + await self.connection.connect() + @property def auth(self): return self.__auth @@ -34,7 +38,7 @@ def auth(self): @property def options(self): return self.__options - + @property def connection(self): """Returns the channels container object""" diff --git a/test/ably/realtimeauthtest.py b/test/ably/realtimeauthtest.py index 626eb12d..e84b5703 100644 --- a/test/ably/realtimeauthtest.py +++ b/test/ably/realtimeauthtest.py @@ -36,4 +36,4 @@ async def test_auth_with_valid_key_format(self): async def test_auth_invalid_key(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) with pytest.raises(AblyAuthException): - await ably.connection.connect() + await ably.connect() From 5153d7c3fee5e5b92c9e0c387af849917acce45a Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 16:36:01 +0100 Subject: [PATCH 148/481] Make Realtime.connect return None when successful --- ably/realtime/connection.py | 4 ++-- test/ably/realtimeauthtest.py | 8 +++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index bedfda18..bf93dfbf 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -12,7 +12,7 @@ def __init__(self, realtime): async def connect(self): self.connected_future = asyncio.Future() asyncio.create_task(self.connect_impl()) - return await self.connected_future + await self.connected_future async def connect_impl(self): async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}') as websocket: @@ -26,7 +26,7 @@ async def ws_read_loop(self): msg = json.loads(raw) action = msg['action'] if (action == 4): # CONNECTED - self.connected_future.set_result(msg) + self.connected_future.set_result(None) if (action == 9): # ERROR error = msg["error"] self.connected_future.set_exception(AblyAuthException(error["message"], error["statusCode"], error["code"])) diff --git a/test/ably/realtimeauthtest.py b/test/ably/realtimeauthtest.py index e84b5703..7297c019 100644 --- a/test/ably/realtimeauthtest.py +++ b/test/ably/realtimeauthtest.py @@ -27,11 +27,9 @@ async def test_auth_with_valid_key_format(self): assert ably.auth.auth_options.key_name == key[0] assert ably.auth.auth_options.key_secret == key[1] - # async def test_auth_connection(self): - # ably = AblyRealtime(self.test_vars["keys"][0]["key_str"]) - # conn = await ably.connection.connect() - # assert conn["action"] == 4 - # assert "connectionDetails" in conn + async def test_auth_connection(self): + ably = await RestSetup.get_ably_realtime() + await ably.connect() async def test_auth_invalid_key(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) From 98ead24cfbccb256d8daa1a731fca00d818f86c4 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 16:41:22 +0100 Subject: [PATCH 149/481] Add Connection.state --- ably/realtime/connection.py | 15 ++++++++++++++- test/ably/realtimeauthtest.py | 3 +++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index bf93dfbf..18485afe 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -2,17 +2,27 @@ import websockets import json from ably.util.exceptions import AblyAuthException +from enum import Enum + + +class ConnectionState(Enum): + INITIALIZED = 'initialized' + CONNECTING = 'connecting' + CONNECTED = 'connected' class RealtimeConnection: def __init__(self, realtime): self.options = realtime.options self.__ably = realtime + self.__state = ConnectionState.INITIALIZED async def connect(self): + self.__state = ConnectionState.CONNECTING self.connected_future = asyncio.Future() asyncio.create_task(self.connect_impl()) await self.connected_future + self.__state = ConnectionState.CONNECTED async def connect_impl(self): async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}') as websocket: @@ -30,8 +40,11 @@ async def ws_read_loop(self): if (action == 9): # ERROR error = msg["error"] self.connected_future.set_exception(AblyAuthException(error["message"], error["statusCode"], error["code"])) - @property def ably(self): return self.__ably + + @property + def state(self): + return self.__state diff --git a/test/ably/realtimeauthtest.py b/test/ably/realtimeauthtest.py index 7297c019..bda9a530 100644 --- a/test/ably/realtimeauthtest.py +++ b/test/ably/realtimeauthtest.py @@ -1,3 +1,4 @@ +from ably.realtime.connection import ConnectionState import pytest from ably import Auth, AblyRealtime from ably.util.exceptions import AblyAuthException @@ -29,7 +30,9 @@ async def test_auth_with_valid_key_format(self): async def test_auth_connection(self): ably = await RestSetup.get_ably_realtime() + assert ably.connection.state == ConnectionState.INITIALIZED await ably.connect() + assert ably.connection.state == ConnectionState.CONNECTED async def test_auth_invalid_key(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) From 5552762ebf136776fc8bf16b81493c6622f83fc1 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 16:48:39 +0100 Subject: [PATCH 150/481] Add Realtime.close method --- ably/realtime/connection.py | 7 +++++++ ably/realtime/realtime.py | 3 +++ test/ably/realtimeauthtest.py | 3 +++ 3 files changed, 13 insertions(+) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 18485afe..baa24922 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -9,6 +9,8 @@ class ConnectionState(Enum): INITIALIZED = 'initialized' CONNECTING = 'connecting' CONNECTED = 'connected' + CLOSING = 'closing' + CLOSED = 'closed' class RealtimeConnection: @@ -24,6 +26,11 @@ async def connect(self): await self.connected_future self.__state = ConnectionState.CONNECTED + async def close(self): + self.__state = ConnectionState.CLOSING + await self.websocket.close() + self.__state = ConnectionState.CLOSED + async def connect_impl(self): async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}') as websocket: self.websocket = websocket diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 71dc5b38..25f57a2a 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -31,6 +31,9 @@ def __init__(self, key=None, **kwargs): async def connect(self): await self.connection.connect() + async def close(self): + await self.connection.close() + @property def auth(self): return self.__auth diff --git a/test/ably/realtimeauthtest.py b/test/ably/realtimeauthtest.py index bda9a530..e9110b0d 100644 --- a/test/ably/realtimeauthtest.py +++ b/test/ably/realtimeauthtest.py @@ -33,8 +33,11 @@ async def test_auth_connection(self): assert ably.connection.state == ConnectionState.INITIALIZED await ably.connect() assert ably.connection.state == ConnectionState.CONNECTED + await ably.close() + assert ably.connection.state == ConnectionState.CLOSED async def test_auth_invalid_key(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) with pytest.raises(AblyAuthException): await ably.connect() + await ably.close() From 91dcea2a5e9dffcf2c33dd9606dd30eae4b037b3 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 16:49:37 +0100 Subject: [PATCH 151/481] Move realimteauthtest.py to realtimeinit_test.py --- test/ably/{realtimeauthtest.py => realtimeinit_test.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/ably/{realtimeauthtest.py => realtimeinit_test.py} (100%) diff --git a/test/ably/realtimeauthtest.py b/test/ably/realtimeinit_test.py similarity index 100% rename from test/ably/realtimeauthtest.py rename to test/ably/realtimeinit_test.py From 17a27efbdcff3b3cc654b644587cbf08f2d8379c Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 16:52:00 +0100 Subject: [PATCH 152/481] Move connection tests to new file --- test/ably/realtimeconnection_test.py | 25 +++++++++++++++++++++++++ test/ably/realtimeinit_test.py | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 test/ably/realtimeconnection_test.py diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py new file mode 100644 index 00000000..00e32759 --- /dev/null +++ b/test/ably/realtimeconnection_test.py @@ -0,0 +1,25 @@ +from ably.realtime.connection import ConnectionState +import pytest +from ably.util.exceptions import AblyAuthException +from test.ably.restsetup import RestSetup +from test.ably.utils import BaseAsyncTestCase + + +class TestRealtimeAuth(BaseAsyncTestCase): + async def setUp(self): + self.test_vars = await RestSetup.get_test_vars() + self.valid_key_format = "api:key" + + async def test_auth_connection(self): + ably = await RestSetup.get_ably_realtime() + assert ably.connection.state == ConnectionState.INITIALIZED + await ably.connect() + assert ably.connection.state == ConnectionState.CONNECTED + await ably.close() + assert ably.connection.state == ConnectionState.CLOSED + + async def test_auth_invalid_key(self): + ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) + with pytest.raises(AblyAuthException): + await ably.connect() + await ably.close() diff --git a/test/ably/realtimeinit_test.py b/test/ably/realtimeinit_test.py index e9110b0d..a85f9576 100644 --- a/test/ably/realtimeinit_test.py +++ b/test/ably/realtimeinit_test.py @@ -1,6 +1,6 @@ from ably.realtime.connection import ConnectionState import pytest -from ably import Auth, AblyRealtime +from ably import Auth from ably.util.exceptions import AblyAuthException from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase From f1b14397212f9fcf34731f6e57a3ab634fb61ad0 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 17:01:41 +0100 Subject: [PATCH 153/481] Ensure connected_future is resolved once --- ably/realtime/connection.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index baa24922..3e4562f6 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -1,9 +1,12 @@ +import logging import asyncio import websockets import json from ably.util.exceptions import AblyAuthException from enum import Enum +log = logging.getLogger(__name__) + class ConnectionState(Enum): INITIALIZED = 'initialized' @@ -18,6 +21,8 @@ def __init__(self, realtime): self.options = realtime.options self.__ably = realtime self.__state = ConnectionState.INITIALIZED + self.connected_future = None + self.websocket = None async def connect(self): self.__state = ConnectionState.CONNECTING @@ -42,11 +47,18 @@ async def ws_read_loop(self): raw = await self.websocket.recv() msg = json.loads(raw) action = msg['action'] - if (action == 4): # CONNECTED - self.connected_future.set_result(None) - if (action == 9): # ERROR + if action == 4: # CONNECTED + if self.connected_future: + self.connected_future.set_result(None) + self.connected_future = None + else: + log.warn('CONNECTED message receieved but connected_future not set') + if action == 9: # ERROR error = msg["error"] - self.connected_future.set_exception(AblyAuthException(error["message"], error["statusCode"], error["code"])) + if error['nonfatal'] is False: + if self.connected_future: + self.connected_future.set_exception(AblyAuthException(error["message"], error["statusCode"], error["code"])) + self.connected_future = None @property def ably(self): From 6c7cbe59e186cb615cfb47f410511bbc9c4f9680 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 17:08:40 +0100 Subject: [PATCH 154/481] Add some state validation to Connection methods --- ably/realtime/connection.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 3e4562f6..bd3b21df 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -25,15 +25,27 @@ def __init__(self, realtime): self.websocket = None async def connect(self): - self.__state = ConnectionState.CONNECTING - self.connected_future = asyncio.Future() - asyncio.create_task(self.connect_impl()) - await self.connected_future - self.__state = ConnectionState.CONNECTED + if self.__state == ConnectionState.CONNECTED: + return + + if self.__state == ConnectionState.CONNECTING: + if self.connected_future is None: + log.fatal('Connection state is CONNECTING but connected_future does not exits') + return + await self.connected_future + else: + self.__state = ConnectionState.CONNECTING + self.connected_future = asyncio.Future() + asyncio.create_task(self.connect_impl()) + await self.connected_future + self.__state = ConnectionState.CONNECTED async def close(self): self.__state = ConnectionState.CLOSING - await self.websocket.close() + if self.websocket: + await self.websocket.close() + else: + log.warn('Connection.closed called while connection already closed') self.__state = ConnectionState.CLOSED async def connect_impl(self): From 92cc1aef6b83a0506612f49a4075bb52c5fbb3a5 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 13 Sep 2022 17:14:34 +0100 Subject: [PATCH 155/481] Add tests for transient connection states --- test/ably/realtimeconnection_test.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 00e32759..134c1f9d 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -1,3 +1,4 @@ +import asyncio from ably.realtime.connection import ConnectionState import pytest from ably.util.exceptions import AblyAuthException @@ -18,6 +19,22 @@ async def test_auth_connection(self): await ably.close() assert ably.connection.state == ConnectionState.CLOSED + async def test_connecting_state(self): + ably = await RestSetup.get_ably_realtime() + task = asyncio.create_task(ably.connect()) + await asyncio.sleep(0) + assert ably.connection.state == ConnectionState.CONNECTING + await task + await ably.close() + + async def test_closing_state(self): + ably = await RestSetup.get_ably_realtime() + await ably.connect() + task = asyncio.create_task(ably.close()) + await asyncio.sleep(0) + assert ably.connection.state == ConnectionState.CLOSING + await task + async def test_auth_invalid_key(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) with pytest.raises(AblyAuthException): From 954efacfb16d936c009bfdc6b22c1566743ed2ae Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 14 Sep 2022 08:29:34 +0100 Subject: [PATCH 156/481] add api key check --- ably/realtime/connection.py | 3 ++- ably/realtime/realtime.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index bd3b21df..a9e24341 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -69,7 +69,8 @@ async def ws_read_loop(self): error = msg["error"] if error['nonfatal'] is False: if self.connected_future: - self.connected_future.set_exception(AblyAuthException(error["message"], error["statusCode"], error["code"])) + self.connected_future.set_exception( + AblyAuthException(error["message"], error["statusCode"], error["code"])) self.connected_future = None @property diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 25f57a2a..36cf1cbe 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -21,7 +21,7 @@ def __init__(self, key=None, **kwargs): if key is not None: options = Options(key=key, **kwargs) else: - options = Options(**kwargs) + raise ValueError("Key is missing. Provide an API key") self.__auth = Auth(self, options) self.__options = options @@ -44,5 +44,5 @@ def options(self): @property def connection(self): - """Returns the channels container object""" + """Establish realtime connection""" return self.__connection From f7b24930a43b0875c0c5bcc7d13d9e775e08d619 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 14 Sep 2022 14:15:28 +0100 Subject: [PATCH 157/481] Change some connection fields to private --- ably/realtime/connection.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index a9e24341..3a154bae 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -21,57 +21,57 @@ def __init__(self, realtime): self.options = realtime.options self.__ably = realtime self.__state = ConnectionState.INITIALIZED - self.connected_future = None - self.websocket = None + self.__connected_future = None + self.__websocket = None async def connect(self): if self.__state == ConnectionState.CONNECTED: return if self.__state == ConnectionState.CONNECTING: - if self.connected_future is None: + if self.__connected_future is None: log.fatal('Connection state is CONNECTING but connected_future does not exits') return - await self.connected_future + await self.__connected_future else: self.__state = ConnectionState.CONNECTING - self.connected_future = asyncio.Future() + self.__connected_future = asyncio.Future() asyncio.create_task(self.connect_impl()) - await self.connected_future + await self.__connected_future self.__state = ConnectionState.CONNECTED async def close(self): self.__state = ConnectionState.CLOSING - if self.websocket: - await self.websocket.close() + if self.__websocket: + await self.__websocket.close() else: log.warn('Connection.closed called while connection already closed') self.__state = ConnectionState.CLOSED async def connect_impl(self): async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}') as websocket: - self.websocket = websocket + self.__websocket = websocket task = asyncio.create_task(self.ws_read_loop()) await task async def ws_read_loop(self): while True: - raw = await self.websocket.recv() + raw = await self.__websocket.recv() msg = json.loads(raw) action = msg['action'] if action == 4: # CONNECTED - if self.connected_future: - self.connected_future.set_result(None) - self.connected_future = None + if self.__connected_future: + self.__connected_future.set_result(None) + self.__connected_future = None else: log.warn('CONNECTED message receieved but connected_future not set') if action == 9: # ERROR error = msg["error"] if error['nonfatal'] is False: - if self.connected_future: - self.connected_future.set_exception( + if self.__connected_future: + self.__connected_future.set_exception( AblyAuthException(error["message"], error["statusCode"], error["code"])) - self.connected_future = None + self.__connected_future = None @property def ably(self): From ae6e19b9a68ae9338f064480244720f4893257c6 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 14 Sep 2022 14:18:02 +0100 Subject: [PATCH 158/481] Add failed ConnectionState --- ably/realtime/connection.py | 2 ++ test/ably/realtimeconnection_test.py | 1 + 2 files changed, 3 insertions(+) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 3a154bae..d5c79f0d 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -14,6 +14,7 @@ class ConnectionState(Enum): CONNECTED = 'connected' CLOSING = 'closing' CLOSED = 'closed' + FAILED = 'failed' class RealtimeConnection: @@ -68,6 +69,7 @@ async def ws_read_loop(self): if action == 9: # ERROR error = msg["error"] if error['nonfatal'] is False: + self.__state = ConnectionState.FAILED if self.__connected_future: self.__connected_future.set_exception( AblyAuthException(error["message"], error["statusCode"], error["code"])) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 134c1f9d..929161a7 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -39,4 +39,5 @@ async def test_auth_invalid_key(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) with pytest.raises(AblyAuthException): await ably.connect() + assert ably.connection.state == ConnectionState.FAILED await ably.close() From 6ae0ad4225c61813078e87dde9695ef00bee6d76 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 14 Sep 2022 15:32:22 +0100 Subject: [PATCH 159/481] change base type of ProtocolMessageAction to IntEnum fix hanging test --- ably/realtime/connection.py | 13 +++++++++---- ably/realtime/realtime.py | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index d5c79f0d..6cf3490f 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -3,7 +3,7 @@ import websockets import json from ably.util.exceptions import AblyAuthException -from enum import Enum +from enum import Enum, IntEnum log = logging.getLogger(__name__) @@ -17,6 +17,11 @@ class ConnectionState(Enum): FAILED = 'failed' +class ProtocolMessageAction(IntEnum): + CONNECTED = 4 + ERROR = 9 + + class RealtimeConnection: def __init__(self, realtime): self.options = realtime.options @@ -60,13 +65,13 @@ async def ws_read_loop(self): raw = await self.__websocket.recv() msg = json.loads(raw) action = msg['action'] - if action == 4: # CONNECTED + if action == ProtocolMessageAction.CONNECTED: # CONNECTED if self.__connected_future: self.__connected_future.set_result(None) self.__connected_future = None else: - log.warn('CONNECTED message receieved but connected_future not set') - if action == 9: # ERROR + log.warn('CONNECTED message received but connected_future not set') + if action == ProtocolMessageAction.ERROR: # ERROR error = msg["error"] if error['nonfatal'] is False: self.__state = ConnectionState.FAILED diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 36cf1cbe..de70e41c 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -21,7 +21,7 @@ def __init__(self, key=None, **kwargs): if key is not None: options = Options(key=key, **kwargs) else: - raise ValueError("Key is missing. Provide an API key") + raise ValueError("Key is missing. Provide an API key.") self.__auth = Auth(self, options) self.__options = options From ab5d9b6935f1cea63d036d74ddec183a31989467 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Thu, 15 Sep 2022 16:25:34 +0100 Subject: [PATCH 160/481] send ably-agent header in realtime connection fix linting error --- ably/realtime/connection.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 6cf3490f..0ec73e67 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -2,6 +2,7 @@ import asyncio import websockets import json +from ably.http.httputils import HttpUtils from ably.util.exceptions import AblyAuthException from enum import Enum, IntEnum @@ -55,7 +56,9 @@ async def close(self): self.__state = ConnectionState.CLOSED async def connect_impl(self): - async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}') as websocket: + headers = HttpUtils.default_get_headers() + async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}', + extra_headers=headers) as websocket: self.__websocket = websocket task = asyncio.create_task(self.ws_read_loop()) await task From 94dbe74857787425b5f9a27037ec8b074002e09d Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Thu, 15 Sep 2022 18:11:39 +0100 Subject: [PATCH 161/481] refactor default header --- ably/http/httputils.py | 12 ++++++++---- ably/realtime/connection.py | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ably/http/httputils.py b/ably/http/httputils.py index 0517f969..53a583a1 100644 --- a/ably/http/httputils.py +++ b/ably/http/httputils.py @@ -15,10 +15,7 @@ class HttpUtils: @staticmethod def default_get_headers(binary=False): - headers = { - "X-Ably-Version": ably.api_version, - "Ably-Agent": 'ably-python/%s python/%s' % (ably.lib_version, platform.python_version()) - } + headers = HttpUtils.default_headers() if binary: headers["Accept"] = HttpUtils.mime_types['binary'] else: @@ -36,3 +33,10 @@ def get_host_header(host): return { 'Host': host, } + + @staticmethod + def default_headers(): + return { + "X-Ably-Version": ably.api_version, + "Ably-Agent": 'ably-python/%s python/%s' % (ably.lib_version, platform.python_version()) + } diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 0ec73e67..0e5cabb8 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -56,7 +56,7 @@ async def close(self): self.__state = ConnectionState.CLOSED async def connect_impl(self): - headers = HttpUtils.default_get_headers() + headers = HttpUtils.default_headers() async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}', extra_headers=headers) as websocket: self.__websocket = websocket From 0e51dd807779a33362992ba524881b12bf6697cf Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Thu, 22 Sep 2022 10:56:35 +0100 Subject: [PATCH 162/481] send close protocol message to ably --- ably/realtime/connection.py | 21 ++++++++++++++++++--- test/ably/realtimeconnection_test.py | 4 ++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 0e5cabb8..8af853c1 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -21,6 +21,8 @@ class ConnectionState(Enum): class ProtocolMessageAction(IntEnum): CONNECTED = 4 ERROR = 9 + CLOSE = 7 + CLOSED = 8 class RealtimeConnection: @@ -29,6 +31,7 @@ def __init__(self, realtime): self.__ably = realtime self.__state = ConnectionState.INITIALIZED self.__connected_future = None + self.__closed_future = None self.__websocket = None async def connect(self): @@ -49,12 +52,20 @@ async def connect(self): async def close(self): self.__state = ConnectionState.CLOSING - if self.__websocket: - await self.__websocket.close() + self.__closed_future = asyncio.Future() + if self.__websocket and self.__state == ConnectionState.CONNECTED: + task = asyncio.create_task(self.close_connection()) + await task else: - log.warn('Connection.closed called while connection already closed') + log.warn('Connection.closed called while connection already closed or not established') self.__state = ConnectionState.CLOSED + async def close_connection(self): + await self.sendProtocolMessage({"action": ProtocolMessageAction.CLOSE}) + + async def sendProtocolMessage(self, protocolMessage): + await self.__websocket.send(json.dumps(protocolMessage)) + async def connect_impl(self): headers = HttpUtils.default_headers() async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}', @@ -82,6 +93,10 @@ async def ws_read_loop(self): self.__connected_future.set_exception( AblyAuthException(error["message"], error["statusCode"], error["code"])) self.__connected_future = None + if action == ProtocolMessageAction.CLOSED: + await self.__websocket.close() + self.__closed_future.set_result(None) + break @property def ably(self): diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 929161a7..203cc0f5 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -27,12 +27,12 @@ async def test_connecting_state(self): await task await ably.close() - async def test_closing_state(self): + async def test_closed_state(self): ably = await RestSetup.get_ably_realtime() await ably.connect() task = asyncio.create_task(ably.close()) await asyncio.sleep(0) - assert ably.connection.state == ConnectionState.CLOSING + assert ably.connection.state == ConnectionState.CLOSED await task async def test_auth_invalid_key(self): From 1eecf45d4e00ca8e7827e4c66117e05ea8ac0598 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 26 Sep 2022 11:31:19 +0100 Subject: [PATCH 163/481] review: await closed future --- ably/realtime/connection.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 8af853c1..3a5a21a4 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -54,13 +54,13 @@ async def close(self): self.__state = ConnectionState.CLOSING self.__closed_future = asyncio.Future() if self.__websocket and self.__state == ConnectionState.CONNECTED: - task = asyncio.create_task(self.close_connection()) - await task + await self.send_close_message() + await self.__closed_future else: log.warn('Connection.closed called while connection already closed or not established') self.__state = ConnectionState.CLOSED - async def close_connection(self): + async def send_close_message(self): await self.sendProtocolMessage({"action": ProtocolMessageAction.CLOSE}) async def sendProtocolMessage(self, protocolMessage): From dd92f5c59b46f237c5a912ae2268932510638b0e Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 26 Sep 2022 23:23:20 +0100 Subject: [PATCH 164/481] refactor: extract Connection internals to ConnectionManager --- ably/realtime/connection.py | 36 ++++++++++++++++++++++++++++++------ ably/realtime/realtime.py | 4 ++-- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 3a5a21a4..0699e2f6 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -25,7 +25,27 @@ class ProtocolMessageAction(IntEnum): CLOSED = 8 -class RealtimeConnection: +class Connection: + def __init__(self, realtime): + self.__realtime = realtime + self.__connection_manager = ConnectionManager(realtime) + self.__state = ConnectionState.INITIALIZED + + async def connect(self): + await self.__connection_manager.connect() + + async def close(self): + await self.__connection_manager.close() + + def on_state_update(self, state): + self.__state = state + + @property + def state(self): + return self.__state + + +class ConnectionManager: def __init__(self, realtime): self.options = realtime.options self.__ably = realtime @@ -34,6 +54,10 @@ def __init__(self, realtime): self.__closed_future = None self.__websocket = None + def enact_state_change(self, state): + self.__state = state + self.ably.connection.on_state_update(state) + async def connect(self): if self.__state == ConnectionState.CONNECTED: return @@ -44,21 +68,21 @@ async def connect(self): return await self.__connected_future else: - self.__state = ConnectionState.CONNECTING + self.enact_state_change(ConnectionState.CONNECTING) self.__connected_future = asyncio.Future() asyncio.create_task(self.connect_impl()) await self.__connected_future - self.__state = ConnectionState.CONNECTED + self.enact_state_change(ConnectionState.CONNECTED) async def close(self): - self.__state = ConnectionState.CLOSING + self.enact_state_change(ConnectionState.CLOSING) self.__closed_future = asyncio.Future() if self.__websocket and self.__state == ConnectionState.CONNECTED: await self.send_close_message() await self.__closed_future else: log.warn('Connection.closed called while connection already closed or not established') - self.__state = ConnectionState.CLOSED + self.enact_state_change(ConnectionState.CLOSED) async def send_close_message(self): await self.sendProtocolMessage({"action": ProtocolMessageAction.CLOSE}) @@ -88,7 +112,7 @@ async def ws_read_loop(self): if action == ProtocolMessageAction.ERROR: # ERROR error = msg["error"] if error['nonfatal'] is False: - self.__state = ConnectionState.FAILED + self.enact_state_change(ConnectionState.FAILED) if self.__connected_future: self.__connected_future.set_exception( AblyAuthException(error["message"], error["statusCode"], error["code"])) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index de70e41c..4f62d576 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -1,5 +1,5 @@ import logging -from ably.realtime.connection import RealtimeConnection +from ably.realtime.connection import Connection from ably.rest.auth import Auth from ably.types.options import Options @@ -26,7 +26,7 @@ def __init__(self, key=None, **kwargs): self.__auth = Auth(self, options) self.__options = options self.key = key - self.__connection = RealtimeConnection(self) + self.__connection = Connection(self) async def connect(self): await self.connection.connect() From d91b622be0e200a19609835adee4ed9f09424c27 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 26 Sep 2022 23:33:17 +0100 Subject: [PATCH 165/481] chore: add loop option --- ably/realtime/realtime.py | 11 +++++++++-- ably/types/options.py | 10 +++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 4f62d576..e22b1da9 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -1,4 +1,5 @@ import logging +import asyncio from ably.realtime.connection import Connection from ably.rest.auth import Auth from ably.types.options import Options @@ -10,7 +11,7 @@ class AblyRealtime: """Ably Realtime Client""" - def __init__(self, key=None, **kwargs): + def __init__(self, key=None, loop=None, **kwargs): """Create an AblyRealtime instance. :Parameters: @@ -18,8 +19,14 @@ def __init__(self, key=None, **kwargs): - `key`: a valid ably key string """ + if loop is None: + try: + loop = asyncio.get_running_loop() + except RuntimeError: + log.warning('Realtime client created outside event loop') + if key is not None: - options = Options(key=key, **kwargs) + options = Options(key=key, loop=loop, **kwargs) else: raise ValueError("Key is missing. Provide an API key.") diff --git a/ably/types/options.py b/ably/types/options.py index 441d87b6..9a4791e0 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -1,9 +1,12 @@ import random import warnings +import logging from ably.transport.defaults import Defaults from ably.types.authoptions import AuthOptions +log = logging.getLogger(__name__) + class Options(AuthOptions): def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, @@ -12,7 +15,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, http_open_timeout=None, http_request_timeout=None, http_max_retry_count=None, http_max_retry_duration=None, fallback_hosts=None, fallback_hosts_use_default=None, fallback_retry_timeout=None, - idempotent_rest_publishing=None, + idempotent_rest_publishing=None, loop=None, **kwargs): super().__init__(**kwargs) @@ -49,6 +52,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, self.__fallback_hosts_use_default = fallback_hosts_use_default self.__fallback_retry_timeout = fallback_retry_timeout self.__idempotent_rest_publishing = idempotent_rest_publishing + self.__loop = loop self.__rest_hosts = self.__get_rest_hosts() @@ -184,6 +188,10 @@ def fallback_retry_timeout(self): def idempotent_rest_publishing(self): return self.__idempotent_rest_publishing + @property + def loop(self): + return self.__loop + def __get_rest_hosts(self): """ Return the list of hosts as they should be tried. First comes the main From 273f0b685a8deb6e2ffce29fae94314b1bfb9c50 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 27 Sep 2022 00:12:16 +0100 Subject: [PATCH 166/481] chore: add pyee dependency --- poetry.lock | 23 +++++++++++++++++++---- pyproject.toml | 1 + 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 68779fba..7c26bd22 100644 --- a/poetry.lock +++ b/poetry.lock @@ -33,10 +33,10 @@ optional = false python-versions = ">=3.5" [package.extras] -dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] -docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] -tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] -tests-no-zope = ["cloudpickle", "cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] +docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] +tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "cloudpickle"] [[package]] name = "certifi" @@ -314,6 +314,17 @@ category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "pyee" +version = "9.0.4" +description = "A port of node.js's EventEmitter to python." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +typing-extensions = "*" + [[package]] name = "pyflakes" version = "2.3.1" @@ -757,6 +768,10 @@ pycryptodome = [ {file = "pycryptodome-3.15.0-pp36-pypy36_pp73-win32.whl", hash = "sha256:9c772c485b27967514d0df1458b56875f4b6d025566bf27399d0c239ff1b369f"}, {file = "pycryptodome-3.15.0.tar.gz", hash = "sha256:9135dddad504592bcc18b0d2d95ce86c3a5ea87ec6447ef25cfedea12d6018b8"}, ] +pyee = [ + {file = "pyee-9.0.4-py2.py3-none-any.whl", hash = "sha256:9f066570130c554e9cc12de5a9d86f57c7ee47fece163bbdaa3e9c933cfbdfa5"}, + {file = "pyee-9.0.4.tar.gz", hash = "sha256:2770c4928abc721f46b705e6a72b0c59480c4a69c9a83ca0b00bb994f1ea4b32"}, +] pyflakes = [ {file = "pyflakes-2.3.1-py2.py3-none-any.whl", hash = "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3"}, {file = "pyflakes-2.3.1.tar.gz", hash = "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"}, diff --git a/pyproject.toml b/pyproject.toml index a3dd5f37..b56ab615 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ h2 = "^4.0.0" pycrypto = { version = "^2.6.1", optional = true } pycryptodome = { version = "*", optional = true } websockets = "^10.3" +pyee = "^9.0.4" [tool.poetry.extras] oldcrypto = ["pycrypto"] From fd423d6dab8f7338f7de84425a450dd25b3842f9 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 27 Sep 2022 00:14:56 +0100 Subject: [PATCH 167/481] feat: queryable connection state --- ably/realtime/connection.py | 22 ++++++++++++++++++---- test/ably/realtimeconnection_test.py | 4 ++-- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 0699e2f6..898b226d 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -1,3 +1,4 @@ +import functools import logging import asyncio import websockets @@ -5,6 +6,7 @@ from ably.http.httputils import HttpUtils from ably.util.exceptions import AblyAuthException from enum import Enum, IntEnum +from pyee.asyncio import AsyncIOEventEmitter log = logging.getLogger(__name__) @@ -25,11 +27,13 @@ class ProtocolMessageAction(IntEnum): CLOSED = 8 -class Connection: +class Connection(AsyncIOEventEmitter): def __init__(self, realtime): self.__realtime = realtime self.__connection_manager = ConnectionManager(realtime) self.__state = ConnectionState.INITIALIZED + self.__connection_manager.on('connectionstate', self.on_state_update) + super().__init__() async def connect(self): await self.__connection_manager.connect() @@ -39,13 +43,18 @@ async def close(self): def on_state_update(self, state): self.__state = state + self.__realtime.options.loop.call_soon(functools.partial(self.emit, state)) @property def state(self): return self.__state + @state.setter + def state(self, value): + self.__state = value -class ConnectionManager: + +class ConnectionManager(AsyncIOEventEmitter): def __init__(self, realtime): self.options = realtime.options self.__ably = realtime @@ -53,10 +62,11 @@ def __init__(self, realtime): self.__connected_future = None self.__closed_future = None self.__websocket = None + super().__init__() def enact_state_change(self, state): self.__state = state - self.ably.connection.on_state_update(state) + self.emit('connectionstate', state) async def connect(self): if self.__state == ConnectionState.CONNECTED: @@ -75,9 +85,11 @@ async def connect(self): self.enact_state_change(ConnectionState.CONNECTED) async def close(self): + if self.__state != ConnectionState.CONNECTED: + log.warn('Connection.closed called while connection state not connected') self.enact_state_change(ConnectionState.CLOSING) self.__closed_future = asyncio.Future() - if self.__websocket and self.__state == ConnectionState.CONNECTED: + if self.__websocket: await self.send_close_message() await self.__closed_future else: @@ -117,8 +129,10 @@ async def ws_read_loop(self): self.__connected_future.set_exception( AblyAuthException(error["message"], error["statusCode"], error["code"])) self.__connected_future = None + self.__websocket = None if action == ProtocolMessageAction.CLOSED: await self.__websocket.close() + self.__websocket = None self.__closed_future.set_result(None) break diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 203cc0f5..929161a7 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -27,12 +27,12 @@ async def test_connecting_state(self): await task await ably.close() - async def test_closed_state(self): + async def test_closing_state(self): ably = await RestSetup.get_ably_realtime() await ably.connect() task = asyncio.create_task(ably.close()) await asyncio.sleep(0) - assert ably.connection.state == ConnectionState.CLOSED + assert ably.connection.state == ConnectionState.CLOSING await task async def test_auth_invalid_key(self): From cd3fcf2c85b33a39e89389f2abe69857033ee2fc Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 27 Sep 2022 00:15:20 +0100 Subject: [PATCH 168/481] test: add tests for connection eventemitter interface --- test/ably/eventemitter_test.py | 51 ++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 test/ably/eventemitter_test.py diff --git a/test/ably/eventemitter_test.py b/test/ably/eventemitter_test.py new file mode 100644 index 00000000..d57f046a --- /dev/null +++ b/test/ably/eventemitter_test.py @@ -0,0 +1,51 @@ +import asyncio +from ably.realtime.connection import ConnectionState +from unittest.mock import Mock +from test.ably.restsetup import RestSetup +from test.ably.utils import BaseAsyncTestCase + + +class TestEventEmitter(BaseAsyncTestCase): + async def setUp(self): + self.test_vars = await RestSetup.get_test_vars() + + async def test_connection_events(self): + realtime = await RestSetup.get_ably_realtime() + listener = Mock() + realtime.connection.add_listener(ConnectionState.CONNECTED, listener) + + await realtime.connect() + + # Listener is only called once event loop is free + listener.assert_not_called() + await asyncio.sleep(0) + listener.assert_called_once() + await realtime.close() + + async def test_event_listener_error(self): + realtime = await RestSetup.get_ably_realtime() + listener = Mock() + + # If a listener throws an exception it should not propagate (#RTE6) + listener.side_effect = Exception() + realtime.connection.add_listener(ConnectionState.CONNECTED, listener) + + await realtime.connect() + + listener.assert_not_called() + await asyncio.sleep(0) + listener.assert_called_once() + await realtime.close() + + async def test_event_emitter_off(self): + realtime = await RestSetup.get_ably_realtime() + listener = Mock() + realtime.connection.add_listener(ConnectionState.CONNECTED, listener) + realtime.connection.remove_listener(ConnectionState.CONNECTED, listener) + + await realtime.connect() + + listener.assert_not_called() + await asyncio.sleep(0) + listener.assert_not_called() + await realtime.close() From 07ed26feeff945faac926b95750fc11588256e14 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 27 Sep 2022 00:57:34 +0100 Subject: [PATCH 169/481] fix: finish tasks gracefully on failed connection --- ably/realtime/connection.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 898b226d..429552a7 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -62,6 +62,7 @@ def __init__(self, realtime): self.__connected_future = None self.__closed_future = None self.__websocket = None + self.connect_impl_task = None super().__init__() def enact_state_change(self, state): @@ -80,7 +81,7 @@ async def connect(self): else: self.enact_state_change(ConnectionState.CONNECTING) self.__connected_future = asyncio.Future() - asyncio.create_task(self.connect_impl()) + self.connect_impl_task = self.ably.options.loop.create_task(self.connect_impl()) await self.__connected_future self.enact_state_change(ConnectionState.CONNECTED) @@ -89,12 +90,14 @@ async def close(self): log.warn('Connection.closed called while connection state not connected') self.enact_state_change(ConnectionState.CLOSING) self.__closed_future = asyncio.Future() - if self.__websocket: + if self.__websocket and self.__state != ConnectionState.FAILED: await self.send_close_message() await self.__closed_future else: log.warn('Connection.closed called while connection already closed or not established') self.enact_state_change(ConnectionState.CLOSED) + if self.connect_impl_task: + await self.connect_impl_task async def send_close_message(self): await self.sendProtocolMessage({"action": ProtocolMessageAction.CLOSE}) @@ -107,8 +110,11 @@ async def connect_impl(self): async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}', extra_headers=headers) as websocket: self.__websocket = websocket - task = asyncio.create_task(self.ws_read_loop()) - await task + task = self.ably.options.loop.create_task(self.ws_read_loop()) + try: + await task + except AblyAuthException: + return async def ws_read_loop(self): while True: @@ -125,11 +131,12 @@ async def ws_read_loop(self): error = msg["error"] if error['nonfatal'] is False: self.enact_state_change(ConnectionState.FAILED) + exception = AblyAuthException(error["message"], error["statusCode"], error["code"]) if self.__connected_future: - self.__connected_future.set_exception( - AblyAuthException(error["message"], error["statusCode"], error["code"])) + self.__connected_future.set_exception(exception) self.__connected_future = None self.__websocket = None + raise exception if action == ProtocolMessageAction.CLOSED: await self.__websocket.close() self.__websocket = None From 154880ff4cc4740663a6177f5ca67a2de8145b1f Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 27 Sep 2022 12:42:35 +0100 Subject: [PATCH 170/481] implement realtime ping --- ably/realtime/connection.py | 26 ++++++++++++++++++++++++- ably/realtime/realtime.py | 3 +++ ably/util/helper.py | 9 +++++++++ test/ably/realtimeconnection_test.py | 29 ++++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 ably/util/helper.py diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 429552a7..6ac5bde3 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -7,6 +7,8 @@ from ably.util.exceptions import AblyAuthException from enum import Enum, IntEnum from pyee.asyncio import AsyncIOEventEmitter +from datetime import datetime +from ably.util import helper log = logging.getLogger(__name__) @@ -21,6 +23,7 @@ class ConnectionState(Enum): class ProtocolMessageAction(IntEnum): + HEARTBEAT = 0 CONNECTED = 4 ERROR = 9 CLOSE = 7 @@ -68,16 +71,19 @@ def __init__(self, realtime): def enact_state_change(self, state): self.__state = state self.emit('connectionstate', state) + self.__ping_future = None async def connect(self): if self.__state == ConnectionState.CONNECTED: + await self.ping() return if self.__state == ConnectionState.CONNECTING: if self.__connected_future is None: - log.fatal('Connection state is CONNECTING but connected_future does not exits') + log.fatal('Connection state is CONNECTING but connected_future does not exist') return await self.__connected_future + await self.ping() else: self.enact_state_change(ConnectionState.CONNECTING) self.__connected_future = asyncio.Future() @@ -116,6 +122,20 @@ async def connect_impl(self): except AblyAuthException: return + async def ping(self): + self.__ping_future = asyncio.Future() + if self.__state in [ConnectionState.CONNECTED, ConnectionState.CONNECTING]: + ping_start_time = datetime.now().timestamp() + await self.sendProtocolMessage({"action": ProtocolMessageAction.HEARTBEAT, + "id": helper.get_random_id()}) + else: + log.error("Cannot send ping request. Connection not in connected or connecting") + return + await self.__ping_future + ping_end_time = datetime.now().timestamp() + response_time_ms = (ping_end_time - ping_start_time) * 1000 + return round(response_time_ms, 2) + async def ws_read_loop(self): while True: raw = await self.__websocket.recv() @@ -142,6 +162,10 @@ async def ws_read_loop(self): self.__websocket = None self.__closed_future.set_result(None) break + if action == ProtocolMessageAction.HEARTBEAT: + if self.__ping_future: + self.__ping_future.set_result(None) + self.__ping_future = None @property def ably(self): diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index e22b1da9..5e3edba2 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -41,6 +41,9 @@ async def connect(self): async def close(self): await self.connection.close() + async def ping(self): + return await self.connection.ping() + @property def auth(self): return self.__auth diff --git a/ably/util/helper.py b/ably/util/helper.py new file mode 100644 index 00000000..0ca32ba1 --- /dev/null +++ b/ably/util/helper.py @@ -0,0 +1,9 @@ +import random +import string + + +def get_random_id(): + # get random string of letters and digits + source = string.ascii_letters + string.digits + random_id = ''.join((random.choice(source) for i in range(8))) + return random_id diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 929161a7..2928e6a7 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -41,3 +41,32 @@ async def test_auth_invalid_key(self): await ably.connect() assert ably.connection.state == ConnectionState.FAILED await ably.close() + + async def test_connection_ping_connected(self): + ably = await RestSetup.get_ably_realtime() + await ably.connect() + response_time_ms = await ably.ping() + assert response_time_ms is not None + assert type(response_time_ms) is float + + async def test_connection_ping_initialized(self): + ably = await RestSetup.get_ably_realtime() + assert ably.connection.state == ConnectionState.INITIALIZED + response_time_ms = await ably.ping() + assert response_time_ms is None + + async def test_connection_ping_failed(self): + ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) + with pytest.raises(AblyAuthException): + await ably.connect() + assert ably.connection.state == ConnectionState.FAILED + response_time_ms = await ably.ping() + assert response_time_ms is None + + async def test_connection_ping_closed(self): + ably = await RestSetup.get_ably_realtime() + await ably.connect() + assert ably.connection.state == ConnectionState.CONNECTED + await ably.close() + response_time_ms = await ably.ping() + assert response_time_ms is None From c9f1cafc231464b3b10dde181a8465fdb1a62f56 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 27 Sep 2022 18:50:11 +0100 Subject: [PATCH 171/481] review: correct rtn13b and rtn13e --- ably/realtime/connection.py | 21 ++++++++++++++------- test/ably/realtimeconnection_test.py | 14 +++++++------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 6ac5bde3..ae2ab701 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -4,7 +4,7 @@ import websockets import json from ably.http.httputils import HttpUtils -from ably.util.exceptions import AblyAuthException +from ably.util.exceptions import AblyAuthException, AblyException from enum import Enum, IntEnum from pyee.asyncio import AsyncIOEventEmitter from datetime import datetime @@ -44,6 +44,9 @@ async def connect(self): async def close(self): await self.__connection_manager.close() + async def ping(self): + return await self.__connection_manager.ping() + def on_state_update(self, state): self.__state = state self.__realtime.options.loop.call_soon(functools.partial(self.emit, state)) @@ -66,12 +69,12 @@ def __init__(self, realtime): self.__closed_future = None self.__websocket = None self.connect_impl_task = None + self.__ping_future = None super().__init__() def enact_state_change(self, state): self.__state = state self.emit('connectionstate', state) - self.__ping_future = None async def connect(self): if self.__state == ConnectionState.CONNECTED: @@ -125,13 +128,14 @@ async def connect_impl(self): async def ping(self): self.__ping_future = asyncio.Future() if self.__state in [ConnectionState.CONNECTED, ConnectionState.CONNECTING]: + self.__ping_id = helper.get_random_id() ping_start_time = datetime.now().timestamp() await self.sendProtocolMessage({"action": ProtocolMessageAction.HEARTBEAT, - "id": helper.get_random_id()}) + "id": self.__ping_id}) else: - log.error("Cannot send ping request. Connection not in connected or connecting") - return - await self.__ping_future + raise AblyException("Cannot send ping request. Calling ping in invalid state", 40000, 400) + if self.__ping_future: + await self.__ping_future ping_end_time = datetime.now().timestamp() response_time_ms = (ping_end_time - ping_start_time) * 1000 return round(response_time_ms, 2) @@ -164,7 +168,10 @@ async def ws_read_loop(self): break if action == ProtocolMessageAction.HEARTBEAT: if self.__ping_future: - self.__ping_future.set_result(None) + # Resolve on heartbeat from ping request. + # TODO: Handle Normal heartbeat if required + if self.__ping_id == msg["id"]: + self.__ping_future.set_result(None) self.__ping_future = None @property diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 2928e6a7..aa27e50a 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -1,7 +1,7 @@ import asyncio from ably.realtime.connection import ConnectionState import pytest -from ably.util.exceptions import AblyAuthException +from ably.util.exceptions import AblyAuthException, AblyException from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase @@ -52,21 +52,21 @@ async def test_connection_ping_connected(self): async def test_connection_ping_initialized(self): ably = await RestSetup.get_ably_realtime() assert ably.connection.state == ConnectionState.INITIALIZED - response_time_ms = await ably.ping() - assert response_time_ms is None + with pytest.raises(AblyException): + await ably.ping() async def test_connection_ping_failed(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) with pytest.raises(AblyAuthException): await ably.connect() assert ably.connection.state == ConnectionState.FAILED - response_time_ms = await ably.ping() - assert response_time_ms is None + with pytest.raises(AblyException): + await ably.ping() async def test_connection_ping_closed(self): ably = await RestSetup.get_ably_realtime() await ably.connect() assert ably.connection.state == ConnectionState.CONNECTED await ably.close() - response_time_ms = await ably.ping() - assert response_time_ms is None + with pytest.raises(AblyException): + await ably.ping() From b390ff224315a404e2a23d294d12df7eff8d09ee Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 28 Sep 2022 14:32:00 +0100 Subject: [PATCH 172/481] refactor realtime ping --- ably/realtime/connection.py | 2 +- test/ably/realtimeconnection_test.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index ae2ab701..32408c6f 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -170,7 +170,7 @@ async def ws_read_loop(self): if self.__ping_future: # Resolve on heartbeat from ping request. # TODO: Handle Normal heartbeat if required - if self.__ping_id == msg["id"]: + if self.__ping_id == msg.get("id"): self.__ping_future.set_result(None) self.__ping_future = None diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index aa27e50a..7a4a2212 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -52,21 +52,28 @@ async def test_connection_ping_connected(self): async def test_connection_ping_initialized(self): ably = await RestSetup.get_ably_realtime() assert ably.connection.state == ConnectionState.INITIALIZED - with pytest.raises(AblyException): + with pytest.raises(AblyException) as exception: await ably.ping() + assert exception.value.code == 400 + assert exception.value.status_code == 40000 async def test_connection_ping_failed(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) with pytest.raises(AblyAuthException): await ably.connect() assert ably.connection.state == ConnectionState.FAILED - with pytest.raises(AblyException): + with pytest.raises(AblyException) as exception: await ably.ping() + assert exception.value.code == 400 + assert exception.value.status_code == 40000 + await ably.close() async def test_connection_ping_closed(self): ably = await RestSetup.get_ably_realtime() await ably.connect() assert ably.connection.state == ConnectionState.CONNECTED await ably.close() - with pytest.raises(AblyException): + with pytest.raises(AblyException) as exception: await ably.ping() + assert exception.value.code == 400 + assert exception.value.status_code == 40000 From 8c485d1a0cf9d6bc792a306397363fd9561300e5 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 27 Sep 2022 23:30:05 +0100 Subject: [PATCH 173/481] feat: RealtimeChannels.get/release --- ably/realtime/realtime.py | 21 +++++++++++++++++++++ ably/realtime/realtime_channel.py | 7 +++++++ 2 files changed, 28 insertions(+) create mode 100644 ably/realtime/realtime_channel.py diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 5e3edba2..7ff1685f 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -3,6 +3,7 @@ from ably.realtime.connection import Connection from ably.rest.auth import Auth from ably.types.options import Options +from ably.realtime.realtime_channel import RealtimeChannel log = logging.getLogger(__name__) @@ -34,6 +35,7 @@ def __init__(self, key=None, loop=None, **kwargs): self.__options = options self.key = key self.__connection = Connection(self) + self.__channels = Channels() async def connect(self): await self.connection.connect() @@ -56,3 +58,22 @@ def options(self): def connection(self): """Establish realtime connection""" return self.__connection + + @property + def channels(self): + return self.__channels + + +class Channels: + def __init__(self): + self.all = {} + + def get(self, name): + if not self.all.get(name): + self.all[name] = RealtimeChannel(name) + return self.all[name] + + def release(self, name): + if not self.all.get(name): + return + del self.all[name] diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py new file mode 100644 index 00000000..a423c722 --- /dev/null +++ b/ably/realtime/realtime_channel.py @@ -0,0 +1,7 @@ +class RealtimeChannel(): + def __init__(self, name): + self.__name = name + + @property + def name(self): + return self.__name From a71d85e7161184470a8cf4465ac3302b3c6c2d64 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 27 Sep 2022 23:30:38 +0100 Subject: [PATCH 174/481] test: RealtimeChannels.get/release --- test/ably/realtimechannel_test.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 test/ably/realtimechannel_test.py diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py new file mode 100644 index 00000000..2b4d162a --- /dev/null +++ b/test/ably/realtimechannel_test.py @@ -0,0 +1,21 @@ +from test.ably.restsetup import RestSetup +from test.ably.utils import BaseAsyncTestCase + + +class TestRealtimeChannel(BaseAsyncTestCase): + async def setUp(self): + self.test_vars = await RestSetup.get_test_vars() + self.valid_key_format = "api:key" + + async def test_channels_get(self): + ably = await RestSetup.get_ably_realtime() + channel = ably.channels.get('my_channel') + assert channel == ably.channels.all['my_channel'] + await ably.close() + + async def test_channels_release(self): + ably = await RestSetup.get_ably_realtime() + ably.channels.get('my_channel') + ably.channels.release('my_channel') + assert ably.channels.all.get('my_channel') is None + await ably.close() From 1ed814e3bc5e2ed58091b18e18373df4f3968a4f Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 28 Sep 2022 00:39:13 +0100 Subject: [PATCH 175/481] feat: RealtimeChannel attach/detach --- ably/realtime/connection.py | 10 +++ ably/realtime/realtime.py | 13 +++- ably/realtime/realtime_channel.py | 118 +++++++++++++++++++++++++++++- 3 files changed, 136 insertions(+), 5 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 32408c6f..a0896a00 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -28,6 +28,10 @@ class ProtocolMessageAction(IntEnum): ERROR = 9 CLOSE = 7 CLOSED = 8 + ATTACH = 10 + ATTACHED = 11 + DETACH = 12 + DETACHED = 13 class Connection(AsyncIOEventEmitter): @@ -59,6 +63,10 @@ def state(self): def state(self, value): self.__state = value + @property + def connection_manager(self): + return self.__connection_manager + class ConnectionManager(AsyncIOEventEmitter): def __init__(self, realtime): @@ -173,6 +181,8 @@ async def ws_read_loop(self): if self.__ping_id == msg.get("id"): self.__ping_future.set_result(None) self.__ping_future = None + if action in [ProtocolMessageAction.ATTACHED, ProtocolMessageAction.DETACHED]: + self.ably.channels.on_channel_message(msg) @property def ably(self): diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 7ff1685f..f8f658bf 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -35,7 +35,7 @@ def __init__(self, key=None, loop=None, **kwargs): self.__options = options self.key = key self.__connection = Connection(self) - self.__channels = Channels() + self.__channels = Channels(self) async def connect(self): await self.connection.connect() @@ -65,15 +65,22 @@ def channels(self): class Channels: - def __init__(self): + def __init__(self, realtime): self.all = {} + self.__realtime = realtime def get(self, name): if not self.all.get(name): - self.all[name] = RealtimeChannel(name) + self.all[name] = RealtimeChannel(self.__realtime, name) return self.all[name] def release(self, name): if not self.all.get(name): return del self.all[name] + + def on_channel_message(self, msg): + channel = self.all.get(msg.get('channel')) + if not channel: + log.warning('Channel message recieved but no channel instance found') + channel.on_message(msg) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index a423c722..34976438 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -1,7 +1,121 @@ -class RealtimeChannel(): - def __init__(self, name): +import asyncio +import logging +from ably.realtime.connection import ConnectionState, ProtocolMessageAction +from ably.util.exceptions import AblyException +from pyee.asyncio import AsyncIOEventEmitter +from enum import Enum + +log = logging.getLogger(__name__) + + +class ChannelState(Enum): + INITIALIZED = 'initialized' + ATTACHING = 'attaching' + ATTACHED = 'attached' + DETACHING = 'detaching' + DETACHED = 'detached' + + +class RealtimeChannel(AsyncIOEventEmitter): + def __init__(self, realtime, name): self.__name = name + self.__attach_future = None + self.__detach_future = None + self.__realtime = realtime + self.__state = ChannelState.INITIALIZED + super().__init__() + + async def attach(self): + # RTL4a - if channel is attached do nothing + if self.state == ChannelState.ATTACHED: + return + + # RTL4b + if self.__realtime.connection.state not in [ConnectionState.CONNECTING, ConnectionState.CONNECTED]: + raise AblyException( + message=f"Unable to attach; channel state = {self.state}", + code=90001, + status_code=400 + ) + + # RTL4h - wait for pending attach/detach + if self.state == ChannelState.ATTACHING: + await self.__attach_future + return + elif self.state == ChannelState.DETACHING: + await self.__detach_future + + self.set_state(ChannelState.ATTACHING) + + # RTL4i - wait for pending connection + if self.__realtime.connection.state == ConnectionState.CONNECTING: + await self.__realtime.connect() + + self.__attach_future = asyncio.Future() + await self.__realtime.connection.connection_manager.sendProtocolMessage( + { + "action": ProtocolMessageAction.ATTACH, + "channel": self.name, + } + ) + await self.__attach_future + self.set_state(ChannelState.ATTACHED) + + async def detach(self): + # RTL5g - raise exception if state invalid + if self.__realtime.connection.state in [ConnectionState.CLOSING, ConnectionState.FAILED]: + raise AblyException( + message=f"Unable to detach; channel state = {self.state}", + code=90001, + status_code=400 + ) + + # RTL5a - if channel already detached do nothing + if self.state in [ChannelState.INITIALIZED, ChannelState.DETACHED]: + return + + # RTL5i - wait for pending attach/detach + if self.state == ChannelState.DETACHING: + await self.__detach_future + return + elif self.state == ChannelState.ATTACHING: + await self.__attach_future + + self.set_state(ChannelState.DETACHING) + + # RTL5h - wait for pending connection + if self.__realtime.connection.state == ConnectionState.CONNECTING: + await self.__realtime.connect() + + self.__detach_future = asyncio.Future() + await self.__realtime.connection.connection_manager.sendProtocolMessage( + { + "action": ProtocolMessageAction.DETACH, + "channel": self.name, + } + ) + await self.__detach_future + self.set_state(ChannelState.DETACHED) + + def on_message(self, msg): + action = msg.get('action') + if action == ProtocolMessageAction.ATTACHED: + if self.__attach_future: + self.__attach_future.set_result(None) + self.__attach_future = None + elif action == ProtocolMessageAction.DETACHED: + if self.__detach_future: + self.__detach_future.set_result(None) + self.__detach_future = None + + def set_state(self, state): + self.__state = state + self.emit(state) @property def name(self): return self.__name + + @property + def state(self): + return self.__state From f36370b68b2d4c224779fb30cff67cd6d5a6b406 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 28 Sep 2022 00:39:22 +0100 Subject: [PATCH 176/481] test: RealtimeChannel attach/detach --- test/ably/realtimechannel_test.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py index 2b4d162a..9f08736c 100644 --- a/test/ably/realtimechannel_test.py +++ b/test/ably/realtimechannel_test.py @@ -1,3 +1,4 @@ +from ably.realtime.realtime_channel import ChannelState from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase @@ -19,3 +20,21 @@ async def test_channels_release(self): ably.channels.release('my_channel') assert ably.channels.all.get('my_channel') is None await ably.close() + + async def test_channel_attach(self): + ably = await RestSetup.get_ably_realtime() + await ably.connect() + channel = ably.channels.get('my_channel') + assert channel.state == ChannelState.INITIALIZED + await channel.attach() + assert channel.state == ChannelState.ATTACHED + await ably.close() + + async def test_channel_detach(self): + ably = await RestSetup.get_ably_realtime() + await ably.connect() + channel = ably.channels.get('my_channel') + await channel.attach() + await channel.detach() + assert channel.state == ChannelState.DETACHED + await ably.close() From 75e76a3b16fb330fe1b6f5992be918f54d1f96a5 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Fri, 30 Sep 2022 01:19:13 +0100 Subject: [PATCH 177/481] fix: ping behaviour fixups --- ably/realtime/connection.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index a0896a00..275c64f8 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -86,7 +86,6 @@ def enact_state_change(self, state): async def connect(self): if self.__state == ConnectionState.CONNECTED: - await self.ping() return if self.__state == ConnectionState.CONNECTING: @@ -94,7 +93,6 @@ async def connect(self): log.fatal('Connection state is CONNECTING but connected_future does not exist') return await self.__connected_future - await self.ping() else: self.enact_state_change(ConnectionState.CONNECTING) self.__connected_future = asyncio.Future() @@ -134,6 +132,10 @@ async def connect_impl(self): return async def ping(self): + if self.__ping_future: + response = await self.__ping_future + return response + self.__ping_future = asyncio.Future() if self.__state in [ConnectionState.CONNECTED, ConnectionState.CONNECTING]: self.__ping_id = helper.get_random_id() @@ -142,8 +144,6 @@ async def ping(self): "id": self.__ping_id}) else: raise AblyException("Cannot send ping request. Calling ping in invalid state", 40000, 400) - if self.__ping_future: - await self.__ping_future ping_end_time = datetime.now().timestamp() response_time_ms = (ping_end_time - ping_start_time) * 1000 return round(response_time_ms, 2) @@ -180,7 +180,7 @@ async def ws_read_loop(self): # TODO: Handle Normal heartbeat if required if self.__ping_id == msg.get("id"): self.__ping_future.set_result(None) - self.__ping_future = None + self.__ping_future = None if action in [ProtocolMessageAction.ATTACHED, ProtocolMessageAction.DETACHED]: self.ably.channels.on_channel_message(msg) From 3fc06b651c1830796cad0eee7f1e2741b388f3c2 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Fri, 30 Sep 2022 22:29:22 +0100 Subject: [PATCH 178/481] refactor: separate connection state checks from implementation --- ably/realtime/connection.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 275c64f8..43672b03 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -76,7 +76,7 @@ def __init__(self, realtime): self.__connected_future = None self.__closed_future = None self.__websocket = None - self.connect_impl_task = None + self.setup_ws_task = None self.__ping_future = None super().__init__() @@ -93,12 +93,11 @@ async def connect(self): log.fatal('Connection state is CONNECTING but connected_future does not exist') return await self.__connected_future + self.enact_state_change(ConnectionState.CONNECTED) else: self.enact_state_change(ConnectionState.CONNECTING) self.__connected_future = asyncio.Future() - self.connect_impl_task = self.ably.options.loop.create_task(self.connect_impl()) - await self.__connected_future - self.enact_state_change(ConnectionState.CONNECTED) + await self.connect_impl() async def close(self): if self.__state != ConnectionState.CONNECTED: @@ -111,8 +110,13 @@ async def close(self): else: log.warn('Connection.closed called while connection already closed or not established') self.enact_state_change(ConnectionState.CLOSED) - if self.connect_impl_task: - await self.connect_impl_task + if self.setup_ws_task: + await self.setup_ws_task + + async def connect_impl(self): + self.setup_ws_task = self.ably.options.loop.create_task(self.setup_ws()) + await self.__connected_future + self.enact_state_change(ConnectionState.CONNECTED) async def send_close_message(self): await self.sendProtocolMessage({"action": ProtocolMessageAction.CLOSE}) @@ -120,7 +124,7 @@ async def send_close_message(self): async def sendProtocolMessage(self, protocolMessage): await self.__websocket.send(json.dumps(protocolMessage)) - async def connect_impl(self): + async def setup_ws(self): headers = HttpUtils.default_headers() async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}', extra_headers=headers) as websocket: From 82d0f07362fc462fe911e75f81d83363c1c5e443 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Fri, 30 Sep 2022 22:31:06 +0100 Subject: [PATCH 179/481] refactor: single initial connection state --- ably/realtime/connection.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 43672b03..ff560fe1 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -38,7 +38,7 @@ class Connection(AsyncIOEventEmitter): def __init__(self, realtime): self.__realtime = realtime self.__connection_manager = ConnectionManager(realtime) - self.__state = ConnectionState.INITIALIZED + self.__connection_manager = ConnectionManager(realtime, self.state) self.__connection_manager.on('connectionstate', self.on_state_update) super().__init__() @@ -69,10 +69,10 @@ def connection_manager(self): class ConnectionManager(AsyncIOEventEmitter): - def __init__(self, realtime): + def __init__(self, realtime, initial_state): self.options = realtime.options self.__ably = realtime - self.__state = ConnectionState.INITIALIZED + self.__state = initial_state self.__connected_future = None self.__closed_future = None self.__websocket = None From ac15ff660e410c42d566f5f0d1613e1dd84d98c6 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 4 Oct 2022 22:26:24 +0100 Subject: [PATCH 180/481] feat: add autoconnect implementation and client option fixes #321 --- ably/realtime/connection.py | 4 ++-- ably/realtime/realtime.py | 3 +++ ably/types/options.py | 7 ++++++- test/ably/realtimeconnection_test.py | 5 +++-- test/ably/realtimeinit_test.py | 10 +++++----- 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index ff560fe1..cba28eaf 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -37,7 +37,7 @@ class ProtocolMessageAction(IntEnum): class Connection(AsyncIOEventEmitter): def __init__(self, realtime): self.__realtime = realtime - self.__connection_manager = ConnectionManager(realtime) + self.__state = ConnectionState.CONNECTING if realtime.options.auto_connect else ConnectionState.INITIALIZED self.__connection_manager = ConnectionManager(realtime, self.state) self.__connection_manager.on('connectionstate', self.on_state_update) super().__init__() @@ -73,7 +73,7 @@ def __init__(self, realtime, initial_state): self.options = realtime.options self.__ably = realtime self.__state = initial_state - self.__connected_future = None + self.__connected_future = asyncio.Future() if initial_state == ConnectionState.CONNECTING else None self.__closed_future = None self.__websocket = None self.setup_ws_task = None diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index f8f658bf..35f711c0 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -37,6 +37,9 @@ def __init__(self, key=None, loop=None, **kwargs): self.__connection = Connection(self) self.__channels = Channels(self) + if options.auto_connect: + asyncio.ensure_future(self.connection.connection_manager.connect_impl()) + async def connect(self): await self.connection.connect() diff --git a/ably/types/options.py b/ably/types/options.py index 9a4791e0..6d254440 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -15,7 +15,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, http_open_timeout=None, http_request_timeout=None, http_max_retry_count=None, http_max_retry_duration=None, fallback_hosts=None, fallback_hosts_use_default=None, fallback_retry_timeout=None, - idempotent_rest_publishing=None, loop=None, + idempotent_rest_publishing=None, loop=None, auto_connect=True, **kwargs): super().__init__(**kwargs) @@ -53,6 +53,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, self.__fallback_retry_timeout = fallback_retry_timeout self.__idempotent_rest_publishing = idempotent_rest_publishing self.__loop = loop + self.__auto_connect = auto_connect self.__rest_hosts = self.__get_rest_hosts() @@ -192,6 +193,10 @@ def idempotent_rest_publishing(self): def loop(self): return self.__loop + @property + def auto_connect(self): + return self.__auto_connect + def __get_rest_hosts(self): """ Return the list of hosts as they should be tried. First comes the main diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 7a4a2212..9695dd3c 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -12,7 +12,7 @@ async def setUp(self): self.valid_key_format = "api:key" async def test_auth_connection(self): - ably = await RestSetup.get_ably_realtime() + ably = await RestSetup.get_ably_realtime(auto_connect=False) assert ably.connection.state == ConnectionState.INITIALIZED await ably.connect() assert ably.connection.state == ConnectionState.CONNECTED @@ -48,9 +48,10 @@ async def test_connection_ping_connected(self): response_time_ms = await ably.ping() assert response_time_ms is not None assert type(response_time_ms) is float + await ably.close() async def test_connection_ping_initialized(self): - ably = await RestSetup.get_ably_realtime() + ably = await RestSetup.get_ably_realtime(auto_connect=False) assert ably.connection.state == ConnectionState.INITIALIZED with pytest.raises(AblyException) as exception: await ably.ping() diff --git a/test/ably/realtimeinit_test.py b/test/ably/realtimeinit_test.py index a85f9576..fdb99a8e 100644 --- a/test/ably/realtimeinit_test.py +++ b/test/ably/realtimeinit_test.py @@ -12,24 +12,24 @@ async def setUp(self): self.valid_key_format = "api:key" async def test_auth_with_valid_key(self): - ably = await RestSetup.get_ably_realtime(key=self.test_vars["keys"][0]["key_str"]) + ably = await RestSetup.get_ably_realtime(key=self.test_vars["keys"][0]["key_str"], auto_connect=False) assert Auth.Method.BASIC == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" assert ably.auth.auth_options.key_name == self.test_vars["keys"][0]['key_name'] assert ably.auth.auth_options.key_secret == self.test_vars["keys"][0]['key_secret'] async def test_auth_incorrect_key(self): with pytest.raises(AblyAuthException): - await RestSetup.get_ably_realtime(key="some invalid key") + await RestSetup.get_ably_realtime(key="some invalid key", auto_connect=False) async def test_auth_with_valid_key_format(self): key = self.valid_key_format.split(":") - ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) + ably = await RestSetup.get_ably_realtime(key=self.valid_key_format, auto_connect=False) assert Auth.Method.BASIC == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" assert ably.auth.auth_options.key_name == key[0] assert ably.auth.auth_options.key_secret == key[1] async def test_auth_connection(self): - ably = await RestSetup.get_ably_realtime() + ably = await RestSetup.get_ably_realtime(auto_connect=False) assert ably.connection.state == ConnectionState.INITIALIZED await ably.connect() assert ably.connection.state == ConnectionState.CONNECTED @@ -37,7 +37,7 @@ async def test_auth_connection(self): assert ably.connection.state == ConnectionState.CLOSED async def test_auth_invalid_key(self): - ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) + ably = await RestSetup.get_ably_realtime(key=self.valid_key_format, auto_connect=False) with pytest.raises(AblyAuthException): await ably.connect() await ably.close() From edcec715cd9f8fb2bb7a73877c20440c5c289a3f Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 4 Oct 2022 22:27:26 +0100 Subject: [PATCH 181/481] test: add test for autoconnect behaviour --- test/ably/realtimeconnection_test.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 9695dd3c..ec6980f1 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -78,3 +78,11 @@ async def test_connection_ping_closed(self): await ably.ping() assert exception.value.code == 400 assert exception.value.status_code == 40000 + + async def test_auto_connect(self): + ably = await RestSetup.get_ably_realtime() + connect_future = asyncio.Future() + ably.connection.on(ConnectionState.CONNECTED, lambda: connect_future.set_result(None)) + await connect_future + assert ably.connection.state == ConnectionState.CONNECTED + await ably.close() From 91ea30051c058996c62008539bdc8d6a672edd67 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 10 Oct 2022 01:44:42 +0100 Subject: [PATCH 182/481] feat: RealtimeChannel.subscribe --- ably/realtime/connection.py | 7 +++++- ably/realtime/realtime_channel.py | 39 +++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index cba28eaf..de267164 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -32,6 +32,7 @@ class ProtocolMessageAction(IntEnum): ATTACHED = 11 DETACH = 12 DETACHED = 13 + MESSAGE = 15 class Connection(AsyncIOEventEmitter): @@ -185,7 +186,11 @@ async def ws_read_loop(self): if self.__ping_id == msg.get("id"): self.__ping_future.set_result(None) self.__ping_future = None - if action in [ProtocolMessageAction.ATTACHED, ProtocolMessageAction.DETACHED]: + if action in ( + ProtocolMessageAction.ATTACHED, + ProtocolMessageAction.DETACHED, + ProtocolMessageAction.MESSAGE + ): self.ably.channels.on_channel_message(msg) @property diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 34976438..5819774b 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -1,6 +1,9 @@ import asyncio import logging +import types + from ably.realtime.connection import ConnectionState, ProtocolMessageAction +from ably.types.message import Message from ably.util.exceptions import AblyException from pyee.asyncio import AsyncIOEventEmitter from enum import Enum @@ -23,6 +26,8 @@ def __init__(self, realtime, name): self.__detach_future = None self.__realtime = realtime self.__state = ChannelState.INITIALIZED + self.__message_emitter = AsyncIOEventEmitter() + self.__all_messages_emitter = AsyncIOEventEmitter() super().__init__() async def attach(self): @@ -97,6 +102,35 @@ async def detach(self): await self.__detach_future self.set_state(ChannelState.DETACHED) + async def subscribe(self, *args): + if isinstance(args[0], str): + event = args[0] + listener = args[1] + elif isinstance(args[0], types.FunctionType) or asyncio.iscoroutinefunction(args[0]): + listener = args[0] + event = None + else: + raise ValueError('invalid subscribe arguments') + + if self.__realtime.connection.state == ConnectionState.CONNECTING: + await self.__realtime.connection.connect() + elif self.__realtime.connection.state != ConnectionState.CONNECTED: + raise AblyException( + 'Cannot subscribe to channel, invalid connection state: {self.__realtime.connection.state}', + 400, + 40000 + ) + + if self.state in (ChannelState.INITIALIZED, ChannelState.ATTACHING, ChannelState.DETACHED): + await self.attach() + + if event is not None: + self.__message_emitter.on(event, listener) + else: + self.__all_messages_emitter.on('message', listener) + + await self.attach() + def on_message(self, msg): action = msg.get('action') if action == ProtocolMessageAction.ATTACHED: @@ -107,6 +141,11 @@ def on_message(self, msg): if self.__detach_future: self.__detach_future.set_result(None) self.__detach_future = None + elif action == ProtocolMessageAction.MESSAGE: + messages = Message.from_encoded_array(msg.get('messages')) + for message in messages: + self.__message_emitter.emit(message.name, message) + self.__all_messages_emitter.emit('message', message) def set_state(self, state): self.__state = state From 0a26b114d1695367ccd97910e3f038f4a63454da Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 10 Oct 2022 01:44:56 +0100 Subject: [PATCH 183/481] test: add tests for RealtimeChannel.Subscribe --- test/ably/realtimechannel_test.py | 105 ++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py index 9f08736c..4fc55180 100644 --- a/test/ably/realtimechannel_test.py +++ b/test/ably/realtimechannel_test.py @@ -1,4 +1,8 @@ +import asyncio +from unittest.mock import Mock +import types from ably.realtime.realtime_channel import ChannelState +from ably.types.message import Message from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase @@ -38,3 +42,104 @@ async def test_channel_detach(self): await channel.detach() assert channel.state == ChannelState.DETACHED await ably.close() + + # RTL7b + async def test_subscribe(self): + ably = await RestSetup.get_ably_realtime() + + first_message_future = asyncio.Future() + second_message_future = asyncio.Future() + + def listener(message): + if not first_message_future.done(): + first_message_future.set_result(message) + else: + second_message_future.set_result(message) + + await ably.connect() + channel = ably.channels.get('my_channel') + await channel.attach() + await channel.subscribe('event', listener) + + # publish a message using rest client + rest = await RestSetup.get_ably_rest() + rest_channel = rest.channels.get('my_channel') + await rest_channel.publish('event', 'data') + message = await first_message_future + + assert isinstance(message, Message) + assert message.name == 'event' + assert message.data == 'data' + + # test that the listener is called again for further publishes + await rest_channel.publish('event', 'data') + await second_message_future + + await ably.close() + await rest.close() + + async def test_subscribe_coroutine(self): + ably = await RestSetup.get_ably_realtime() + await ably.connect() + channel = ably.channels.get('my_channel') + await channel.attach() + + message_future = asyncio.Future() + + # AsyncMock doesn't work in python 3.7 so use an actual coroutine + async def listener(msg): + message_future.set_result(msg) + + await channel.subscribe('event', listener) + + # publish a message using rest client + rest = await RestSetup.get_ably_rest() + rest_channel = rest.channels.get('my_channel') + await rest_channel.publish('event', 'data') + + message = await message_future + assert isinstance(message, Message) + assert message.name == 'event' + assert message.data == 'data' + + await ably.close() + await rest.close() + + # RTL7a + async def test_subscribe_all_events(self): + ably = await RestSetup.get_ably_realtime() + await ably.connect() + channel = ably.channels.get('my_channel') + await channel.attach() + + message_future = asyncio.Future() + listener = Mock(spec=types.FunctionType, side_effect=lambda msg: message_future.set_result(msg)) + await channel.subscribe(listener) + + # publish a message using rest client + rest = await RestSetup.get_ably_rest() + rest_channel = rest.channels.get('my_channel') + await rest_channel.publish('event', 'data') + message = await message_future + + listener.assert_called_once() + assert isinstance(message, Message) + assert message.name == 'event' + assert message.data == 'data' + + await ably.close() + await rest.close() + + # RTL7c + async def test_subscribe_auto_attach(self): + ably = await RestSetup.get_ably_realtime() + await ably.connect() + channel = ably.channels.get('my_channel') + assert channel.state == ChannelState.INITIALIZED + + listener = Mock() + await channel.subscribe('event', listener) + + assert channel.state == ChannelState.ATTACHED + + await ably.close() From ccdd2d1dfc91972718456f2712f0eee58b95eaf3 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 10 Oct 2022 01:55:55 +0100 Subject: [PATCH 184/481] feat: RealtimeChannel.unsubscribe --- ably/realtime/realtime_channel.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 5819774b..ad9bd224 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -131,6 +131,27 @@ async def subscribe(self, *args): await self.attach() + def unsubscribe(self, *args): + if len(args) == 0: + event = None + listener = None + elif isinstance(args[0], str): + event = args[0] + listener = args[1] + elif isinstance(args[0], types.FunctionType) or asyncio.iscoroutinefunction(args[0]): + listener = args[0] + event = None + else: + raise ValueError('invalid unsubscribe arguments') + + if listener is None: + self.__message_emitter.remove_all_listeners() + self.__all_messages_emitter.remove_all_listeners() + elif event is not None: + self.__message_emitter.remove_listener(event, listener) + else: + self.__all_messages_emitter.remove_listener('message', listener) + def on_message(self, msg): action = msg.get('action') if action == ProtocolMessageAction.ATTACHED: From 17059a90aed7a5a5cbc106cd31db9d8165fd0ce8 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 10 Oct 2022 01:56:10 +0100 Subject: [PATCH 185/481] test: add tests for RealtimeChannel.unsubscribe --- test/ably/realtimechannel_test.py | 57 +++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py index 4fc55180..4ee30357 100644 --- a/test/ably/realtimechannel_test.py +++ b/test/ably/realtimechannel_test.py @@ -143,3 +143,60 @@ async def test_subscribe_auto_attach(self): assert channel.state == ChannelState.ATTACHED await ably.close() + + # RTL8b + async def test_unsubscribe(self): + ably = await RestSetup.get_ably_realtime() + await ably.connect() + channel = ably.channels.get('my_channel') + await channel.attach() + + message_future = asyncio.Future() + listener = Mock(side_effect=lambda msg: message_future.set_result(msg)) + await channel.subscribe('event', listener) + + # publish a message using rest client + rest = await RestSetup.get_ably_rest() + rest_channel = rest.channels.get('my_channel') + await rest_channel.publish('event', 'data') + await message_future + listener.assert_called_once() + + # unsubscribe the listener from the channel + channel.unsubscribe('event', listener) + + # test that the listener is not called again for further publishes + await rest_channel.publish('event', 'data') + await asyncio.sleep(1) + assert listener.call_count == 1 + + await ably.close() + await rest.close() + + # RTL8c + async def test_unsubscribe_all(self): + ably = await RestSetup.get_ably_realtime() + await ably.connect() + channel = ably.channels.get('my_channel') + await channel.attach() + message_future = asyncio.Future() + listener = Mock(side_effect=lambda msg: message_future.set_result(msg)) + await channel.subscribe('event', listener) + + # publish a message using rest client + rest = await RestSetup.get_ably_rest() + rest_channel = rest.channels.get('my_channel') + await rest_channel.publish('event', 'data') + await message_future + listener.assert_called_once() + + # unsubscribe all listeners from the channel + channel.unsubscribe() + + # test that the listener is not called again for further publishes + await rest_channel.publish('event', 'data') + await asyncio.sleep(1) + assert listener.call_count == 1 + + await ably.close() + await rest.close() From 93911381e0bffa1d0734a3bca81a056bf0b750c5 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 12 Oct 2022 20:52:07 +0100 Subject: [PATCH 186/481] refactor: improve error messages for subscribe args --- ably/realtime/realtime_channel.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index ad9bd224..ed84ce32 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -105,6 +105,8 @@ async def detach(self): async def subscribe(self, *args): if isinstance(args[0], str): event = args[0] + if not args[1]: + raise ValueError("channel.subscribe called without listener") listener = args[1] elif isinstance(args[0], types.FunctionType) or asyncio.iscoroutinefunction(args[0]): listener = args[0] @@ -137,6 +139,8 @@ def unsubscribe(self, *args): listener = None elif isinstance(args[0], str): event = args[0] + if not args[1]: + raise ValueError("channel.unsubscribe called without listener") listener = args[1] elif isinstance(args[0], types.FunctionType) or asyncio.iscoroutinefunction(args[0]): listener = args[0] From 1fdcc28319b0b306e74ffedce1eed9b69e4a7e23 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 13 Oct 2022 14:28:24 +0100 Subject: [PATCH 187/481] refactor: extract is_function_or_coroutine helper --- ably/realtime/realtime_channel.py | 7 ++++--- ably/util/helper.py | 6 ++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index ed84ce32..44196484 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -1,6 +1,5 @@ import asyncio import logging -import types from ably.realtime.connection import ConnectionState, ProtocolMessageAction from ably.types.message import Message @@ -8,6 +7,8 @@ from pyee.asyncio import AsyncIOEventEmitter from enum import Enum +from ably.util.helper import is_function_or_coroutine + log = logging.getLogger(__name__) @@ -108,7 +109,7 @@ async def subscribe(self, *args): if not args[1]: raise ValueError("channel.subscribe called without listener") listener = args[1] - elif isinstance(args[0], types.FunctionType) or asyncio.iscoroutinefunction(args[0]): + elif is_function_or_coroutine(args[0]): listener = args[0] event = None else: @@ -142,7 +143,7 @@ def unsubscribe(self, *args): if not args[1]: raise ValueError("channel.unsubscribe called without listener") listener = args[1] - elif isinstance(args[0], types.FunctionType) or asyncio.iscoroutinefunction(args[0]): + elif is_function_or_coroutine(args[0]): listener = args[0] event = None else: diff --git a/ably/util/helper.py b/ably/util/helper.py index 0ca32ba1..c3b427ac 100644 --- a/ably/util/helper.py +++ b/ably/util/helper.py @@ -1,5 +1,7 @@ import random import string +import types +import asyncio def get_random_id(): @@ -7,3 +9,7 @@ def get_random_id(): source = string.ascii_letters + string.digits random_id = ''.join((random.choice(source) for i in range(8))) return random_id + + +def is_function_or_coroutine(value): + return isinstance(value, types.FunctionType) or asyncio.iscoroutinefunction(value) From b7966a695f1a99b0907d86be40f1f9300fd6416e Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 13 Oct 2022 14:42:53 +0100 Subject: [PATCH 188/481] refactor: validate subscribe/unsubscribe listener args --- ably/realtime/realtime_channel.py | 4 ++++ test/ably/realtimechannel_test.py | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 44196484..9d949d97 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -108,6 +108,8 @@ async def subscribe(self, *args): event = args[0] if not args[1]: raise ValueError("channel.subscribe called without listener") + if not is_function_or_coroutine(args[1]): + raise ValueError("subscribe listener must be function or coroutine function") listener = args[1] elif is_function_or_coroutine(args[0]): listener = args[0] @@ -142,6 +144,8 @@ def unsubscribe(self, *args): event = args[0] if not args[1]: raise ValueError("channel.unsubscribe called without listener") + if not is_function_or_coroutine(args[1]): + raise ValueError("unsubscribe listener must be a function or coroutine function") listener = args[1] elif is_function_or_coroutine(args[0]): listener = args[0] diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py index 4ee30357..d7acb215 100644 --- a/test/ably/realtimechannel_test.py +++ b/test/ably/realtimechannel_test.py @@ -137,7 +137,7 @@ async def test_subscribe_auto_attach(self): channel = ably.channels.get('my_channel') assert channel.state == ChannelState.INITIALIZED - listener = Mock() + listener = Mock(spec=types.FunctionType) await channel.subscribe('event', listener) assert channel.state == ChannelState.ATTACHED @@ -152,7 +152,7 @@ async def test_unsubscribe(self): await channel.attach() message_future = asyncio.Future() - listener = Mock(side_effect=lambda msg: message_future.set_result(msg)) + listener = Mock(spec=types.FunctionType, side_effect=lambda msg: message_future.set_result(msg)) await channel.subscribe('event', listener) # publish a message using rest client @@ -180,7 +180,7 @@ async def test_unsubscribe_all(self): channel = ably.channels.get('my_channel') await channel.attach() message_future = asyncio.Future() - listener = Mock(side_effect=lambda msg: message_future.set_result(msg)) + listener = Mock(spec=types.FunctionType, side_effect=lambda msg: message_future.set_result(msg)) await channel.subscribe('event', listener) # publish a message using rest client From 8f29bb9c7830c6db663b48549238176f2997246a Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 12 Oct 2022 22:52:31 +0100 Subject: [PATCH 189/481] feat: ConnectionStateChange fixes: #320 --- ably/realtime/connection.py | 22 ++++++++++++++++------ test/ably/realtimeconnection_test.py | 2 +- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index de267164..fae9e200 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -9,6 +9,8 @@ from pyee.asyncio import AsyncIOEventEmitter from datetime import datetime from ably.util import helper +from dataclasses import dataclass +from typing import Optional log = logging.getLogger(__name__) @@ -22,6 +24,13 @@ class ConnectionState(Enum): FAILED = 'failed' +@dataclass +class ConnectionStateChange: + previous: ConnectionState + current: ConnectionState + reason: Optional[AblyException] = None + + class ProtocolMessageAction(IntEnum): HEARTBEAT = 0 CONNECTED = 4 @@ -52,9 +61,9 @@ async def close(self): async def ping(self): return await self.__connection_manager.ping() - def on_state_update(self, state): - self.__state = state - self.__realtime.options.loop.call_soon(functools.partial(self.emit, state)) + def on_state_update(self, state_change): + self.__state = state_change.current + self.__realtime.options.loop.call_soon(functools.partial(self.emit, state_change.current, state_change)) @property def state(self): @@ -81,9 +90,10 @@ def __init__(self, realtime, initial_state): self.__ping_future = None super().__init__() - def enact_state_change(self, state): + def enact_state_change(self, state, reason=None): + current_state = self.__state self.__state = state - self.emit('connectionstate', state) + self.emit('connectionstate', ConnectionStateChange(current_state, state, reason)) async def connect(self): if self.__state == ConnectionState.CONNECTED: @@ -167,8 +177,8 @@ async def ws_read_loop(self): if action == ProtocolMessageAction.ERROR: # ERROR error = msg["error"] if error['nonfatal'] is False: - self.enact_state_change(ConnectionState.FAILED) exception = AblyAuthException(error["message"], error["statusCode"], error["code"]) + self.enact_state_change(ConnectionState.FAILED, exception) if self.__connected_future: self.__connected_future.set_exception(exception) self.__connected_future = None diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index ec6980f1..220836d3 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -82,7 +82,7 @@ async def test_connection_ping_closed(self): async def test_auto_connect(self): ably = await RestSetup.get_ably_realtime() connect_future = asyncio.Future() - ably.connection.on(ConnectionState.CONNECTED, lambda: connect_future.set_result(None)) + ably.connection.on(ConnectionState.CONNECTED, lambda change: connect_future.set_result(change)) await connect_future assert ably.connection.state == ConnectionState.CONNECTED await ably.close() From 28c164111d44f2af7c3295bd4bca400dcdd8d117 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 12 Oct 2022 22:53:11 +0100 Subject: [PATCH 190/481] test: add tests for ConnectionStateChange --- test/ably/realtimeconnection_test.py | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 220836d3..41ab1d5d 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -86,3 +86,39 @@ async def test_auto_connect(self): await connect_future assert ably.connection.state == ConnectionState.CONNECTED await ably.close() + + async def test_connection_state_change(self): + ably = await RestSetup.get_ably_realtime() + + connected_future = asyncio.Future() + + def on_state_change(change): + connected_future.set_result(change) + + ably.connection.on(ConnectionState.CONNECTED, on_state_change) + + state_change = await connected_future + assert state_change.previous == ConnectionState.CONNECTING + assert state_change.current == ConnectionState.CONNECTED + await ably.close() + + async def test_connection_state_change_reason(self): + ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) + + failed_changes = [] + + def on_state_change(change): + failed_changes.append(change) + + ably.connection.on(ConnectionState.FAILED, on_state_change) + + with pytest.raises(AblyAuthException) as exception: + await ably.connect() + + assert len(failed_changes) == 1 + state_change = failed_changes[0] + assert state_change is not None + assert state_change.previous == ConnectionState.CONNECTING + assert state_change.current == ConnectionState.FAILED + assert state_change.reason == exception.value + await ably.close() From ee6038e135b96f94c9875f33df59c0c99e7a08ee Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 18 Oct 2022 10:48:33 +0100 Subject: [PATCH 191/481] refine public api --- ably/realtime/connection.py | 154 +++++++++++++++++++++++++++++- ably/realtime/realtime.py | 127 ++++++++++++++++++++++-- ably/realtime/realtime_channel.py | 112 +++++++++++++++++++++- 3 files changed, 377 insertions(+), 16 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index fae9e200..aef43646 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -45,7 +45,40 @@ class ProtocolMessageAction(IntEnum): class Connection(AsyncIOEventEmitter): + """Ably Realtime Connection + + Enables the management of a connection to Ably + + Attributes + ---------- + realtime: any + Realtime client + state: str + Connection state + connection_manager: ConnectionManager + Connection manager + + + Methods + ------- + connect() + Establishes a realtime connection + close() + Closes a realtime connection + ping() + Pings a realtime connection + on_state_update(state_change) + Update and emit current state + """ + def __init__(self, realtime): + """Constructs a Connection object. + + Parameters + ---------- + realtime: any + Ably realtime client + """ self.__realtime = realtime self.__state = ConnectionState.CONNECTING if realtime.options.auto_connect else ConnectionState.INITIALIZED self.__connection_manager = ConnectionManager(realtime, self.state) @@ -53,33 +86,95 @@ def __init__(self, realtime): super().__init__() async def connect(self): + """Establishes a realtime connection. + + Causes the connection to open, entering the connecting state + """ await self.__connection_manager.connect() async def close(self): + """Causes the connection to close, entering the closing state. + + Once closed, the library will not attempt to re-establish the + connection without an explicit call to connect() + """ await self.__connection_manager.close() async def ping(self): + """ + Send a ping to the realtime connection + """ return await self.__connection_manager.ping() def on_state_update(self, state_change): + """Update and emit the connection state + """ self.__state = state_change.current self.__realtime.options.loop.call_soon(functools.partial(self.emit, state_change.current, state_change)) @property def state(self): + """Returns connection state""" return self.__state @state.setter def state(self, value): + """Sets connection state""" self.__state = value @property def connection_manager(self): + """Returns connection manager""" return self.__connection_manager class ConnectionManager(AsyncIOEventEmitter): + """Ably Realtime Connection + + Attributes + ---------- + realtime: any + Ably realtime client + initial_state: str + Initial connection state + ably: any + Ably object + state: str + Connection state + + + Methods + ------- + enact_state_change(state, reason=None) + Set new state + connect() + Establishes a realtime connection + close() + Closes a realtime connection + ping() + Pings a realtime connection + connect_impl() + Send a connection to ably websocket + send_close_message() + Send a close protocol message to ably + send_protocol_message(protocol_message) + Send protocol message to ably + setup_ws() + Set up ably websocket connection + ws_read_loop() + Handle response from ably websocket + """ + def __init__(self, realtime, initial_state): + """Constructs a Connection object. + + Parameters + ---------- + realtime: any + Ably realtime client + initial_state: any + Initial connection state + """ self.options = realtime.options self.__ably = realtime self.__state = initial_state @@ -91,11 +186,26 @@ def __init__(self, realtime, initial_state): super().__init__() def enact_state_change(self, state, reason=None): + """Sets new connection state + + Parameters + ---------- + state: any + The current connection state + reason: AblyException, optional + Error object describing the last error received if a connection failure occurs + """ current_state = self.__state self.__state = state self.emit('connectionstate', ConnectionStateChange(current_state, state, reason)) async def connect(self): + """Establishes a realtime connection. + + Explicitly calling connect() is unnecessary unless the autoConnect attribute of the ClientOptions object + is false. Unless already connected or connecting, this method causes the connection to open, entering the + CONNECTING state. + """ if self.__state == ConnectionState.CONNECTED: return @@ -111,6 +221,11 @@ async def connect(self): await self.connect_impl() async def close(self): + """Causes the connection to close, entering the closing state. + + Once closed, the library will not attempt to re-establish the + connection without an explicit call to connect() + """ if self.__state != ConnectionState.CONNECTED: log.warn('Connection.closed called while connection state not connected') self.enact_state_change(ConnectionState.CLOSING) @@ -125,17 +240,27 @@ async def close(self): await self.setup_ws_task async def connect_impl(self): + """Send a connection to ably websocket """ self.setup_ws_task = self.ably.options.loop.create_task(self.setup_ws()) await self.__connected_future self.enact_state_change(ConnectionState.CONNECTED) async def send_close_message(self): - await self.sendProtocolMessage({"action": ProtocolMessageAction.CLOSE}) + """Send a close protocol message to ably""" + await self.send_protocol_message({"action": ProtocolMessageAction.CLOSE}) - async def sendProtocolMessage(self, protocolMessage): - await self.__websocket.send(json.dumps(protocolMessage)) + async def send_protocol_message(self, protocol_message): + """Send protocol message to ably""" + await self.__websocket.send(json.dumps(protocol_message)) async def setup_ws(self): + """Set up ably websocket connection + + Raises + ------ + AblyAuthException + If connection cannot be established + """ headers = HttpUtils.default_headers() async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}', extra_headers=headers) as websocket: @@ -147,6 +272,22 @@ async def setup_ws(self): return async def ping(self): + """Send a ping to the realtime connection + + When connected, sends a heartbeat ping to the Ably server and executes + the callback with any error and the response time in milliseconds when + a heartbeat ping request is echoed from the server. + + Raises + ------ + AblyException + If ping request cannot be sent due to invalid state + + Returns + ------- + float + The response time in milliseconds + """ if self.__ping_future: response = await self.__ping_future return response @@ -155,8 +296,8 @@ async def ping(self): if self.__state in [ConnectionState.CONNECTED, ConnectionState.CONNECTING]: self.__ping_id = helper.get_random_id() ping_start_time = datetime.now().timestamp() - await self.sendProtocolMessage({"action": ProtocolMessageAction.HEARTBEAT, - "id": self.__ping_id}) + await self.send_protocol_message({"action": ProtocolMessageAction.HEARTBEAT, + "id": self.__ping_id}) else: raise AblyException("Cannot send ping request. Calling ping in invalid state", 40000, 400) ping_end_time = datetime.now().timestamp() @@ -164,6 +305,7 @@ async def ping(self): return round(response_time_ms, 2) async def ws_read_loop(self): + """Handle response from ably websocket""" while True: raw = await self.__websocket.recv() msg = json.loads(raw) @@ -205,8 +347,10 @@ async def ws_read_loop(self): @property def ably(self): + """Returns ably client""" return self.__ably @property def state(self): + """Returns channel state""" return self.__state diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 35f711c0..10bdf518 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -10,16 +10,49 @@ class AblyRealtime: - """Ably Realtime Client""" + """ + Ably Realtime Client + + Attributes + ---------- + key: str + A valid ably key string + loop: AbstractEventLoop + asyncio running event loop + auth: Auth + authentication object + options: Options + auth options + connection: Connection + realtime connection object + channels: Channels + realtime channel object + + Methods + ------- + connect() + Establishes a realtime connection + close() + Closes a realtime connection + ping() + Pings a realtime connection + """ def __init__(self, key=None, loop=None, **kwargs): - """Create an AblyRealtime instance. - - :Parameters: - **Credentials** - - `key`: a valid ably key string + """Constructs a RealtimeClient object using an Ably API key or token string. + + Parameters + ---------- + key: str + A valid ably key string + loop: AbstractEventLoop, optional + asyncio running event loop + + Raises + ------ + ValueError + If no authentication key is not provided """ - if loop is None: try: loop = asyncio.get_running_loop() @@ -41,49 +74,125 @@ def __init__(self, key=None, loop=None, **kwargs): asyncio.ensure_future(self.connection.connection_manager.connect_impl()) async def connect(self): + """Establishes a realtime connection. + + Explicitly calling connect() is unnecessary unless the autoConnect attribute of the ClientOptions object + is false. Unless already connected or connecting, this method causes the connection to open, entering the + CONNECTING state. + """ await self.connection.connect() async def close(self): + """Causes the connection to close, entering the closing state. + + Once closed, the library will not attempt to re-establish the + connection without an explicit call to connect() + """ await self.connection.close() async def ping(self): + """Send a ping to the realtime connection + + When connected, sends a heartbeat ping to the Ably server and executes + the callback with any error and the response time in milliseconds when + a heartbeat ping request is echoed from the server. + + Returns + ------- + float + The response time in milliseconds + """ return await self.connection.ping() @property def auth(self): + """Returns the auth object""" return self.__auth @property def options(self): + """Returns the auth options object""" return self.__options @property def connection(self): - """Establish realtime connection""" + """Returns the realtime connection object""" return self.__connection @property def channels(self): + """Returns the realtime channel object""" return self.__channels class Channels: + """ + Establish ably realtime channel + + Attributes + ---------- + realtime: any + Ably realtime client object + + Methods + ------- + get(name) + Gets a channel + release(name) + Releases a channel + on_channel_message(msg) + Receives message on a channel + """ + def __init__(self, realtime): + """Initial a realtime channel using the realtime object + + Parameters + ---------- + realtime: any + Ably realtime client object + """ self.all = {} self.__realtime = realtime def get(self, name): + """Creates a new RealtimeChannel object, or returns the existing channel object. + + Parameters + ---------- + + name: str + Channel name + """ if not self.all.get(name): self.all[name] = RealtimeChannel(self.__realtime, name) return self.all[name] def release(self, name): + """Releases a RealtimeChannel object, deleting it, and enabling it to be garbage collected + + It also removes any listeners associated with the channel. + To release a channel, the channel state must be INITIALIZED, DETACHED, or FAILED. + + + Parameters + ---------- + name: str + Channel name + """ if not self.all.get(name): return del self.all[name] def on_channel_message(self, msg): + """Receives message on a realtime channel + + Parameters + ---------- + msg: str + Channel message to receive + """ channel = self.all.get(msg.get('channel')) if not channel: - log.warning('Channel message recieved but no channel instance found') + log.warning('Channel message received but no channel instance found') channel.on_message(msg) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 9d949d97..07ba9611 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -21,7 +21,46 @@ class ChannelState(Enum): class RealtimeChannel(AsyncIOEventEmitter): + """ + Ably Realtime Channel + + Attributes + ---------- + realtime: any + Ably realtime client + name: str + Channel name + state: str + Channel state + + Methods + ------- + attach() + Attach to channel + detach() + Detach from channel + subscribe(*args) + Subscribe to a channel + unsubscribe() + Unsubscribe from a channel + on_message(msg) + Emit channel message + set_state(state) + Set channel state + """ + def __init__(self, realtime, name): + """Constructs a Realtime channel object. + + Parameters + ---------- + realtime: any + Ably realtime client + name: str + Channel name + state: str + Channel state + """ self.__name = name self.__attach_future = None self.__detach_future = None @@ -32,6 +71,16 @@ def __init__(self, realtime, name): super().__init__() async def attach(self): + """Attach to channel + + Attach to this channel ensuring the channel is created in the Ably system and all messages published + on the channel are received by any channel listeners registered using subscribe + + Raises + ------ + AblyException + If unable to attach channel + """ # RTL4a - if channel is attached do nothing if self.state == ChannelState.ATTACHED: return @@ -58,7 +107,7 @@ async def attach(self): await self.__realtime.connect() self.__attach_future = asyncio.Future() - await self.__realtime.connection.connection_manager.sendProtocolMessage( + await self.__realtime.connection.connection_manager.send_protocol_message( { "action": ProtocolMessageAction.ATTACH, "channel": self.name, @@ -68,6 +117,17 @@ async def attach(self): self.set_state(ChannelState.ATTACHED) async def detach(self): + """Detach from channel + + Any resulting channel state change is emitted to any listeners registered + Once all clients globally have detached from the channel, the channel will be released + in the Ably service within two minutes. + + Raises + ------ + AblyException + If unable to detach channel + """ # RTL5g - raise exception if state invalid if self.__realtime.connection.state in [ConnectionState.CLOSING, ConnectionState.FAILED]: raise AblyException( @@ -94,7 +154,7 @@ async def detach(self): await self.__realtime.connect() self.__detach_future = asyncio.Future() - await self.__realtime.connection.connection_manager.sendProtocolMessage( + await self.__realtime.connection.connection_manager.send_protocol_message( { "action": ProtocolMessageAction.DETACH, "channel": self.name, @@ -104,6 +164,22 @@ async def detach(self): self.set_state(ChannelState.DETACHED) async def subscribe(self, *args): + """Subscribe to a channel + + Registers a listener for messages on the channel. + + Parameters + ---------- + *args: event, listener, optional + Subscribe event and listener + + Raises + ------ + AblyException + If unable to subscribe to a channel due to invalid connection state + ValueError + If no valid subscribe arguments are passed + """ if isinstance(args[0], str): event = args[0] if not args[1]: @@ -137,6 +213,22 @@ async def subscribe(self, *args): await self.attach() def unsubscribe(self, *args): + """Unsubscribe from a channel + + Deregister the given listener for (for any/all event names). + This removes an earlier event-specific subscription. + + Parameters + ---------- + *args: event, listener, optional + Subscribe event and listener + + Raises + ------ + ValueError + If no valid unsubscribe arguments are passed, no listener or listener is not a function + or coroutine + """ if len(args) == 0: event = None listener = None @@ -162,6 +254,13 @@ def unsubscribe(self, *args): self.__all_messages_emitter.remove_listener('message', listener) def on_message(self, msg): + """Emit channel message + + Parameters + ---------- + msg: str + Channel message + """ action = msg.get('action') if action == ProtocolMessageAction.ATTACHED: if self.__attach_future: @@ -178,13 +277,22 @@ def on_message(self, msg): self.__all_messages_emitter.emit('message', message) def set_state(self, state): + """Set channel state + + Parameters + ---------- + state: str + New channel state + """ self.__state = state self.emit(state) @property def name(self): + """Returns channel name""" return self.__name @property def state(self): + """Returns channel state""" return self.__state From f50bd7f968de3a675c96fe5bb3171fdeacf182e6 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 18 Oct 2022 14:08:31 +0100 Subject: [PATCH 192/481] undocument internal apis --- ably/realtime/connection.py | 105 ++---------------------------------- ably/realtime/realtime.py | 5 ++ 2 files changed, 8 insertions(+), 102 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index aef43646..f985ce30 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -67,8 +67,6 @@ class Connection(AsyncIOEventEmitter): Closes a realtime connection ping() Pings a realtime connection - on_state_update(state_change) - Update and emit current state """ def __init__(self, realtime): @@ -82,7 +80,7 @@ def __init__(self, realtime): self.__realtime = realtime self.__state = ConnectionState.CONNECTING if realtime.options.auto_connect else ConnectionState.INITIALIZED self.__connection_manager = ConnectionManager(realtime, self.state) - self.__connection_manager.on('connectionstate', self.on_state_update) + self.__connection_manager.on('connectionstate', self.__on_state_update) super().__init__() async def connect(self): @@ -106,15 +104,13 @@ async def ping(self): """ return await self.__connection_manager.ping() - def on_state_update(self, state_change): - """Update and emit the connection state - """ + def __on_state_update(self, state_change): self.__state = state_change.current self.__realtime.options.loop.call_soon(functools.partial(self.emit, state_change.current, state_change)) @property def state(self): - """Returns connection state""" + """The current Channel state of the channel""" return self.__state @state.setter @@ -124,57 +120,11 @@ def state(self, value): @property def connection_manager(self): - """Returns connection manager""" return self.__connection_manager class ConnectionManager(AsyncIOEventEmitter): - """Ably Realtime Connection - - Attributes - ---------- - realtime: any - Ably realtime client - initial_state: str - Initial connection state - ably: any - Ably object - state: str - Connection state - - - Methods - ------- - enact_state_change(state, reason=None) - Set new state - connect() - Establishes a realtime connection - close() - Closes a realtime connection - ping() - Pings a realtime connection - connect_impl() - Send a connection to ably websocket - send_close_message() - Send a close protocol message to ably - send_protocol_message(protocol_message) - Send protocol message to ably - setup_ws() - Set up ably websocket connection - ws_read_loop() - Handle response from ably websocket - """ - def __init__(self, realtime, initial_state): - """Constructs a Connection object. - - Parameters - ---------- - realtime: any - Ably realtime client - initial_state: any - Initial connection state - """ self.options = realtime.options self.__ably = realtime self.__state = initial_state @@ -186,26 +136,11 @@ def __init__(self, realtime, initial_state): super().__init__() def enact_state_change(self, state, reason=None): - """Sets new connection state - - Parameters - ---------- - state: any - The current connection state - reason: AblyException, optional - Error object describing the last error received if a connection failure occurs - """ current_state = self.__state self.__state = state self.emit('connectionstate', ConnectionStateChange(current_state, state, reason)) async def connect(self): - """Establishes a realtime connection. - - Explicitly calling connect() is unnecessary unless the autoConnect attribute of the ClientOptions object - is false. Unless already connected or connecting, this method causes the connection to open, entering the - CONNECTING state. - """ if self.__state == ConnectionState.CONNECTED: return @@ -221,11 +156,6 @@ async def connect(self): await self.connect_impl() async def close(self): - """Causes the connection to close, entering the closing state. - - Once closed, the library will not attempt to re-establish the - connection without an explicit call to connect() - """ if self.__state != ConnectionState.CONNECTED: log.warn('Connection.closed called while connection state not connected') self.enact_state_change(ConnectionState.CLOSING) @@ -240,27 +170,17 @@ async def close(self): await self.setup_ws_task async def connect_impl(self): - """Send a connection to ably websocket """ self.setup_ws_task = self.ably.options.loop.create_task(self.setup_ws()) await self.__connected_future self.enact_state_change(ConnectionState.CONNECTED) async def send_close_message(self): - """Send a close protocol message to ably""" await self.send_protocol_message({"action": ProtocolMessageAction.CLOSE}) async def send_protocol_message(self, protocol_message): - """Send protocol message to ably""" await self.__websocket.send(json.dumps(protocol_message)) async def setup_ws(self): - """Set up ably websocket connection - - Raises - ------ - AblyAuthException - If connection cannot be established - """ headers = HttpUtils.default_headers() async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}', extra_headers=headers) as websocket: @@ -272,22 +192,6 @@ async def setup_ws(self): return async def ping(self): - """Send a ping to the realtime connection - - When connected, sends a heartbeat ping to the Ably server and executes - the callback with any error and the response time in milliseconds when - a heartbeat ping request is echoed from the server. - - Raises - ------ - AblyException - If ping request cannot be sent due to invalid state - - Returns - ------- - float - The response time in milliseconds - """ if self.__ping_future: response = await self.__ping_future return response @@ -305,7 +209,6 @@ async def ping(self): return round(response_time_ms, 2) async def ws_read_loop(self): - """Handle response from ably websocket""" while True: raw = await self.__websocket.recv() msg = json.loads(raw) @@ -347,10 +250,8 @@ async def ws_read_loop(self): @property def ably(self): - """Returns ably client""" return self.__ably @property def state(self): - """Returns channel state""" return self.__state diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 10bdf518..16b8dd17 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -97,6 +97,11 @@ async def ping(self): the callback with any error and the response time in milliseconds when a heartbeat ping request is echoed from the server. + Raises + ------ + AblyException + If ping request cannot be sent due to invalid state + Returns ------- float From 97d97c71f8becd1d4dff25df298a27a772ec1555 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Thu, 20 Oct 2022 10:07:50 +0100 Subject: [PATCH 193/481] remove documentation on private methods --- ably/realtime/connection.py | 16 ++-------- ably/realtime/realtime.py | 30 +++--------------- ably/realtime/realtime_channel.py | 51 +++++++++++-------------------- 3 files changed, 24 insertions(+), 73 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index f985ce30..b820ed5f 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -51,12 +51,8 @@ class Connection(AsyncIOEventEmitter): Attributes ---------- - realtime: any - Realtime client state: str Connection state - connection_manager: ConnectionManager - Connection manager Methods @@ -70,13 +66,6 @@ class Connection(AsyncIOEventEmitter): """ def __init__(self, realtime): - """Constructs a Connection object. - - Parameters - ---------- - realtime: any - Ably realtime client - """ self.__realtime = realtime self.__state = ConnectionState.CONNECTING if realtime.options.auto_connect else ConnectionState.INITIALIZED self.__connection_manager = ConnectionManager(realtime, self.state) @@ -110,12 +99,11 @@ def __on_state_update(self, state_change): @property def state(self): - """The current Channel state of the channel""" + """The current connection state of the connection""" return self.__state @state.setter def state(self, value): - """Sets connection state""" self.__state = value @property @@ -246,7 +234,7 @@ async def ws_read_loop(self): ProtocolMessageAction.DETACHED, ProtocolMessageAction.MESSAGE ): - self.ably.channels.on_channel_message(msg) + self.ably.channels._on_channel_message(msg) @property def ably(self): diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 16b8dd17..db77222f 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -39,7 +39,7 @@ class AblyRealtime: """ def __init__(self, key=None, loop=None, **kwargs): - """Constructs a RealtimeClient object using an Ably API key or token string. + """Constructs a RealtimeClient object using an Ably API key. Parameters ---------- @@ -131,13 +131,7 @@ def channels(self): class Channels: - """ - Establish ably realtime channel - - Attributes - ---------- - realtime: any - Ably realtime client object + """Creates and destroys RealtimeChannel objects. Methods ------- @@ -145,18 +139,9 @@ class Channels: Gets a channel release(name) Releases a channel - on_channel_message(msg) - Receives message on a channel """ def __init__(self, realtime): - """Initial a realtime channel using the realtime object - - Parameters - ---------- - realtime: any - Ably realtime client object - """ self.all = {} self.__realtime = realtime @@ -189,15 +174,8 @@ def release(self, name): return del self.all[name] - def on_channel_message(self, msg): - """Receives message on a realtime channel - - Parameters - ---------- - msg: str - Channel message to receive - """ + def _on_channel_message(self, msg): channel = self.all.get(msg.get('channel')) if not channel: log.warning('Channel message received but no channel instance found') - channel.on_message(msg) + channel._on_message(msg) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 07ba9611..8d40eed2 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -26,8 +26,6 @@ class RealtimeChannel(AsyncIOEventEmitter): Attributes ---------- - realtime: any - Ably realtime client name: str Channel name state: str @@ -43,24 +41,9 @@ class RealtimeChannel(AsyncIOEventEmitter): Subscribe to a channel unsubscribe() Unsubscribe from a channel - on_message(msg) - Emit channel message - set_state(state) - Set channel state """ def __init__(self, realtime, name): - """Constructs a Realtime channel object. - - Parameters - ---------- - realtime: any - Ably realtime client - name: str - Channel name - state: str - Channel state - """ self.__name = name self.__attach_future = None self.__detach_future = None @@ -167,12 +150,22 @@ async def subscribe(self, *args): """Subscribe to a channel Registers a listener for messages on the channel. + The caller supplies a listener function, which is called + each time one or more messages arrives on the channel. + + The function resolves once the channel is attached. Parameters ---------- *args: event, listener, optional Subscribe event and listener + arg1(event): str + Subscribe to messages with the given event name + + arg2(listener): any + Subscribe to all messages on the channel + Raises ------ AblyException @@ -221,7 +214,13 @@ def unsubscribe(self, *args): Parameters ---------- *args: event, listener, optional - Subscribe event and listener + Unsubscribe event and listener + + arg1(event): str + Unsubscribe to messages with the given event name + + arg2(listener): any + Unsubscribe to all messages on the channel Raises ------ @@ -253,14 +252,7 @@ def unsubscribe(self, *args): else: self.__all_messages_emitter.remove_listener('message', listener) - def on_message(self, msg): - """Emit channel message - - Parameters - ---------- - msg: str - Channel message - """ + def _on_message(self, msg): action = msg.get('action') if action == ProtocolMessageAction.ATTACHED: if self.__attach_future: @@ -277,13 +269,6 @@ def on_message(self, msg): self.__all_messages_emitter.emit('message', message) def set_state(self, state): - """Set channel state - - Parameters - ---------- - state: str - New channel state - """ self.__state = state self.emit(state) From c258a0937f88d1ea8e1cb139a36a9696f7cb64af Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Thu, 20 Oct 2022 13:39:24 +0100 Subject: [PATCH 194/481] review: update docstring documentation --- ably/realtime/connection.py | 31 ++++++++++++++++++++-------- ably/realtime/realtime.py | 31 ++++------------------------ ably/realtime/realtime_channel.py | 22 ++++++++++++-------- test/ably/realtimeconnection_test.py | 8 +++---- 4 files changed, 43 insertions(+), 49 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index b820ed5f..bf3ffe22 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -68,8 +68,8 @@ class Connection(AsyncIOEventEmitter): def __init__(self, realtime): self.__realtime = realtime self.__state = ConnectionState.CONNECTING if realtime.options.auto_connect else ConnectionState.INITIALIZED - self.__connection_manager = ConnectionManager(realtime, self.state) - self.__connection_manager.on('connectionstate', self.__on_state_update) + self.__connection_manager = ConnectionManager(self.__realtime, self.state) + self.__connection_manager.on('connectionstate', self._on_state_update) super().__init__() async def connect(self): @@ -88,12 +88,25 @@ async def close(self): await self.__connection_manager.close() async def ping(self): - """ - Send a ping to the realtime connection + """Send a ping to the realtime connection + + When connected, sends a heartbeat ping to the Ably server and executes + the callback with any error and the response time in milliseconds when + a heartbeat ping request is echoed from the server. + + Raises + ------ + AblyException + If ping request cannot be sent due to invalid state + + Returns + ------- + float + The response time in milliseconds """ return await self.__connection_manager.ping() - def __on_state_update(self, state_change): + def _on_state_update(self, state_change): self.__state = state_change.current self.__realtime.options.loop.call_soon(functools.partial(self.emit, state_change.current, state_change)) @@ -158,7 +171,7 @@ async def close(self): await self.setup_ws_task async def connect_impl(self): - self.setup_ws_task = self.ably.options.loop.create_task(self.setup_ws()) + self.setup_ws_task = self.__ably.options.loop.create_task(self.setup_ws()) await self.__connected_future self.enact_state_change(ConnectionState.CONNECTED) @@ -170,10 +183,10 @@ async def send_protocol_message(self, protocol_message): async def setup_ws(self): headers = HttpUtils.default_headers() - async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.ably.key}', + async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.__ably.key}', extra_headers=headers) as websocket: self.__websocket = websocket - task = self.ably.options.loop.create_task(self.ws_read_loop()) + task = self.__ably.options.loop.create_task(self.ws_read_loop()) try: await task except AblyAuthException: @@ -234,7 +247,7 @@ async def ws_read_loop(self): ProtocolMessageAction.DETACHED, ProtocolMessageAction.MESSAGE ): - self.ably.channels._on_channel_message(msg) + self.__ably.channels._on_channel_message(msg) @property def ably(self): diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index db77222f..5ddc2e1e 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -15,14 +15,12 @@ class AblyRealtime: Attributes ---------- - key: str - A valid ably key string loop: AbstractEventLoop asyncio running event loop auth: Auth authentication object options: Options - auth options + auth options object connection: Connection realtime connection object channels: Channels @@ -31,11 +29,9 @@ class AblyRealtime: Methods ------- connect() - Establishes a realtime connection + Establishes the realtime connection close() - Closes a realtime connection - ping() - Pings a realtime connection + Closes the realtime connection """ def __init__(self, key=None, loop=None, **kwargs): @@ -44,7 +40,7 @@ def __init__(self, key=None, loop=None, **kwargs): Parameters ---------- key: str - A valid ably key string + A valid ably API key string loop: AbstractEventLoop, optional asyncio running event loop @@ -90,25 +86,6 @@ async def close(self): """ await self.connection.close() - async def ping(self): - """Send a ping to the realtime connection - - When connected, sends a heartbeat ping to the Ably server and executes - the callback with any error and the response time in milliseconds when - a heartbeat ping request is echoed from the server. - - Raises - ------ - AblyException - If ping request cannot be sent due to invalid state - - Returns - ------- - float - The response time in milliseconds - """ - return await self.connection.ping() - @property def auth(self): """Returns the auth object""" diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 8d40eed2..34c01770 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -38,9 +38,9 @@ class RealtimeChannel(AsyncIOEventEmitter): detach() Detach from channel subscribe(*args) - Subscribe to a channel - unsubscribe() - Unsubscribe from a channel + Subscribe to messages on a channel + unsubscribe(*args) + Unsubscribe to messages from a channel """ def __init__(self, realtime, name): @@ -157,15 +157,17 @@ async def subscribe(self, *args): Parameters ---------- - *args: event, listener, optional + *args: event, listener Subscribe event and listener - arg1(event): str + arg1(event): str, optional Subscribe to messages with the given event name - arg2(listener): any + arg2(listener): callable Subscribe to all messages on the channel + When no event is provided, arg1 is used as the listener. + Raises ------ AblyException @@ -213,15 +215,17 @@ def unsubscribe(self, *args): Parameters ---------- - *args: event, listener, optional + *args: event, listener Unsubscribe event and listener - arg1(event): str + arg1(event): str, optional Unsubscribe to messages with the given event name - arg2(listener): any + arg2(listener): callable Unsubscribe to all messages on the channel + When no event is provided, arg1 is used as the listener. + Raises ------ ValueError diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 41ab1d5d..72647a31 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -45,7 +45,7 @@ async def test_auth_invalid_key(self): async def test_connection_ping_connected(self): ably = await RestSetup.get_ably_realtime() await ably.connect() - response_time_ms = await ably.ping() + response_time_ms = await ably.connection.ping() assert response_time_ms is not None assert type(response_time_ms) is float await ably.close() @@ -54,7 +54,7 @@ async def test_connection_ping_initialized(self): ably = await RestSetup.get_ably_realtime(auto_connect=False) assert ably.connection.state == ConnectionState.INITIALIZED with pytest.raises(AblyException) as exception: - await ably.ping() + await ably.connection.ping() assert exception.value.code == 400 assert exception.value.status_code == 40000 @@ -64,7 +64,7 @@ async def test_connection_ping_failed(self): await ably.connect() assert ably.connection.state == ConnectionState.FAILED with pytest.raises(AblyException) as exception: - await ably.ping() + await ably.connection.ping() assert exception.value.code == 400 assert exception.value.status_code == 40000 await ably.close() @@ -75,7 +75,7 @@ async def test_connection_ping_closed(self): assert ably.connection.state == ConnectionState.CONNECTED await ably.close() with pytest.raises(AblyException) as exception: - await ably.ping() + await ably.connection.ping() assert exception.value.code == 400 assert exception.value.status_code == 40000 From e5408af68a21073c9199443f9820292e8da99387 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 20 Oct 2022 18:07:55 +0100 Subject: [PATCH 195/481] docs: add param description for auto_connect --- ably/realtime/realtime.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 5ddc2e1e..87276053 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -43,6 +43,10 @@ def __init__(self, key=None, loop=None, **kwargs): A valid ably API key string loop: AbstractEventLoop, optional asyncio running event loop + auto_connect: bool + When true, the client connects to Ably as soon as it is instantiated. + You can set this to false and explicitly connect to Ably using the + connect() method. The default is true. Raises ------ From 22c90cc855723cef79ffb533910d2a4d8852b29f Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 20 Oct 2022 10:14:09 +0100 Subject: [PATCH 196/481] chore: improve logging in realtime.py --- ably/realtime/realtime.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 87276053..1b9bfe4f 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -64,6 +64,8 @@ def __init__(self, key=None, loop=None, **kwargs): else: raise ValueError("Key is missing. Provide an API key.") + log.info(f'Realtime client initialised with options: {vars(options)}') + self.__auth = Auth(self, options) self.__options = options self.key = key @@ -80,14 +82,15 @@ async def connect(self): is false. Unless already connected or connecting, this method causes the connection to open, entering the CONNECTING state. """ + log.info('Realtime.connect() called') await self.connection.connect() async def close(self): """Causes the connection to close, entering the closing state. - Once closed, the library will not attempt to re-establish the connection without an explicit call to connect() """ + log.info('Realtime.close() called') await self.connection.close() @property @@ -156,7 +159,20 @@ def release(self, name): del self.all[name] def _on_channel_message(self, msg): + channel_name = msg.get('channel') + if not channel_name: + log.error( + 'Channels.on_channel_message()', + f'received event without channel, action = {msg.get("action")}' + ) + return + channel = self.all.get(msg.get('channel')) if not channel: - log.warning('Channel message received but no channel instance found') + log.warning( + 'Channels.on_channel_message()', + f'receieved event for non-existent channel: {channel_name}' + ) + return + channel._on_message(msg) From 2734a535bb7d0106eb445be5545392d1212d2567 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 20 Oct 2022 10:21:21 +0100 Subject: [PATCH 197/481] chore: add detailed logging to connection.py --- ably/realtime/connection.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index bf3ffe22..6ea4db8d 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -107,6 +107,7 @@ async def ping(self): return await self.__connection_manager.ping() def _on_state_update(self, state_change): + log.info(f'Connection state changing from {self.state} to {state_change.current}') self.__state = state_change.current self.__realtime.options.loop.call_soon(functools.partial(self.emit, state_change.current, state_change)) @@ -158,14 +159,14 @@ async def connect(self): async def close(self): if self.__state != ConnectionState.CONNECTED: - log.warn('Connection.closed called while connection state not connected') + log.warning('Connection.closed called while connection state not connected') self.enact_state_change(ConnectionState.CLOSING) self.__closed_future = asyncio.Future() if self.__websocket and self.__state != ConnectionState.FAILED: await self.send_close_message() await self.__closed_future else: - log.warn('Connection.closed called while connection already closed or not established') + log.warning('Connection.closed called while connection already closed or not established') self.enact_state_change(ConnectionState.CLOSED) if self.setup_ws_task: await self.setup_ws_task @@ -178,13 +179,17 @@ async def connect_impl(self): async def send_close_message(self): await self.send_protocol_message({"action": ProtocolMessageAction.CLOSE}) - async def send_protocol_message(self, protocol_message): - await self.__websocket.send(json.dumps(protocol_message)) + async def send_protocol_message(self, protocolMessage): + raw_msg = json.dumps(protocolMessage) + log.info('send_protocol_message(): sending {raw_msg}') + await self.__websocket.send(raw_msg) async def setup_ws(self): headers = HttpUtils.default_headers() - async with websockets.connect(f'wss://{self.options.realtime_host}?key={self.__ably.key}', - extra_headers=headers) as websocket: + ws_url = f'wss://{self.options.realtime_host}?key={self.__ably.key}' + log.info(f'setup_ws(): attempting to connect to {ws_url}') + async with websockets.connect(ws_url, extra_headers=headers) as websocket: + log.info(f'setup_ws(): connection established to {ws_url}') self.__websocket = websocket task = self.__ably.options.loop.create_task(self.ws_read_loop()) try: @@ -213,6 +218,7 @@ async def ws_read_loop(self): while True: raw = await self.__websocket.recv() msg = json.loads(raw) + log.info(f'ws_read_loop(): receieved protocol message: {msg}') action = msg['action'] if action == ProtocolMessageAction.CONNECTED: # CONNECTED if self.__connected_future: From 25fabfd6888ae639ad364d1e2d818578a4494218 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 20 Oct 2022 10:26:03 +0100 Subject: [PATCH 198/481] chore: add detailed logging to realtime_channel.py --- ably/realtime/realtime_channel.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 34c01770..12d2bc95 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -64,6 +64,9 @@ async def attach(self): AblyException If unable to attach channel """ + + log.info(f'RealtimeChannel.attach() called, channel = {self.name}') + # RTL4a - if channel is attached do nothing if self.state == ChannelState.ATTACHED: return @@ -111,6 +114,9 @@ async def detach(self): AblyException If unable to detach channel """ + + log.info(f'RealtimeChannel.detach() called, channel = {self.name}') + # RTL5g - raise exception if state invalid if self.__realtime.connection.state in [ConnectionState.CLOSING, ConnectionState.FAILED]: raise AblyException( @@ -188,6 +194,8 @@ async def subscribe(self, *args): else: raise ValueError('invalid subscribe arguments') + log.info(f'RealtimeChannel.subscribe called, channel = {self.name}, event = {event}') + if self.__realtime.connection.state == ConnectionState.CONNECTING: await self.__realtime.connection.connect() elif self.__realtime.connection.state != ConnectionState.CONNECTED: @@ -248,6 +256,8 @@ def unsubscribe(self, *args): else: raise ValueError('invalid unsubscribe arguments') + log.info(f'RealtimeChannel.unsubscribe called, channel = {self.name}, event = {event}') + if listener is None: self.__message_emitter.remove_all_listeners() self.__all_messages_emitter.remove_all_listeners() From d2863cf877e772b6aa7ec73513adc3a906cc9a55 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 24 Oct 2022 09:19:02 +0100 Subject: [PATCH 199/481] update readme with realtime doc --- README.md | 72 ++++++++++++------------------------------------------- 1 file changed, 15 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index e6132b80..ee5ae041 100644 --- a/README.md +++ b/README.md @@ -54,10 +54,11 @@ introduced by version 1.2.0. ## Usage +### Using the Rest API + All examples assume a client and/or channel has been created in one of the following ways: With closing the client manually: - ```python from ably import AblyRest @@ -196,56 +197,31 @@ await client.time() await client.close() ``` -## Realtime client (beta) - -We currently have a preview version of our first ever Python realtime client available for beta testing. -Currently the realtime client only supports authentication using basic auth and message subscription. -Realtime publishing, token authentication, and realtime presence are upcoming but not yet supported. -Check out the [roadmap](./roadmap.md) to see our plan for the realtime client. - -### Installing the realtime client - -The beta realtime client is available as a [PyPI](https://pypi.org/project/ably/2.0.0b2/) package. - -``` -pip install ably==2.0.0b2 -``` - -### Using the realtime client - +### Using the Realtime API +The python realtime API currently only supports authentication with ably API key. #### Creating a client - ```python from ably import AblyRealtime async def main(): client = AblyRealtime('api:key') + channel = client.channels.get('channel_name) ``` -#### Get a realtime channel instance - -```python -channel = client.channels.get('channel_name') -``` - -#### Subscribing to messages on a channel - +#### Subscribing to a channel for event ```python +message_future = asyncio.Future() def listener(message): - print(message.data) + message_future.set_result(message) -# Subscribe to messages with the 'event' name -await channel.subscribe('event', listener) +channel.subscribe('event', listener) -# Subscribe to all messages on a channel +# Subscribe using only listener await channel.subscribe(listener) ``` -Note that `channel.subscribe` is a coroutine function and will resolve when the channel is attached - -#### Unsubscribing from messages on a channel - +#### Unsubscribing from a channel for event ```python # unsubscribe the listener from the channel channel.unsubscribe('event', listener) @@ -254,33 +230,16 @@ channel.unsubscribe('event', listener) channel.unsubscribe() ``` -#### Subscribe to connection state change - -```python -# subscribe to 'failed' connection state -client.connection.on('failed', listener) - -# subscribe to 'connected' connection state -client.connection.on('connected', listener) - -# subscribe to all connection state changes -client.connection.on(listener) -``` - -#### Attach to a channel - +#### Attach a channel ```python await channel.attach() ``` - #### Detach from a channel - ```python await channel.detach() ``` #### Managing a connection - ```python # Establish a realtime connection. # Explicitly calling connect() is unnecessary unless the autoConnect attribute of the ClientOptions object is false @@ -289,10 +248,9 @@ await client.connect() # Close a connection await client.close() -# Send a ping -time_in_ms = await client.connection.ping() +# Ping a connection +await client.connection.ping() ``` - ## Resources Visit https://ably.com/docs for a complete API reference and more examples. @@ -307,7 +265,7 @@ for the set of versions that currently undergo CI testing. ## Known Limitations -Currently, this SDK only supports [Ably REST](https://ably.com/docs/rest). +Currently, this SDK only supports [Ably REST](https://ably.com/docs/rest) and subscribe/unsubscribe functionality of [Ably Realtime](https://ably.com/docs/realtime) as documented above. However, you can use the [MQTT adapter](https://ably.com/docs/mqtt) to implement [Ably's Realtime](https://ably.com/docs/realtime) features using Python. See [our roadmap for this SDK](roadmap.md) for more information. From 6b7cecdd57820ec4b63dbb473781753dddfba137 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 25 Oct 2022 10:52:51 +0100 Subject: [PATCH 200/481] review: update readme --- README.md | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index ee5ae041..df8ee190 100644 --- a/README.md +++ b/README.md @@ -205,23 +205,27 @@ from ably import AblyRealtime async def main(): client = AblyRealtime('api:key') - channel = client.channels.get('channel_name) ``` -#### Subscribing to a channel for event +#### Connecting to a channel +```python +channel = client.channels.get('channel_name) +``` +#### Subscribing to messages on a channel ```python -message_future = asyncio.Future() def listener(message): - message_future.set_result(message) + print(message.data) -channel.subscribe('event', listener) +# Subscribe to messages with the 'event' name +await channel.subscribe('event', listener) -# Subscribe using only listener +# Subscribe to all messages on a channel await channel.subscribe(listener) ``` +Note that `channel.subscribe` is a coroutine function and will resolve when the channel is attached -#### Unsubscribing from a channel for event +#### Unsubscribing from messages on a channel ```python # unsubscribe the listener from the channel channel.unsubscribe('event', listener) @@ -230,7 +234,7 @@ channel.unsubscribe('event', listener) channel.unsubscribe() ``` -#### Attach a channel +#### Attach to a channel ```python await channel.attach() ``` @@ -248,8 +252,8 @@ await client.connect() # Close a connection await client.close() -# Ping a connection -await client.connection.ping() +# Send a ping +time_in_ms = await client.connection.ping() ``` ## Resources @@ -265,7 +269,7 @@ for the set of versions that currently undergo CI testing. ## Known Limitations -Currently, this SDK only supports [Ably REST](https://ably.com/docs/rest) and subscribe/unsubscribe functionality of [Ably Realtime](https://ably.com/docs/realtime) as documented above. +Currently, this SDK only supports [Ably REST](https://ably.com/docs/rest) and realtime message subscription as documented above. However, you can use the [MQTT adapter](https://ably.com/docs/mqtt) to implement [Ably's Realtime](https://ably.com/docs/realtime) features using Python. See [our roadmap for this SDK](roadmap.md) for more information. From f1a063ad6523a8f7a6d6ed19a7b9c706a709d950 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 31 Oct 2022 12:00:58 +0000 Subject: [PATCH 201/481] add info on connection state --- README.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index df8ee190..60e5aa32 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,7 @@ await client.close() ``` ### Using the Realtime API -The python realtime API currently only supports authentication with ably API key. +The python realtime client currently only supports basic authentication. #### Creating a client ```python from ably import AblyRealtime @@ -207,9 +207,9 @@ async def main(): client = AblyRealtime('api:key') ``` -#### Connecting to a channel +#### Get a realtime channel instance ```python -channel = client.channels.get('channel_name) +channel = client.channels.get('channel_name') ``` #### Subscribing to messages on a channel ```python @@ -234,6 +234,16 @@ channel.unsubscribe('event', listener) channel.unsubscribe() ``` +#### Subscribe to connection state change +```python +from ably.realtime.connection import ConnectionState +# subscribe to failed connection state +client.connection.on(ConnectionState.FAILED, listener) + +# subscribe to connected connection state +client.connection.on(ConnectionState.CONNECTED, listener) +``` + #### Attach to a channel ```python await channel.attach() From 8342adf8f1302e29176c3cd5b7bb1b241f0b6d69 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 1 Nov 2022 12:38:39 +0000 Subject: [PATCH 202/481] refactor: use string-based enums --- ably/realtime/connection.py | 2 +- ably/realtime/realtime_channel.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 6ea4db8d..2c923439 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -15,7 +15,7 @@ log = logging.getLogger(__name__) -class ConnectionState(Enum): +class ConnectionState(str, Enum): INITIALIZED = 'initialized' CONNECTING = 'connecting' CONNECTED = 'connected' diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 12d2bc95..c60fb6fd 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -12,7 +12,7 @@ log = logging.getLogger(__name__) -class ChannelState(Enum): +class ChannelState(str, Enum): INITIALIZED = 'initialized' ATTACHING = 'attaching' ATTACHED = 'attached' From a650bcd18a4f53a844ef13e5cf0af50e73f181d9 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 1 Nov 2022 12:39:08 +0000 Subject: [PATCH 203/481] doc: use string-based enums in usage examples --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 60e5aa32..7479203a 100644 --- a/README.md +++ b/README.md @@ -236,12 +236,11 @@ channel.unsubscribe() #### Subscribe to connection state change ```python -from ably.realtime.connection import ConnectionState -# subscribe to failed connection state -client.connection.on(ConnectionState.FAILED, listener) +# subscribe to 'failed' connection state +client.connection.on('failed', listener) -# subscribe to connected connection state -client.connection.on(ConnectionState.CONNECTED, listener) +# subscribe to 'connected' connection state +client.connection.on('connected', listener) ``` #### Attach to a channel From 8b6d9efd619a8c50f72dd61303a39b5f4916d2ab Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 24 Oct 2022 14:31:04 +0100 Subject: [PATCH 204/481] add environment client option --- test/ably/restsetup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ably/restsetup.py b/test/ably/restsetup.py index efab592d..5cd73c1d 100644 --- a/test/ably/restsetup.py +++ b/test/ably/restsetup.py @@ -15,7 +15,7 @@ tls = (os.environ.get('ABLY_TLS') or "true").lower() == "true" host = os.environ.get('ABLY_HOST', 'sandbox-rest.ably.io') -realtime_host = 'sandbox-realtime.ably.io' +realtime_host = os.environ.get('ABLY_HOST', 'sandbox-realtime.ably.io') environment = os.environ.get('ABLY_ENV') port = 80 From 0f1ae2dee2651fc55f00eae41e0720d3f13e0dfb Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 25 Oct 2022 14:16:54 +0100 Subject: [PATCH 205/481] add environment option --- ably/types/options.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ably/types/options.py b/ably/types/options.py index 6d254440..00ea4de3 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -30,8 +30,10 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, from ably import api_version idempotent_rest_publishing = api_version >= '1.2' + if environment is None: + environment = Defaults.environment if realtime_host is None: - realtime_host = Defaults.realtime_host + realtime_host = f'{environment}-{Defaults.realtime_host}' self.__client_id = client_id self.__log_level = log_level From ad39e7d818fd9e23a342d2c3284c28982191e9a5 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 1 Nov 2022 13:56:17 +0000 Subject: [PATCH 206/481] refactor realtime host option --- ably/types/options.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/ably/types/options.py b/ably/types/options.py index 00ea4de3..861833ba 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -30,11 +30,6 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, from ably import api_version idempotent_rest_publishing = api_version >= '1.2' - if environment is None: - environment = Defaults.environment - if realtime_host is None: - realtime_host = f'{environment}-{Defaults.realtime_host}' - self.__client_id = client_id self.__log_level = log_level self.__tls = tls @@ -58,6 +53,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, self.__auto_connect = auto_connect self.__rest_hosts = self.__get_rest_hosts() + self.__realtime_hosts = self.__get_realtime_hosts() @property def client_id(self): @@ -255,11 +251,22 @@ def __get_rest_hosts(self): hosts = hosts[:http_max_retry_count] return hosts + def __get_realtime_hosts(self): + if self.realtime_host is not None: + return self.realtime_host + elif self.environment is not None: + return f'{self.environment}-{Defaults.realtime_host}' + else: + return Defaults.realtime_host + def get_rest_hosts(self): return self.__rest_hosts def get_rest_host(self): return self.__rest_hosts[0] + def get_realtime_host(self): + return self.__realtime_hosts + def get_fallback_rest_hosts(self): return self.__rest_hosts[1:] From 8f22560a123011c17427eb2bccd6b5d0094b2189 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 1 Nov 2022 19:10:22 +0000 Subject: [PATCH 207/481] update API documentation with client option --- ably/realtime/realtime.py | 5 +++++ ably/types/options.py | 3 +++ 2 files changed, 8 insertions(+) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 1b9bfe4f..5563a317 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -47,6 +47,11 @@ def __init__(self, key=None, loop=None, **kwargs): When true, the client connects to Ably as soon as it is instantiated. You can set this to false and explicitly connect to Ably using the connect() method. The default is true. + **kwargs: client options + realtime_host: str + The host to connect to. Defaults to `realtime.ably.io` + environment: str + The environment to use. Defaults to `production` Raises ------ diff --git a/ably/types/options.py b/ably/types/options.py index 861833ba..d5f8cc4f 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -26,6 +26,9 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, if environment is not None and rest_host is not None: raise ValueError('specify rest_host or environment, not both') + if environment is not None and realtime_host is not None: + raise ValueError('specify realtime_host or environment, not both') + if idempotent_rest_publishing is None: from ably import api_version idempotent_rest_publishing = api_version >= '1.2' From 757810c3dbcd45e441537582fa79732833aa0738 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 2 Nov 2022 10:45:56 +0000 Subject: [PATCH 208/481] update realtime API docstring --- ably/realtime/realtime.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 5563a317..7b46ec67 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -49,9 +49,10 @@ def __init__(self, key=None, loop=None, **kwargs): connect() method. The default is true. **kwargs: client options realtime_host: str - The host to connect to. Defaults to `realtime.ably.io` + Enables a non-default Ably host to be specified for realtime connections. + For development environments only. The default value is realtime.ably.io. environment: str - The environment to use. Defaults to `production` + Enables a custom environment to be used with the Ably service. Defaults to `production` Raises ------ From 910b09fc55ab3f16b0e4fbbc340184331258ff2c Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 1 Nov 2022 12:18:59 +0000 Subject: [PATCH 209/481] feat: EventEmitter methods with no event argument --- ably/realtime/connection.py | 10 +++--- ably/realtime/realtime_channel.py | 31 ++++++++---------- ably/util/eventemitter.py | 54 +++++++++++++++++++++++++++++++ ably/util/helper.py | 6 ++-- test/ably/eventemitter_test.py | 42 ++++++++++++++++-------- test/ably/realtimechannel_test.py | 37 ++++++++++++++------- 6 files changed, 130 insertions(+), 50 deletions(-) create mode 100644 ably/util/eventemitter.py diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 2c923439..1e194c89 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -5,8 +5,8 @@ import json from ably.http.httputils import HttpUtils from ably.util.exceptions import AblyAuthException, AblyException +from ably.util.eventemitter import EventEmitter from enum import Enum, IntEnum -from pyee.asyncio import AsyncIOEventEmitter from datetime import datetime from ably.util import helper from dataclasses import dataclass @@ -44,7 +44,7 @@ class ProtocolMessageAction(IntEnum): MESSAGE = 15 -class Connection(AsyncIOEventEmitter): +class Connection(EventEmitter): """Ably Realtime Connection Enables the management of a connection to Ably @@ -109,7 +109,7 @@ async def ping(self): def _on_state_update(self, state_change): log.info(f'Connection state changing from {self.state} to {state_change.current}') self.__state = state_change.current - self.__realtime.options.loop.call_soon(functools.partial(self.emit, state_change.current, state_change)) + self.__realtime.options.loop.call_soon(functools.partial(self._emit, state_change.current, state_change)) @property def state(self): @@ -125,7 +125,7 @@ def connection_manager(self): return self.__connection_manager -class ConnectionManager(AsyncIOEventEmitter): +class ConnectionManager(EventEmitter): def __init__(self, realtime, initial_state): self.options = realtime.options self.__ably = realtime @@ -140,7 +140,7 @@ def __init__(self, realtime, initial_state): def enact_state_change(self, state, reason=None): current_state = self.__state self.__state = state - self.emit('connectionstate', ConnectionStateChange(current_state, state, reason)) + self._emit('connectionstate', ConnectionStateChange(current_state, state, reason)) async def connect(self): if self.__state == ConnectionState.CONNECTED: diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index c60fb6fd..9a431be8 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -3,11 +3,11 @@ from ably.realtime.connection import ConnectionState, ProtocolMessageAction from ably.types.message import Message +from ably.util.eventemitter import EventEmitter from ably.util.exceptions import AblyException -from pyee.asyncio import AsyncIOEventEmitter from enum import Enum -from ably.util.helper import is_function_or_coroutine +from ably.util.helper import is_callable_or_coroutine log = logging.getLogger(__name__) @@ -20,7 +20,7 @@ class ChannelState(str, Enum): DETACHED = 'detached' -class RealtimeChannel(AsyncIOEventEmitter): +class RealtimeChannel(EventEmitter): """ Ably Realtime Channel @@ -49,8 +49,7 @@ def __init__(self, realtime, name): self.__detach_future = None self.__realtime = realtime self.__state = ChannelState.INITIALIZED - self.__message_emitter = AsyncIOEventEmitter() - self.__all_messages_emitter = AsyncIOEventEmitter() + self.__message_emitter = EventEmitter() super().__init__() async def attach(self): @@ -185,10 +184,10 @@ async def subscribe(self, *args): event = args[0] if not args[1]: raise ValueError("channel.subscribe called without listener") - if not is_function_or_coroutine(args[1]): + if not is_callable_or_coroutine(args[1]): raise ValueError("subscribe listener must be function or coroutine function") listener = args[1] - elif is_function_or_coroutine(args[0]): + elif is_callable_or_coroutine(args[0]): listener = args[0] event = None else: @@ -211,7 +210,7 @@ async def subscribe(self, *args): if event is not None: self.__message_emitter.on(event, listener) else: - self.__all_messages_emitter.on('message', listener) + self.__message_emitter.on(listener) await self.attach() @@ -247,10 +246,10 @@ def unsubscribe(self, *args): event = args[0] if not args[1]: raise ValueError("channel.unsubscribe called without listener") - if not is_function_or_coroutine(args[1]): + if not is_callable_or_coroutine(args[1]): raise ValueError("unsubscribe listener must be a function or coroutine function") listener = args[1] - elif is_function_or_coroutine(args[0]): + elif is_callable_or_coroutine(args[0]): listener = args[0] event = None else: @@ -259,12 +258,11 @@ def unsubscribe(self, *args): log.info(f'RealtimeChannel.unsubscribe called, channel = {self.name}, event = {event}') if listener is None: - self.__message_emitter.remove_all_listeners() - self.__all_messages_emitter.remove_all_listeners() + self.__message_emitter.off() elif event is not None: - self.__message_emitter.remove_listener(event, listener) + self.__message_emitter.off(event, listener) else: - self.__all_messages_emitter.remove_listener('message', listener) + self.__message_emitter.off(listener) def _on_message(self, msg): action = msg.get('action') @@ -279,12 +277,11 @@ def _on_message(self, msg): elif action == ProtocolMessageAction.MESSAGE: messages = Message.from_encoded_array(msg.get('messages')) for message in messages: - self.__message_emitter.emit(message.name, message) - self.__all_messages_emitter.emit('message', message) + self.__message_emitter._emit(message.name, message) def set_state(self, state): self.__state = state - self.emit(state) + self._emit(state) @property def name(self): diff --git a/ably/util/eventemitter.py b/ably/util/eventemitter.py new file mode 100644 index 00000000..f688ef71 --- /dev/null +++ b/ably/util/eventemitter.py @@ -0,0 +1,54 @@ +from pyee.asyncio import AsyncIOEventEmitter + +from ably.util.helper import is_callable_or_coroutine + +# pyee's event emitter doesn't support attaching a listener to all events +# so to patch it, we create a wrapper which uses two event emitters, one +# is used to listen to all events and this arbitrary string is the event name +# used to emit all events on that listener +_all_event = 'all' + + +def _is_named_event_args(*args): + return len(args) == 2 and is_callable_or_coroutine(args[1]) + + +def _is_all_event_args(*args): + return len(args) == 1 and is_callable_or_coroutine(args[0]) + + +class EventEmitter: + def __init__(self): + self.__named_event_emitter = AsyncIOEventEmitter() + self.__all_event_emitter = AsyncIOEventEmitter() + + def on(self, *args): + if _is_all_event_args(*args): + self.__all_event_emitter.add_listener(_all_event, args[0]) + elif _is_named_event_args(*args): + self.__named_event_emitter.add_listener(args[0], args[1]) + else: + raise ValueError("EventEmitter.on(): invalid args") + + def once(self, *args): + if _is_all_event_args(*args): + self.__all_event_emitter.once(_all_event, args[0]) + elif _is_named_event_args(*args): + self.__named_event_emitter.once(args[0], args[1]) + else: + raise ValueError("EventEmitter.once(): invalid args") + + def off(self, *args): + if len(args) == 0: + self.__all_event_emitter.remove_all_listeners() + self.__named_event_emitter.remove_all_listeners() + elif _is_all_event_args(*args): + self.__all_event_emitter.remove_listener(_all_event, args[0]) + elif _is_named_event_args(*args): + self.__named_event_emitter.remove_listener(args[0], args[1]) + else: + raise ValueError("EventEmitter.once(): invalid args") + + def _emit(self, *args): + self.__named_event_emitter.emit(*args) + self.__all_event_emitter.emit(_all_event, *args[1:]) diff --git a/ably/util/helper.py b/ably/util/helper.py index c3b427ac..cead99d9 100644 --- a/ably/util/helper.py +++ b/ably/util/helper.py @@ -1,6 +1,6 @@ +import inspect import random import string -import types import asyncio @@ -11,5 +11,5 @@ def get_random_id(): return random_id -def is_function_or_coroutine(value): - return isinstance(value, types.FunctionType) or asyncio.iscoroutinefunction(value) +def is_callable_or_coroutine(value): + return asyncio.iscoroutinefunction(value) or inspect.isfunction(value) or inspect.ismethod(value) diff --git a/test/ably/eventemitter_test.py b/test/ably/eventemitter_test.py index d57f046a..deda7626 100644 --- a/test/ably/eventemitter_test.py +++ b/test/ably/eventemitter_test.py @@ -1,6 +1,5 @@ import asyncio from ably.realtime.connection import ConnectionState -from unittest.mock import Mock from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase @@ -11,41 +10,56 @@ async def setUp(self): async def test_connection_events(self): realtime = await RestSetup.get_ably_realtime() - listener = Mock() - realtime.connection.add_listener(ConnectionState.CONNECTED, listener) + call_count = 0 + + def listener(_): + nonlocal call_count + call_count += 1 + + realtime.connection.on(ConnectionState.CONNECTED, listener) await realtime.connect() # Listener is only called once event loop is free - listener.assert_not_called() + assert call_count == 0 await asyncio.sleep(0) - listener.assert_called_once() + assert call_count == 1 await realtime.close() async def test_event_listener_error(self): realtime = await RestSetup.get_ably_realtime() - listener = Mock() + call_count = 0 + + def listener(_): + nonlocal call_count + call_count += 1 + raise Exception() # If a listener throws an exception it should not propagate (#RTE6) listener.side_effect = Exception() - realtime.connection.add_listener(ConnectionState.CONNECTED, listener) + realtime.connection.on(ConnectionState.CONNECTED, listener) await realtime.connect() - listener.assert_not_called() + assert call_count == 0 await asyncio.sleep(0) - listener.assert_called_once() + assert call_count == 1 await realtime.close() async def test_event_emitter_off(self): realtime = await RestSetup.get_ably_realtime() - listener = Mock() - realtime.connection.add_listener(ConnectionState.CONNECTED, listener) - realtime.connection.remove_listener(ConnectionState.CONNECTED, listener) + call_count = 0 + + def listener(_): + nonlocal call_count + call_count += 1 + + realtime.connection.on(ConnectionState.CONNECTED, listener) + realtime.connection.off(ConnectionState.CONNECTED, listener) await realtime.connect() - listener.assert_not_called() + assert call_count == 0 await asyncio.sleep(0) - listener.assert_not_called() + assert call_count == 0 await realtime.close() diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py index d7acb215..90072e9d 100644 --- a/test/ably/realtimechannel_test.py +++ b/test/ably/realtimechannel_test.py @@ -1,6 +1,4 @@ import asyncio -from unittest.mock import Mock -import types from ably.realtime.realtime_channel import ChannelState from ably.types.message import Message from test.ably.restsetup import RestSetup @@ -113,7 +111,10 @@ async def test_subscribe_all_events(self): await channel.attach() message_future = asyncio.Future() - listener = Mock(spec=types.FunctionType, side_effect=lambda msg: message_future.set_result(msg)) + + def listener(msg): + message_future.set_result(msg) + await channel.subscribe(listener) # publish a message using rest client @@ -122,7 +123,6 @@ async def test_subscribe_all_events(self): await rest_channel.publish('event', 'data') message = await message_future - listener.assert_called_once() assert isinstance(message, Message) assert message.name == 'event' assert message.data == 'data' @@ -137,7 +137,9 @@ async def test_subscribe_auto_attach(self): channel = ably.channels.get('my_channel') assert channel.state == ChannelState.INITIALIZED - listener = Mock(spec=types.FunctionType) + def listener(_): + pass + await channel.subscribe('event', listener) assert channel.state == ChannelState.ATTACHED @@ -152,7 +154,13 @@ async def test_unsubscribe(self): await channel.attach() message_future = asyncio.Future() - listener = Mock(spec=types.FunctionType, side_effect=lambda msg: message_future.set_result(msg)) + call_count = 0 + + def listener(msg): + nonlocal call_count + call_count += 1 + message_future.set_result(msg) + await channel.subscribe('event', listener) # publish a message using rest client @@ -160,7 +168,7 @@ async def test_unsubscribe(self): rest_channel = rest.channels.get('my_channel') await rest_channel.publish('event', 'data') await message_future - listener.assert_called_once() + assert call_count == 1 # unsubscribe the listener from the channel channel.unsubscribe('event', listener) @@ -168,7 +176,7 @@ async def test_unsubscribe(self): # test that the listener is not called again for further publishes await rest_channel.publish('event', 'data') await asyncio.sleep(1) - assert listener.call_count == 1 + assert call_count == 1 await ably.close() await rest.close() @@ -179,8 +187,15 @@ async def test_unsubscribe_all(self): await ably.connect() channel = ably.channels.get('my_channel') await channel.attach() + message_future = asyncio.Future() - listener = Mock(spec=types.FunctionType, side_effect=lambda msg: message_future.set_result(msg)) + call_count = 0 + + def listener(msg): + nonlocal call_count + call_count += 1 + message_future.set_result(msg) + await channel.subscribe('event', listener) # publish a message using rest client @@ -188,7 +203,7 @@ async def test_unsubscribe_all(self): rest_channel = rest.channels.get('my_channel') await rest_channel.publish('event', 'data') await message_future - listener.assert_called_once() + assert call_count == 1 # unsubscribe all listeners from the channel channel.unsubscribe() @@ -196,7 +211,7 @@ async def test_unsubscribe_all(self): # test that the listener is not called again for further publishes await rest_channel.publish('event', 'data') await asyncio.sleep(1) - assert listener.call_count == 1 + assert call_count == 1 await ably.close() await rest.close() From e2aec263ddfd717d7e455405bae83f6fdcf2e0ef Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 1 Nov 2022 12:32:12 +0000 Subject: [PATCH 210/481] doc: add docstrings for patched EventEmitter --- ably/util/eventemitter.py | 51 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/ably/util/eventemitter.py b/ably/util/eventemitter.py index f688ef71..6e737719 100644 --- a/ably/util/eventemitter.py +++ b/ably/util/eventemitter.py @@ -18,11 +18,37 @@ def _is_all_event_args(*args): class EventEmitter: + """ + A generic interface for event registration and delivery used in a number of the types in the Realtime client + library. For example, the Connection object emits events for connection state using the EventEmitter pattern. + + Methods + ------- + on(*args) + Attach to channel + once(*args) + Detach from channel + off() + Subscribe to messages on a channel + """ def __init__(self): self.__named_event_emitter = AsyncIOEventEmitter() self.__all_event_emitter = AsyncIOEventEmitter() def on(self, *args): + """ + Registers the provided listener for the specified event, if provided, and otherwise for all events. + If on() is called more than once with the same listener and event, the listener is added multiple times to + its listener registry. Therefore, as an example, assuming the same listener is registered twice using + on(), and an event is emitted once, the listener would be invoked twice. + + Parameters + ---------- + name : str + The named event to listen for. + listener : callable + The event listener. + """ if _is_all_event_args(*args): self.__all_event_emitter.add_listener(_all_event, args[0]) elif _is_named_event_args(*args): @@ -31,6 +57,20 @@ def on(self, *args): raise ValueError("EventEmitter.on(): invalid args") def once(self, *args): + """ + Registers the provided listener for the first event that is emitted. If once() is called more than once + with the same listener, the listener is added multiple times to its listener registry. Therefore, as an + example, assuming the same listener is registered twice using once(), and an event is emitted once, the + listener would be invoked twice. However, all subsequent events emitted would not invoke the listener as + once() ensures that each registration is only invoked once. + + Parameters + ---------- + name : str + The named event to listen for. + listener : callable + The event listener. + """ if _is_all_event_args(*args): self.__all_event_emitter.once(_all_event, args[0]) elif _is_named_event_args(*args): @@ -39,6 +79,17 @@ def once(self, *args): raise ValueError("EventEmitter.once(): invalid args") def off(self, *args): + """ + Removes all registrations that match both the specified listener and, if provided, the specified event. + If called with no arguments, deregisters all registrations, for all events and listeners. + + Parameters + ---------- + name : str + The named event to listen for. + listener : callable + The event listener. + """ if len(args) == 0: self.__all_event_emitter.remove_all_listeners() self.__named_event_emitter.remove_all_listeners() From 5ed49229b307239bdeeec136713a72369073b479 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 1 Nov 2022 12:47:00 +0000 Subject: [PATCH 211/481] doc: add usage example for listening to all connection state changes --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 7479203a..919b3331 100644 --- a/README.md +++ b/README.md @@ -241,6 +241,9 @@ client.connection.on('failed', listener) # subscribe to 'connected' connection state client.connection.on('connected', listener) + +# subscribe to all connection state changes +client.connection.on(listener) ``` #### Attach to a channel From d1290ebb9224bda1122f729e5ab821a4018c7560 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 2 Nov 2022 14:33:01 +0000 Subject: [PATCH 212/481] chore: bump version number for 2.0.0-beta.1 release --- ably/__init__.py | 2 +- pyproject.toml | 2 +- test/ably/resthttp_test.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ably/__init__.py b/ably/__init__.py index 5e05eca1..ed9c6e09 100644 --- a/ably/__init__.py +++ b/ably/__init__.py @@ -15,4 +15,4 @@ logger.addHandler(logging.NullHandler()) api_version = '1.2' -lib_version = '1.2.2' +lib_version = '2.0.0-beta.1' diff --git a/pyproject.toml b/pyproject.toml index b56ab615..3231aa0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "ably" -version = "1.2.2" +version = "2.0.0-beta.1" description = "Python REST client library SDK for Ably realtime messaging service" license = "Apache-2.0" authors = ["Ably <support@ably.com>"] diff --git a/test/ably/resthttp_test.py b/test/ably/resthttp_test.py index 7ac80015..ed9db26c 100644 --- a/test/ably/resthttp_test.py +++ b/test/ably/resthttp_test.py @@ -204,7 +204,7 @@ async def test_request_headers(self): # Agent assert 'Ably-Agent' in r.request.headers - expr = r"^ably-python\/\d.\d.\d python\/\d.\d+.\d+$" + expr = r"^ably-python\/\d.\d.\d(-beta\.\d)? python\/\d.\d+.\d+$" assert re.search(expr, r.request.headers['Ably-Agent']) await ably.close() From 989c4feee5c22c39fedc9942c3a639f64eee52b4 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 2 Nov 2022 14:40:27 +0000 Subject: [PATCH 213/481] chore: update CHANGELOG for 2.0.0-beta.1 release --- CHANGELOG.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4929727..87265643 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Change Log +## [v2.0.0-beta.1](https://github.com/ably/ably-python/tree/v2.0.0-beta.1) + +[Full Changelog](https://github.com/ably/ably-python/compare/v1.2.1...2.0.0-beta.1) + +- Create Basic Api Key connection [\#311](https://github.com/ably/ably-python/pull/311) +- Send Ably-Agent header in realtime connection [\#314](https://github.com/ably/ably-python/pull/314) +- Close client service [\#315](https://github.com/ably/ably-python/pull/315) +- Implement EventEmitter interface on Connection [\#316](https://github.com/ably/ably-python/pull/316) +- Finish tasks gracefully on failed connection [\#317](https://github.com/ably/ably-python/pull/317) +- Implement realtime ping [\#318](https://github.com/ably/ably-python/pull/318) +- Realtime channel attach/detach [\#319](https://github.com/ably/ably-python/pull/319) +- Add `auto_connect` implementation and client option [\#325](https://github.com/ably/ably-python/pull/325) +- RealtimeChannel subscribe/unsubscribe [\#326](https://github.com/ably/ably-python/pull/326) +- ConnectionStateChange [\#327](https://github.com/ably/ably-python/pull/327) +- Improve realtime logging [\#330](https://github.com/ably/ably-python/pull/330) +- Update readme with realtime documentation [\#334](334](https://github.com/ably/ably-python/pull/334) +- Use string-based enums [\#351](https://github.com/ably/ably-python/pull/351) +- Add environment client option for realtime [\#335](https://github.com/ably/ably-python/pull/335) +- EventEmitter: allow signatures with no event arg [\#350](https://github.com/ably/ably-python/pull/350) + ## [v1.2.2](https://github.com/ably/ably-python/tree/v1.2.2) [Full Changelog](https://github.com/ably/ably-python/compare/v1.2.1...v1.2.2) From b7b29f604228549c639e3d22e61d9cde2e32b5a5 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 2 Nov 2022 14:55:43 +0000 Subject: [PATCH 214/481] chore: add a blurb to 2.0.0-beta.1 changelog notes --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87265643..f49c8c65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## [v2.0.0-beta.1](https://github.com/ably/ably-python/tree/v2.0.0-beta.1) +**New ably-python realtime client**: This beta release features our first ever python realtime client! Currently the realtime client only supports basic authentication and realtime message subscription. Check out the README for usage examples. + [Full Changelog](https://github.com/ably/ably-python/compare/v1.2.1...2.0.0-beta.1) - Create Basic Api Key connection [\#311](https://github.com/ably/ably-python/pull/311) From f0a3b267665c5b6fafc593e847d83dcbf84c44f6 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 2 Nov 2022 14:31:19 +0000 Subject: [PATCH 215/481] add connection error reason field --- ably/realtime/connection.py | 9 +++++++++ test/ably/realtimeconnection_test.py | 7 +++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 1e194c89..5bd0a4d4 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -53,6 +53,8 @@ class Connection(EventEmitter): ---------- state: str Connection state + errorReason: error + An ErrorInfo object describing the last error which occurred on the channel, if any. Methods @@ -67,6 +69,7 @@ class Connection(EventEmitter): def __init__(self, realtime): self.__realtime = realtime + self.__error_reason = None self.__state = ConnectionState.CONNECTING if realtime.options.auto_connect else ConnectionState.INITIALIZED self.__connection_manager = ConnectionManager(self.__realtime, self.state) self.__connection_manager.on('connectionstate', self._on_state_update) @@ -109,6 +112,7 @@ async def ping(self): def _on_state_update(self, state_change): log.info(f'Connection state changing from {self.state} to {state_change.current}') self.__state = state_change.current + self.__error_reason = state_change.reason self.__realtime.options.loop.call_soon(functools.partial(self._emit, state_change.current, state_change)) @property @@ -116,6 +120,11 @@ def state(self): """The current connection state of the connection""" return self.__state + @property + def error_reason(self): + """An object describing the last error which occurred on the channel, if any.""" + return self.__error_reason + @state.setter def state(self, value): self.__state = value diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 72647a31..303c1883 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -37,9 +37,10 @@ async def test_closing_state(self): async def test_auth_invalid_key(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) - with pytest.raises(AblyAuthException): + with pytest.raises(AblyAuthException) as exception: await ably.connect() assert ably.connection.state == ConnectionState.FAILED + assert ably.connection.error_reason == exception.value await ably.close() async def test_connection_ping_connected(self): @@ -60,9 +61,10 @@ async def test_connection_ping_initialized(self): async def test_connection_ping_failed(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) - with pytest.raises(AblyAuthException): + with pytest.raises(AblyAuthException) as exception: await ably.connect() assert ably.connection.state == ConnectionState.FAILED + assert ably.connection.error_reason == exception.value with pytest.raises(AblyException) as exception: await ably.connection.ping() assert exception.value.code == 400 @@ -121,4 +123,5 @@ def on_state_change(change): assert state_change.previous == ConnectionState.CONNECTING assert state_change.current == ConnectionState.FAILED assert state_change.reason == exception.value + assert ably.connection.error_reason == exception.value await ably.close() From bcae1507602161756b7242289fc5a2dc6a4d7fcd Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 2 Nov 2022 16:50:42 +0000 Subject: [PATCH 216/481] update error_reason docstring --- ably/realtime/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 5bd0a4d4..f698e18d 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -53,7 +53,7 @@ class Connection(EventEmitter): ---------- state: str Connection state - errorReason: error + error_reason: ErrorInfo An ErrorInfo object describing the last error which occurred on the channel, if any. From 56f300053434992c89b9397b40c7bcbcd423af06 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 3 Nov 2022 13:56:28 +0000 Subject: [PATCH 217/481] chore: update pyproject description for realtime client --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3231aa0f..62119ca8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "ably" version = "2.0.0-beta.1" -description = "Python REST client library SDK for Ably realtime messaging service" +description = "Python REST and Realtime client library SDK for Ably realtime messaging service" license = "Apache-2.0" authors = ["Ably <support@ably.com>"] readme = "LONG_DESCRIPTION.rst" From 2c96825853c5ba2d7cba364bf48b5fc7198fb704 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 7 Nov 2022 09:57:00 +0000 Subject: [PATCH 218/481] fix realtime host url --- ably/realtime/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index f698e18d..01f6ea75 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -195,7 +195,7 @@ async def send_protocol_message(self, protocolMessage): async def setup_ws(self): headers = HttpUtils.default_headers() - ws_url = f'wss://{self.options.realtime_host}?key={self.__ably.key}' + ws_url = f'wss://{self.options.get_realtime_host()}?key={self.__ably.key}' log.info(f'setup_ws(): attempting to connect to {ws_url}') async with websockets.connect(ws_url, extra_headers=headers) as websocket: log.info(f'setup_ws(): connection established to {ws_url}') From 7ff0466ca8c49edabe7b69cafa0ba2f50136f150 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 7 Nov 2022 16:45:36 +0000 Subject: [PATCH 219/481] chore: bump version for 2.0.0-beta.2 release --- ably/__init__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ably/__init__.py b/ably/__init__.py index ed9c6e09..1d0d927c 100644 --- a/ably/__init__.py +++ b/ably/__init__.py @@ -15,4 +15,4 @@ logger.addHandler(logging.NullHandler()) api_version = '1.2' -lib_version = '2.0.0-beta.1' +lib_version = '2.0.0-beta.2' diff --git a/pyproject.toml b/pyproject.toml index 62119ca8..b0044934 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "ably" -version = "2.0.0-beta.1" +version = "2.0.0-beta.2" description = "Python REST and Realtime client library SDK for Ably realtime messaging service" license = "Apache-2.0" authors = ["Ably <support@ably.com>"] From d74bbb29d8f0650b1f6f7810717ed273b02b80f4 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 7 Nov 2022 16:48:19 +0000 Subject: [PATCH 220/481] chore: update changelog for 2.0.0-beta.2 release --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f49c8c65..8350dc32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Change Log +## [v2.0.0-beta.2](https://github.com/ably/ably-python/tree/v2.0.0-beta.2) + +[Full Changelog](https://github.com/ably/ably-python/compare/v2.0.0-beta.1...v2.0.0-beta.2) +- Fix a bug with realtime_host configuration [\#358](https://github.com/ably/ably-python/pull/358) + ## [v2.0.0-beta.1](https://github.com/ably/ably-python/tree/v2.0.0-beta.1) **New ably-python realtime client**: This beta release features our first ever python realtime client! Currently the realtime client only supports basic authentication and realtime message subscription. Check out the README for usage examples. From 3403fedcb6b2e099c3732d2c49c5aaeea29a2e6b Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 7 Nov 2022 11:33:28 +0000 Subject: [PATCH 221/481] add realtime request timeout --- ably/realtime/connection.py | 7 ++++++- ably/transport/defaults.py | 1 + ably/types/options.py | 10 +++++++++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 01f6ea75..0f631735 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -225,7 +225,12 @@ async def ping(self): async def ws_read_loop(self): while True: - raw = await self.__websocket.recv() + try: + raw = await asyncio.wait_for(self.__websocket.recv(), self.options.realtime_request_timeout) + except asyncio.TimeoutError: + exception = AblyException("Realtime request timeout", 504, 50003) + self.enact_state_change(ConnectionState.FAILED, exception) + raise exception msg = json.loads(raw) log.info(f'ws_read_loop(): receieved protocol message: {msg}') action = msg['action'] diff --git a/ably/transport/defaults.py b/ably/transport/defaults.py index c5fa1d04..3612501f 100644 --- a/ably/transport/defaults.py +++ b/ably/transport/defaults.py @@ -19,6 +19,7 @@ class Defaults: suspended_timeout = 60000 comet_recv_timeout = 90000 comet_send_timeout = 10000 + realtime_request_timeout = 10 transports = [] # ["web_socket", "comet"] diff --git a/ably/types/options.py b/ably/types/options.py index d5f8cc4f..da07a495 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -12,7 +12,7 @@ class Options(AuthOptions): def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realtime_host=None, port=0, tls_port=0, use_binary_protocol=True, queue_messages=False, recover=False, environment=None, - http_open_timeout=None, http_request_timeout=None, + http_open_timeout=None, http_request_timeout=None, realtime_request_timeout=None, http_max_retry_count=None, http_max_retry_duration=None, fallback_hosts=None, fallback_hosts_use_default=None, fallback_retry_timeout=None, idempotent_rest_publishing=None, loop=None, auto_connect=True, @@ -23,6 +23,9 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, if fallback_retry_timeout is None: fallback_retry_timeout = Defaults.fallback_retry_timeout + if realtime_request_timeout is None: + realtime_request_timeout = Defaults.realtime_request_timeout + if environment is not None and rest_host is not None: raise ValueError('specify rest_host or environment, not both') @@ -46,6 +49,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, self.__environment = environment self.__http_open_timeout = http_open_timeout self.__http_request_timeout = http_request_timeout + self.__realtime_request_timeout = realtime_request_timeout self.__http_max_retry_count = http_max_retry_count self.__http_max_retry_duration = http_max_retry_duration self.__fallback_hosts = fallback_hosts @@ -154,6 +158,10 @@ def http_open_timeout(self, value): def http_request_timeout(self): return self.__http_request_timeout + @property + def realtime_request_timeout(self): + return self.__realtime_request_timeout + @http_request_timeout.setter def http_request_timeout(self, value): self.__http_request_timeout = value From 81e560d11126463b3410e83a9736e3a2b06c4a92 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Fri, 11 Nov 2022 09:45:13 +0000 Subject: [PATCH 222/481] change request timeout implementation --- ably/realtime/connection.py | 28 ++++++++++++++++++---------- ably/realtime/realtime_channel.py | 22 ++++++++++++++++------ test/ably/realtimeconnection_test.py | 7 +++++++ 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 0f631735..ad4043eb 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -159,7 +159,10 @@ async def connect(self): if self.__connected_future is None: log.fatal('Connection state is CONNECTING but connected_future does not exist') return - await self.__connected_future + try: + await asyncio.wait_for(self.__connected_future, self.options.realtime_request_timeout) + except asyncio.TimeoutError: + raise AblyException("Realtime request timeout", 504, 50003) self.enact_state_change(ConnectionState.CONNECTED) else: self.enact_state_change(ConnectionState.CONNECTING) @@ -173,7 +176,10 @@ async def close(self): self.__closed_future = asyncio.Future() if self.__websocket and self.__state != ConnectionState.FAILED: await self.send_close_message() - await self.__closed_future + try: + await asyncio.wait_for(self.__closed_future, self.options.realtime_request_timeout) + except asyncio.TimeoutError: + raise AblyException("Realtime request timeout", 504, 50003) else: log.warning('Connection.closed called while connection already closed or not established') self.enact_state_change(ConnectionState.CLOSED) @@ -219,24 +225,25 @@ async def ping(self): "id": self.__ping_id}) else: raise AblyException("Cannot send ping request. Calling ping in invalid state", 40000, 400) + try: + await asyncio.wait_for(self.__ping_future, self.options.realtime_request_timeout) + except asyncio.TimeoutError: + raise AblyException("Realtime request timeout", 504, 50003) + ping_end_time = datetime.now().timestamp() response_time_ms = (ping_end_time - ping_start_time) * 1000 return round(response_time_ms, 2) async def ws_read_loop(self): while True: - try: - raw = await asyncio.wait_for(self.__websocket.recv(), self.options.realtime_request_timeout) - except asyncio.TimeoutError: - exception = AblyException("Realtime request timeout", 504, 50003) - self.enact_state_change(ConnectionState.FAILED, exception) - raise exception + raw = await self.__websocket.recv() msg = json.loads(raw) log.info(f'ws_read_loop(): receieved protocol message: {msg}') action = msg['action'] if action == ProtocolMessageAction.CONNECTED: # CONNECTED if self.__connected_future: - self.__connected_future.set_result(None) + if not self.__connected_future.cancelled(): + self.__connected_future.set_result(None) self.__connected_future = None else: log.warn('CONNECTED message received but connected_future not set') @@ -260,7 +267,8 @@ async def ws_read_loop(self): # Resolve on heartbeat from ping request. # TODO: Handle Normal heartbeat if required if self.__ping_id == msg.get("id"): - self.__ping_future.set_result(None) + if not self.__ping_future.cancelled(): + self.__ping_future.set_result(None) self.__ping_future = None if action in ( ProtocolMessageAction.ATTACHED, diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 9a431be8..fda64c2d 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -80,10 +80,12 @@ async def attach(self): # RTL4h - wait for pending attach/detach if self.state == ChannelState.ATTACHING: - await self.__attach_future + if self.__attach_future and not self.__attach_future.cancelled(): + await self.__attach_future return elif self.state == ChannelState.DETACHING: - await self.__detach_future + if self.__detach_future and not self.__detach_future.cancelled(): + await self.__detach_future self.set_state(ChannelState.ATTACHING) @@ -98,7 +100,10 @@ async def attach(self): "channel": self.name, } ) - await self.__attach_future + try: + await asyncio.wait_for(self.__attach_future, self.__realtime.options.realtime_request_timeout) + except asyncio.TimeoutError: + raise AblyException("Realtime request timeout", 504, 50003) self.set_state(ChannelState.ATTACHED) async def detach(self): @@ -130,10 +135,12 @@ async def detach(self): # RTL5i - wait for pending attach/detach if self.state == ChannelState.DETACHING: - await self.__detach_future + if self.__attach_future and not self.__detach_future.cancelled(): + await self.__detach_future return elif self.state == ChannelState.ATTACHING: - await self.__attach_future + if self.__attach_future and not self.__attach_future.cancelled(): + await self.__attach_future self.set_state(ChannelState.DETACHING) @@ -148,7 +155,10 @@ async def detach(self): "channel": self.name, } ) - await self.__detach_future + try: + await asyncio.wait_for(self.__detach_future, self.__realtime.options.realtime_request_timeout) + except asyncio.TimeoutError: + raise AblyException("Realtime request timeout", 504, 50003) self.set_state(ChannelState.DETACHED) async def subscribe(self, *args): diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 303c1883..dbb7dfd5 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -125,3 +125,10 @@ def on_state_change(change): assert state_change.reason == exception.value assert ably.connection.error_reason == exception.value await ably.close() + + async def test_realtime_request_timeout_connect(self): + ably = await RestSetup.get_ably_realtime(realtime_request_timeout=0.000001) + with pytest.raises(AblyException) as exception: + await ably.connect() + assert exception.value.code == 50003 + assert exception.value.status_code == 504 From cba7b42a4ddee0e8afcde8bb33eecce38e3d1d19 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Fri, 11 Nov 2022 10:45:19 +0000 Subject: [PATCH 223/481] update with disconnected state --- ably/realtime/connection.py | 5 ++++- test/ably/realtimeconnection_test.py | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index ad4043eb..c5cfff7f 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -19,6 +19,7 @@ class ConnectionState(str, Enum): INITIALIZED = 'initialized' CONNECTING = 'connecting' CONNECTED = 'connected' + DISCONNECTED = 'disconnected' CLOSING = 'closing' CLOSED = 'closed' FAILED = 'failed' @@ -162,7 +163,9 @@ async def connect(self): try: await asyncio.wait_for(self.__connected_future, self.options.realtime_request_timeout) except asyncio.TimeoutError: - raise AblyException("Realtime request timeout", 504, 50003) + exception = AblyException("Realtime request timeout", 504, 50003) + self.enact_state_change(ConnectionState.DISCONNECTED, exception) + raise exception self.enact_state_change(ConnectionState.CONNECTED) else: self.enact_state_change(ConnectionState.CONNECTING) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index dbb7dfd5..5a6557ed 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -132,3 +132,6 @@ async def test_realtime_request_timeout_connect(self): await ably.connect() assert exception.value.code == 50003 assert exception.value.status_code == 504 + assert ably.connection.state == ConnectionState.DISCONNECTED + assert ably.connection.error_reason == exception.value + ably.close() From 93a0d9c767ebd596a2f23cb2410cfa92bae90245 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Fri, 11 Nov 2022 15:06:46 +0000 Subject: [PATCH 224/481] refactor connect timeout --- ably/realtime/connection.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index c5cfff7f..a0f46b87 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -161,9 +161,9 @@ async def connect(self): log.fatal('Connection state is CONNECTING but connected_future does not exist') return try: - await asyncio.wait_for(self.__connected_future, self.options.realtime_request_timeout) - except asyncio.TimeoutError: - exception = AblyException("Realtime request timeout", 504, 50003) + await self.__connected_future + except asyncio.CancelledError: + exception = AblyException("Connection cancelled due to request timeout", 504, 50003) self.enact_state_change(ConnectionState.DISCONNECTED, exception) raise exception self.enact_state_change(ConnectionState.CONNECTED) @@ -191,7 +191,12 @@ async def close(self): async def connect_impl(self): self.setup_ws_task = self.__ably.options.loop.create_task(self.setup_ws()) - await self.__connected_future + try: + await asyncio.wait_for(self.__connected_future, self.options.realtime_request_timeout) + except asyncio.TimeoutError: + exception = AblyException("Realtime request timeout", 504, 50003) + self.enact_state_change(ConnectionState.DISCONNECTED, exception) + raise exception self.enact_state_change(ConnectionState.CONNECTED) async def send_close_message(self): From b7c18ac1368f6e3b217c853af769ccac6e64844b Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 14 Nov 2022 14:30:49 +0000 Subject: [PATCH 225/481] review: rename and document realtime timeout --- ably/realtime/connection.py | 6 +++--- ably/realtime/realtime.py | 4 ++++ ably/realtime/realtime_channel.py | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index a0f46b87..44f1a6f5 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -182,7 +182,7 @@ async def close(self): try: await asyncio.wait_for(self.__closed_future, self.options.realtime_request_timeout) except asyncio.TimeoutError: - raise AblyException("Realtime request timeout", 504, 50003) + raise AblyException("Timeout waiting for connection close response", 504, 50003) else: log.warning('Connection.closed called while connection already closed or not established') self.enact_state_change(ConnectionState.CLOSED) @@ -194,7 +194,7 @@ async def connect_impl(self): try: await asyncio.wait_for(self.__connected_future, self.options.realtime_request_timeout) except asyncio.TimeoutError: - exception = AblyException("Realtime request timeout", 504, 50003) + exception = AblyException("Timeout waiting for realtime connection", 504, 50003) self.enact_state_change(ConnectionState.DISCONNECTED, exception) raise exception self.enact_state_change(ConnectionState.CONNECTED) @@ -236,7 +236,7 @@ async def ping(self): try: await asyncio.wait_for(self.__ping_future, self.options.realtime_request_timeout) except asyncio.TimeoutError: - raise AblyException("Realtime request timeout", 504, 50003) + raise AblyException("Timeout waiting for ping response", 504, 50003) ping_end_time = datetime.now().timestamp() response_time_ms = (ping_end_time - ping_start_time) * 1000 diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 7b46ec67..08ffb01c 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -53,6 +53,10 @@ def __init__(self, key=None, loop=None, **kwargs): For development environments only. The default value is realtime.ably.io. environment: str Enables a custom environment to be used with the Ably service. Defaults to `production` + realtime_request_timeout: float + Timeout (in seconds) for the wait of acknowledgement for operations performed via a realtime + connection. Operations include establishing a connection with Ably, or sending a HEARTBEAT, + CONNECT, ATTACH, DETACH or CLOSE request. The default is 10 seconds. Raises ------ diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index fda64c2d..4a4aa4c6 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -103,7 +103,7 @@ async def attach(self): try: await asyncio.wait_for(self.__attach_future, self.__realtime.options.realtime_request_timeout) except asyncio.TimeoutError: - raise AblyException("Realtime request timeout", 504, 50003) + raise AblyException("Timeout waiting for channel attach", 504, 50003) self.set_state(ChannelState.ATTACHED) async def detach(self): @@ -158,7 +158,7 @@ async def detach(self): try: await asyncio.wait_for(self.__detach_future, self.__realtime.options.realtime_request_timeout) except asyncio.TimeoutError: - raise AblyException("Realtime request timeout", 504, 50003) + raise AblyException("Timeout waiting for channel detach", 504, 50003) self.set_state(ChannelState.DETACHED) async def subscribe(self, *args): From f236c9adde459dd26d5e45fb1ff1f1c730d3be2b Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 15 Nov 2022 15:26:22 +0000 Subject: [PATCH 226/481] add more timeout test --- ably/realtime/connection.py | 12 ++++++--- ably/realtime/realtime.py | 4 +-- ably/realtime/realtime_channel.py | 22 ++++++++++----- ably/transport/defaults.py | 2 +- test/ably/realtimechannel_test.py | 40 ++++++++++++++++++++++++++++ test/ably/realtimeconnection_test.py | 19 ++++++++++++- 6 files changed, 85 insertions(+), 14 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 44f1a6f5..12b213c1 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -145,6 +145,7 @@ def __init__(self, realtime, initial_state): self.__websocket = None self.setup_ws_task = None self.__ping_future = None + self.timeout_in_secs = self.options.realtime_request_timeout / 1000 super().__init__() def enact_state_change(self, state, reason=None): @@ -180,7 +181,7 @@ async def close(self): if self.__websocket and self.__state != ConnectionState.FAILED: await self.send_close_message() try: - await asyncio.wait_for(self.__closed_future, self.options.realtime_request_timeout) + await asyncio.wait_for(self.__closed_future, self.timeout_in_secs) except asyncio.TimeoutError: raise AblyException("Timeout waiting for connection close response", 504, 50003) else: @@ -192,7 +193,7 @@ async def close(self): async def connect_impl(self): self.setup_ws_task = self.__ably.options.loop.create_task(self.setup_ws()) try: - await asyncio.wait_for(self.__connected_future, self.options.realtime_request_timeout) + await asyncio.wait_for(self.__connected_future, self.timeout_in_secs) except asyncio.TimeoutError: exception = AblyException("Timeout waiting for realtime connection", 504, 50003) self.enact_state_change(ConnectionState.DISCONNECTED, exception) @@ -222,7 +223,10 @@ async def setup_ws(self): async def ping(self): if self.__ping_future: - response = await self.__ping_future + try: + response = await self.__ping_future + except asyncio.CancelledError: + raise AblyException("Ping request cancelled due to request timeout", 504, 50003) return response self.__ping_future = asyncio.Future() @@ -234,7 +238,7 @@ async def ping(self): else: raise AblyException("Cannot send ping request. Calling ping in invalid state", 40000, 400) try: - await asyncio.wait_for(self.__ping_future, self.options.realtime_request_timeout) + await asyncio.wait_for(self.__ping_future, self.timeout_in_secs) except asyncio.TimeoutError: raise AblyException("Timeout waiting for ping response", 504, 50003) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 08ffb01c..5373e331 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -54,9 +54,9 @@ def __init__(self, key=None, loop=None, **kwargs): environment: str Enables a custom environment to be used with the Ably service. Defaults to `production` realtime_request_timeout: float - Timeout (in seconds) for the wait of acknowledgement for operations performed via a realtime + Timeout (in milliseconds) for the wait of acknowledgement for operations performed via a realtime connection. Operations include establishing a connection with Ably, or sending a HEARTBEAT, - CONNECT, ATTACH, DETACH or CLOSE request. The default is 10 seconds. + CONNECT, ATTACH, DETACH or CLOSE request. The default is 10 seconds(10000 milliseconds). Raises ------ diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 4a4aa4c6..67a37b05 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -50,6 +50,7 @@ def __init__(self, realtime, name): self.__realtime = realtime self.__state = ChannelState.INITIALIZED self.__message_emitter = EventEmitter() + self.timeout_in_secs = self.__realtime.options.realtime_request_timeout / 1000 super().__init__() async def attach(self): @@ -80,12 +81,17 @@ async def attach(self): # RTL4h - wait for pending attach/detach if self.state == ChannelState.ATTACHING: - if self.__attach_future and not self.__attach_future.cancelled(): + try: await self.__attach_future + except asyncio.CancelledError: + raise AblyException("Unable to attach channel due to request timeout", 504, 50003) return elif self.state == ChannelState.DETACHING: - if self.__detach_future and not self.__detach_future.cancelled(): + try: await self.__detach_future + except asyncio.CancelledError: + raise AblyException("Unable to detach channel due to request timeout", 504, 50003) + return self.set_state(ChannelState.ATTACHING) @@ -101,7 +107,7 @@ async def attach(self): } ) try: - await asyncio.wait_for(self.__attach_future, self.__realtime.options.realtime_request_timeout) + await asyncio.wait_for(self.__attach_future, self.timeout_in_secs) except asyncio.TimeoutError: raise AblyException("Timeout waiting for channel attach", 504, 50003) self.set_state(ChannelState.ATTACHED) @@ -135,12 +141,16 @@ async def detach(self): # RTL5i - wait for pending attach/detach if self.state == ChannelState.DETACHING: - if self.__attach_future and not self.__detach_future.cancelled(): + try: await self.__detach_future + except asyncio.CancelledError: + raise AblyException("Unable to detach channel due to request timeout", 504, 50003) return elif self.state == ChannelState.ATTACHING: - if self.__attach_future and not self.__attach_future.cancelled(): + try: await self.__attach_future + except asyncio.CancelledError: + raise AblyException("Unable to attach channel due to request timeout", 504, 50003) self.set_state(ChannelState.DETACHING) @@ -156,7 +166,7 @@ async def detach(self): } ) try: - await asyncio.wait_for(self.__detach_future, self.__realtime.options.realtime_request_timeout) + await asyncio.wait_for(self.__detach_future, self.timeout_in_secs) except asyncio.TimeoutError: raise AblyException("Timeout waiting for channel detach", 504, 50003) self.set_state(ChannelState.DETACHED) diff --git a/ably/transport/defaults.py b/ably/transport/defaults.py index 3612501f..cc67fed0 100644 --- a/ably/transport/defaults.py +++ b/ably/transport/defaults.py @@ -19,7 +19,7 @@ class Defaults: suspended_timeout = 60000 comet_recv_timeout = 90000 comet_send_timeout = 10000 - realtime_request_timeout = 10 + realtime_request_timeout = 10000 transports = [] # ["web_socket", "comet"] diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py index 90072e9d..2b6f6667 100644 --- a/test/ably/realtimechannel_test.py +++ b/test/ably/realtimechannel_test.py @@ -1,8 +1,11 @@ import asyncio +import pytest from ably.realtime.realtime_channel import ChannelState from ably.types.message import Message from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase +from ably.realtime.connection import ProtocolMessageAction +from ably.util.exceptions import AblyException class TestRealtimeChannel(BaseAsyncTestCase): @@ -215,3 +218,40 @@ def listener(msg): await ably.close() await rest.close() + + async def test_realtime_request_timeout_attach(self): + ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) + await ably.connect() + original_send_protocol_message = ably.connection.connection_manager.send_protocol_message + + async def new_send_protocol_message(msg): + if msg.get('action') == ProtocolMessageAction.ATTACH: + return + await original_send_protocol_message(msg) + ably.connection.connection_manager.send_protocol_message = new_send_protocol_message + + channel = ably.channels.get('channel_name') + with pytest.raises(AblyException) as exception: + await channel.attach() + assert exception.value.code == 50003 + assert exception.value.status_code == 504 + await ably.close() + + async def test_realtime_request_timeout_detach(self): + ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) + await ably.connect() + original_send_protocol_message = ably.connection.connection_manager.send_protocol_message + + async def new_send_protocol_message(msg): + if msg.get('action') == ProtocolMessageAction.DETACH: + return + await original_send_protocol_message(msg) + ably.connection.connection_manager.send_protocol_message = new_send_protocol_message + + channel = ably.channels.get('channel_name') + await channel.attach() + with pytest.raises(AblyException) as exception: + await channel.detach() + assert exception.value.code == 50003 + assert exception.value.status_code == 504 + await ably.close() diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 5a6557ed..d5266eca 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -1,5 +1,5 @@ import asyncio -from ably.realtime.connection import ConnectionState +from ably.realtime.connection import ConnectionState, ProtocolMessageAction import pytest from ably.util.exceptions import AblyAuthException, AblyException from test.ably.restsetup import RestSetup @@ -135,3 +135,20 @@ async def test_realtime_request_timeout_connect(self): assert ably.connection.state == ConnectionState.DISCONNECTED assert ably.connection.error_reason == exception.value ably.close() + + async def test_realtime_request_timeout_ping(self): + ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) + await ably.connect() + original_send_protocol_message = ably.connection.connection_manager.send_protocol_message + + async def new_send_protocol_message(msg): + if msg.get('action') == ProtocolMessageAction.HEARTBEAT: + return + await original_send_protocol_message(msg) + ably.connection.connection_manager.send_protocol_message = new_send_protocol_message + + with pytest.raises(AblyException) as exception: + await ably.connection.ping() + assert exception.value.code == 50003 + assert exception.value.status_code == 504 + await ably.close() From 9da1c8de0d9d1f6a4c61f3dbdb85609b16b7a4b9 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 15 Nov 2022 17:03:26 +0000 Subject: [PATCH 227/481] add test for close request timeout --- test/ably/realtimeconnection_test.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index d5266eca..5ec9a0b7 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -152,3 +152,19 @@ async def new_send_protocol_message(msg): assert exception.value.code == 50003 assert exception.value.status_code == 504 await ably.close() + + async def test_realtime_request_timeout_close(self): + ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) + await ably.connect() + original_send_protocol_message = ably.connection.connection_manager.send_protocol_message + + async def new_send_protocol_message(msg): + if msg.get('action') == ProtocolMessageAction.CLOSE: + return + await original_send_protocol_message(msg) + ably.connection.connection_manager.send_protocol_message = new_send_protocol_message + + with pytest.raises(AblyException) as exception: + await ably.close() + assert exception.value.code == 50003 + assert exception.value.status_code == 504 From 4cde24312ace9b9a9f2120d37b3013a18bbcd1e2 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 16 Nov 2022 10:41:45 +0000 Subject: [PATCH 228/481] make timeout internal --- ably/realtime/connection.py | 8 ++++---- ably/realtime/realtime_channel.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 12b213c1..ad3e777b 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -145,7 +145,7 @@ def __init__(self, realtime, initial_state): self.__websocket = None self.setup_ws_task = None self.__ping_future = None - self.timeout_in_secs = self.options.realtime_request_timeout / 1000 + self.__timeout_in_secs = self.options.realtime_request_timeout / 1000 super().__init__() def enact_state_change(self, state, reason=None): @@ -181,7 +181,7 @@ async def close(self): if self.__websocket and self.__state != ConnectionState.FAILED: await self.send_close_message() try: - await asyncio.wait_for(self.__closed_future, self.timeout_in_secs) + await asyncio.wait_for(self.__closed_future, self.__timeout_in_secs) except asyncio.TimeoutError: raise AblyException("Timeout waiting for connection close response", 504, 50003) else: @@ -193,7 +193,7 @@ async def close(self): async def connect_impl(self): self.setup_ws_task = self.__ably.options.loop.create_task(self.setup_ws()) try: - await asyncio.wait_for(self.__connected_future, self.timeout_in_secs) + await asyncio.wait_for(self.__connected_future, self.__timeout_in_secs) except asyncio.TimeoutError: exception = AblyException("Timeout waiting for realtime connection", 504, 50003) self.enact_state_change(ConnectionState.DISCONNECTED, exception) @@ -238,7 +238,7 @@ async def ping(self): else: raise AblyException("Cannot send ping request. Calling ping in invalid state", 40000, 400) try: - await asyncio.wait_for(self.__ping_future, self.timeout_in_secs) + await asyncio.wait_for(self.__ping_future, self.__timeout_in_secs) except asyncio.TimeoutError: raise AblyException("Timeout waiting for ping response", 504, 50003) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 67a37b05..75e3f5e1 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -50,7 +50,7 @@ def __init__(self, realtime, name): self.__realtime = realtime self.__state = ChannelState.INITIALIZED self.__message_emitter = EventEmitter() - self.timeout_in_secs = self.__realtime.options.realtime_request_timeout / 1000 + self.__timeout_in_secs = self.__realtime.options.realtime_request_timeout / 1000 super().__init__() async def attach(self): @@ -107,7 +107,7 @@ async def attach(self): } ) try: - await asyncio.wait_for(self.__attach_future, self.timeout_in_secs) + await asyncio.wait_for(self.__attach_future, self.__timeout_in_secs) except asyncio.TimeoutError: raise AblyException("Timeout waiting for channel attach", 504, 50003) self.set_state(ChannelState.ATTACHED) @@ -166,7 +166,7 @@ async def detach(self): } ) try: - await asyncio.wait_for(self.__detach_future, self.timeout_in_secs) + await asyncio.wait_for(self.__detach_future, self.__timeout_in_secs) except asyncio.TimeoutError: raise AblyException("Timeout waiting for channel detach", 504, 50003) self.set_state(ChannelState.DETACHED) From 3112832eefaf396a9dc5555d9ed74e87965d0ff5 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 15 Nov 2022 14:12:34 +0000 Subject: [PATCH 229/481] refactor: Realtime extends Rest --- ably/realtime/realtime.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 5373e331..7417d113 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -2,6 +2,7 @@ import asyncio from ably.realtime.connection import Connection from ably.rest.auth import Auth +from ably.rest.rest import AblyRest from ably.types.options import Options from ably.realtime.realtime_channel import RealtimeChannel @@ -9,7 +10,7 @@ log = logging.getLogger(__name__) -class AblyRealtime: +class AblyRealtime(AblyRest): """ Ably Realtime Client @@ -63,6 +64,8 @@ def __init__(self, key=None, loop=None, **kwargs): ValueError If no authentication key is not provided """ + super().__init__(key, **kwargs) + if loop is None: try: loop = asyncio.get_running_loop() @@ -102,6 +105,7 @@ async def close(self): """ log.info('Realtime.close() called') await self.connection.close() + await super().close() @property def auth(self): From 0ca9c725aa35269ff6c8d0bb2ddcee79beb7aef0 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 16 Nov 2022 00:44:01 +0000 Subject: [PATCH 230/481] refactor: RealtimeChannel extends Channel --- ably/realtime/realtime_channel.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 75e3f5e1..36cc6703 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -2,6 +2,7 @@ import logging from ably.realtime.connection import ConnectionState, ProtocolMessageAction +from ably.rest.channel import Channel from ably.types.message import Message from ably.util.eventemitter import EventEmitter from ably.util.exceptions import AblyException @@ -20,7 +21,7 @@ class ChannelState(str, Enum): DETACHED = 'detached' -class RealtimeChannel(EventEmitter): +class RealtimeChannel(EventEmitter, Channel): """ Ably Realtime Channel @@ -44,6 +45,7 @@ class RealtimeChannel(EventEmitter): """ def __init__(self, realtime, name): + EventEmitter.__init__(self) self.__name = name self.__attach_future = None self.__detach_future = None @@ -51,7 +53,7 @@ def __init__(self, realtime, name): self.__state = ChannelState.INITIALIZED self.__message_emitter = EventEmitter() self.__timeout_in_secs = self.__realtime.options.realtime_request_timeout / 1000 - super().__init__() + Channel.__init__(self, realtime, name, {}) async def attach(self): """Attach to channel From bc0b380d9b62d068cc1b090e519599e5964b099b Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 16 Nov 2022 00:44:17 +0000 Subject: [PATCH 231/481] test: use Rest methods on Realtime for publishing --- test/ably/realtimechannel_test.py | 7 ++----- test/ably/restsetup.py | 3 ++- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py index 2b6f6667..c95488cf 100644 --- a/test/ably/realtimechannel_test.py +++ b/test/ably/realtimechannel_test.py @@ -63,9 +63,7 @@ def listener(message): await channel.subscribe('event', listener) # publish a message using rest client - rest = await RestSetup.get_ably_rest() - rest_channel = rest.channels.get('my_channel') - await rest_channel.publish('event', 'data') + await channel.publish('event', 'data') message = await first_message_future assert isinstance(message, Message) @@ -73,11 +71,10 @@ def listener(message): assert message.data == 'data' # test that the listener is called again for further publishes - await rest_channel.publish('event', 'data') + await channel.publish('event', 'data') await second_message_future await ably.close() - await rest.close() async def test_subscribe_coroutine(self): ably = await RestSetup.get_ably_realtime() diff --git a/test/ably/restsetup.py b/test/ably/restsetup.py index 5cd73c1d..32097567 100644 --- a/test/ably/restsetup.py +++ b/test/ably/restsetup.py @@ -87,7 +87,8 @@ async def get_ably_realtime(cls, **kw): test_vars = await RestSetup.get_test_vars() options = { 'key': test_vars["keys"][0]["key_str"], - 'realtime_host': realtime_host, + 'realtime_host': test_vars["realtime_host"], + 'rest_host': test_vars["host"], 'port': test_vars["port"], 'tls_port': test_vars["tls_port"], 'tls': test_vars["tls"], From 7c9aa3744b00943d022b11d738fc2ab7705dae0e Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 28 Nov 2022 11:46:13 +0000 Subject: [PATCH 232/481] clear connection error reason connect is called --- ably/realtime/connection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index ad3e777b..372f4f2d 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -81,6 +81,7 @@ async def connect(self): Causes the connection to open, entering the connecting state """ + self.__error_reason = None await self.__connection_manager.connect() async def close(self): From 8c1896c1a0297c2c3cae3cdb2f43f8b9a619a9af Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 28 Nov 2022 13:59:18 +0000 Subject: [PATCH 233/481] send api protocol version --- ably/realtime/connection.py | 3 ++- ably/transport/defaults.py | 2 +- ably/types/options.py | 5 +++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 372f4f2d..ba4c2184 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -211,7 +211,8 @@ async def send_protocol_message(self, protocolMessage): async def setup_ws(self): headers = HttpUtils.default_headers() - ws_url = f'wss://{self.options.get_realtime_host()}?key={self.__ably.key}' + ws_url = (f'wss://{self.options.get_realtime_host()}?key={self.__ably.key}' + f'&v={self.options.protocol_version}') log.info(f'setup_ws(): attempting to connect to {ws_url}') async with websockets.connect(ws_url, extra_headers=headers) as websocket: log.info(f'setup_ws(): connection established to {ws_url}') diff --git a/ably/transport/defaults.py b/ably/transport/defaults.py index cc67fed0..60303ef5 100644 --- a/ably/transport/defaults.py +++ b/ably/transport/defaults.py @@ -1,5 +1,5 @@ class Defaults: - protocol_version = 1 + protocol_version = "2" fallback_hosts = [ "a.ably-realtime.com", "b.ably-realtime.com", diff --git a/ably/types/options.py b/ably/types/options.py index da07a495..403620d6 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -61,6 +61,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, self.__rest_hosts = self.__get_rest_hosts() self.__realtime_hosts = self.__get_realtime_hosts() + self.__protocol_version = Defaults.protocol_version @property def client_id(self): @@ -206,6 +207,10 @@ def loop(self): def auto_connect(self): return self.__auto_connect + @property + def protocol_version(self): + return self.__protocol_version + def __get_rest_hosts(self): """ Return the list of hosts as they should be tried. First comes the main From 7c31d2b6300c3422e6b0a3035c62b28aa3d3a887 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 30 Nov 2022 15:20:24 +0000 Subject: [PATCH 234/481] review: encode url params --- ably/realtime/connection.py | 8 ++++++-- ably/types/options.py | 5 ----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index ba4c2184..9a3fe37e 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -3,7 +3,9 @@ import asyncio import websockets import json +import urllib.parse from ably.http.httputils import HttpUtils +from ably.transport.defaults import Defaults from ably.util.exceptions import AblyAuthException, AblyException from ably.util.eventemitter import EventEmitter from enum import Enum, IntEnum @@ -211,8 +213,10 @@ async def send_protocol_message(self, protocolMessage): async def setup_ws(self): headers = HttpUtils.default_headers() - ws_url = (f'wss://{self.options.get_realtime_host()}?key={self.__ably.key}' - f'&v={self.options.protocol_version}') + protocol_version = Defaults.protocol_version + params = {"key": self.__ably.key, "v": protocol_version} + query_params = urllib.parse.urlencode(params) + ws_url = (f'wss://{self.options.get_realtime_host()}?{query_params}') log.info(f'setup_ws(): attempting to connect to {ws_url}') async with websockets.connect(ws_url, extra_headers=headers) as websocket: log.info(f'setup_ws(): connection established to {ws_url}') diff --git a/ably/types/options.py b/ably/types/options.py index 403620d6..da07a495 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -61,7 +61,6 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, self.__rest_hosts = self.__get_rest_hosts() self.__realtime_hosts = self.__get_realtime_hosts() - self.__protocol_version = Defaults.protocol_version @property def client_id(self): @@ -207,10 +206,6 @@ def loop(self): def auto_connect(self): return self.__auto_connect - @property - def protocol_version(self): - return self.__protocol_version - def __get_rest_hosts(self): """ Return the list of hosts as they should be tried. First comes the main From 761e17429fb7971037099fc9caf87816b76b752c Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 21 Nov 2022 17:15:21 +0000 Subject: [PATCH 235/481] implement disconnected retry timeout --- ably/realtime/connection.py | 16 ++++++++++++++-- ably/realtime/realtime.py | 4 +++- ably/transport/defaults.py | 1 + ably/types/options.py | 12 ++++++++++-- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 9a3fe37e..ea5ba381 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -167,8 +167,10 @@ async def connect(self): try: await self.__connected_future except asyncio.CancelledError: - exception = AblyException("Connection cancelled due to request timeout", 504, 50003) + exception = AblyException( + "Connection cancelled due to request timeout. Attempting reconnection...", 504, 50003) self.enact_state_change(ConnectionState.DISCONNECTED, exception) + log.info('Connection cancelled due to request timeout. Attempting reconnection...') raise exception self.enact_state_change(ConnectionState.CONNECTED) else: @@ -193,14 +195,24 @@ async def close(self): if self.setup_ws_task: await self.setup_ws_task + def on_setup_ws_done(self, task): + exception = task.exception() + if exception is not None: + if self.__connected_future and not self.__connected_future.cancelled(): + self.__connected_future.set_exception(exception) + self.enact_state_change(ConnectionState.DISCONNECTED, exception) + async def connect_impl(self): self.setup_ws_task = self.__ably.options.loop.create_task(self.setup_ws()) + self.setup_ws_task.add_done_callback(self.on_setup_ws_done) try: await asyncio.wait_for(self.__connected_future, self.__timeout_in_secs) except asyncio.TimeoutError: exception = AblyException("Timeout waiting for realtime connection", 504, 50003) self.enact_state_change(ConnectionState.DISCONNECTED, exception) - raise exception + await asyncio.sleep(self.options.disconnected_retry_timeout / 1000) + log.info('Attempting reconnection') + await self.connect() self.enact_state_change(ConnectionState.CONNECTED) async def send_close_message(self): diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 7417d113..4539f460 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -58,7 +58,9 @@ def __init__(self, key=None, loop=None, **kwargs): Timeout (in milliseconds) for the wait of acknowledgement for operations performed via a realtime connection. Operations include establishing a connection with Ably, or sending a HEARTBEAT, CONNECT, ATTACH, DETACH or CLOSE request. The default is 10 seconds(10000 milliseconds). - + disconnected_retry_timeout: float + If the connection is still in the DISCONNECTED state after this delay, the client library will + attempt to reconnect automatically. The default is 15 seconds. Raises ------ ValueError diff --git a/ably/transport/defaults.py b/ably/transport/defaults.py index 60303ef5..79f72ca9 100644 --- a/ably/transport/defaults.py +++ b/ably/transport/defaults.py @@ -20,6 +20,7 @@ class Defaults: comet_recv_timeout = 90000 comet_send_timeout = 10000 realtime_request_timeout = 10000 + disconnected_retry_timeout = 15000 transports = [] # ["web_socket", "comet"] diff --git a/ably/types/options.py b/ably/types/options.py index da07a495..0a926992 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -13,8 +13,8 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realtime_host=None, port=0, tls_port=0, use_binary_protocol=True, queue_messages=False, recover=False, environment=None, http_open_timeout=None, http_request_timeout=None, realtime_request_timeout=None, - http_max_retry_count=None, http_max_retry_duration=None, - fallback_hosts=None, fallback_hosts_use_default=None, fallback_retry_timeout=None, + http_max_retry_count=None, http_max_retry_duration=None, fallback_hosts=None, + fallback_hosts_use_default=None, fallback_retry_timeout=None, disconnected_retry_timeout=None, idempotent_rest_publishing=None, loop=None, auto_connect=True, **kwargs): super().__init__(**kwargs) @@ -26,6 +26,9 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, if realtime_request_timeout is None: realtime_request_timeout = Defaults.realtime_request_timeout + if disconnected_retry_timeout is None: + disconnected_retry_timeout = Defaults.disconnected_retry_timeout + if environment is not None and rest_host is not None: raise ValueError('specify rest_host or environment, not both') @@ -55,6 +58,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, self.__fallback_hosts = fallback_hosts self.__fallback_hosts_use_default = fallback_hosts_use_default self.__fallback_retry_timeout = fallback_retry_timeout + self.__disconnected_retry_timeout = disconnected_retry_timeout self.__idempotent_rest_publishing = idempotent_rest_publishing self.__loop = loop self.__auto_connect = auto_connect @@ -194,6 +198,10 @@ def fallback_hosts_use_default(self): def fallback_retry_timeout(self): return self.__fallback_retry_timeout + @property + def disconnected_retry_timeout(self): + return self.__disconnected_retry_timeout + @property def idempotent_rest_publishing(self): return self.__idempotent_rest_publishing From 72c3d52f589642bb1b49b46cf87a23b9b34b89d3 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 23 Nov 2022 16:12:31 +0000 Subject: [PATCH 236/481] add test for disconnected retry --- test/ably/realtimeconnection_test.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 5ec9a0b7..73c38a82 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -168,3 +168,26 @@ async def new_send_protocol_message(msg): await ably.close() assert exception.value.code == 50003 assert exception.value.status_code == 504 + + async def test_disconnected_retry_timeout(self): + ably = await RestSetup.get_ably_realtime(realtime_request_timeout=0.001, + disconnected_retry_timeout=2000) + state_changes = [] + + def on_state_change(state_change): + state_changes.append(state_change) + + ably.connection.on(on_state_change) + + with pytest.raises(AblyException) as exception: + await ably.connect() + assert exception.value.code == 50003 + assert exception.value.status_code == 504 + assert ably.connection.state == ConnectionState.DISCONNECTED + # 2 state changes happens per retry. + # Retry timeout of 2 secs, will retry connection twice in 3 and/or 4 seconds, resulting in 4 state changes + await asyncio.sleep(4) + assert len(state_changes) == 4 + assert state_changes[0].previous == ConnectionState.CONNECTING + assert state_changes[0].current == ConnectionState.DISCONNECTED + ably.close() From 41880b36532f5d7fb596af943fb3b3c496bd1018 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Fri, 25 Nov 2022 12:15:57 +0000 Subject: [PATCH 237/481] change retry implementation --- ably/realtime/connection.py | 10 ++++++++-- ably/realtime/realtime.py | 2 +- ably/transport/defaults.py | 2 +- test/ably/realtimeconnection_test.py | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index ea5ba381..805f11c2 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -165,12 +165,14 @@ async def connect(self): log.fatal('Connection state is CONNECTING but connected_future does not exist') return try: + print("toh") await self.__connected_future except asyncio.CancelledError: exception = AblyException( "Connection cancelled due to request timeout. Attempting reconnection...", 504, 50003) self.enact_state_change(ConnectionState.DISCONNECTED, exception) log.info('Connection cancelled due to request timeout. Attempting reconnection...') + print("cancelled error") raise exception self.enact_state_change(ConnectionState.CONNECTED) else: @@ -209,10 +211,14 @@ async def connect_impl(self): await asyncio.wait_for(self.__connected_future, self.__timeout_in_secs) except asyncio.TimeoutError: exception = AblyException("Timeout waiting for realtime connection", 504, 50003) - self.enact_state_change(ConnectionState.DISCONNECTED, exception) + self.enact_state_change(ConnectionState.CONNECTING, exception) await asyncio.sleep(self.options.disconnected_retry_timeout / 1000) log.info('Attempting reconnection') - await self.connect() + self.__connected_future = asyncio.Future() + print("timeout error") + # task.add_done_callback(self.on_setup_ws_done) + # task = self.__ably.options.loop.create_task(self.connect()) + self.__ably.options.loop.create_task(self.connect()) self.enact_state_change(ConnectionState.CONNECTED) async def send_close_message(self): diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 4539f460..4bc0aaa9 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -86,7 +86,7 @@ def __init__(self, key=None, loop=None, **kwargs): self.key = key self.__connection = Connection(self) self.__channels = Channels(self) - + print(options.auto_connect, "+++") if options.auto_connect: asyncio.ensure_future(self.connection.connection_manager.connect_impl()) diff --git a/ably/transport/defaults.py b/ably/transport/defaults.py index 79f72ca9..6b0fec88 100644 --- a/ably/transport/defaults.py +++ b/ably/transport/defaults.py @@ -20,7 +20,7 @@ class Defaults: comet_recv_timeout = 90000 comet_send_timeout = 10000 realtime_request_timeout = 10000 - disconnected_retry_timeout = 15000 + disconnected_retry_timeout = 1500 transports = [] # ["web_socket", "comet"] diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 73c38a82..6d8b25c2 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -171,7 +171,7 @@ async def new_send_protocol_message(msg): async def test_disconnected_retry_timeout(self): ably = await RestSetup.get_ably_realtime(realtime_request_timeout=0.001, - disconnected_retry_timeout=2000) + disconnected_retry_timeout=2000, auto_connect=False) state_changes = [] def on_state_change(state_change): From 6e0a1a14e9f59a9cf9d503e759a8accf9453831c Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 29 Nov 2022 11:57:17 +0000 Subject: [PATCH 238/481] refactor: create WebSocketTransport class --- ably/realtime/websockettransport.py | 107 ++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 ably/realtime/websockettransport.py diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py new file mode 100644 index 00000000..1409e5bd --- /dev/null +++ b/ably/realtime/websockettransport.py @@ -0,0 +1,107 @@ +from __future__ import annotations +from typing import TYPE_CHECKING +import asyncio +from enum import IntEnum +import json +import logging +from ably.http.httputils import HttpUtils +from websockets.client import WebSocketClientProtocol, connect as ws_connect +from websockets.exceptions import ConnectionClosedOK + +if TYPE_CHECKING: + from ably.realtime.connection import ConnectionManager + +log = logging.getLogger(__name__) + + +class ProtocolMessageAction(IntEnum): + HEARTBEAT = 0 + CONNECTED = 4 + ERROR = 9 + CLOSE = 7 + CLOSED = 8 + ATTACH = 10 + ATTACHED = 11 + DETACH = 12 + DETACHED = 13 + MESSAGE = 15 + + +class WebSocketTransport: + def __init__(self, connection_manager: ConnectionManager): + self.websocket: WebSocketClientProtocol | None = None + self.read_loop: asyncio.Task | None = None + self.connect_task: asyncio.Task | None = None + self.ws_connect_task: asyncio.Task | None = None + self.connection_manager = connection_manager + self.is_connected = False + + async def connect(self): + headers = HttpUtils.default_headers() + host = self.connection_manager.options.get_realtime_host() + key = self.connection_manager.ably.key + ws_url = f'wss://{host}?key={key}' + log.info(f'connect(): attempting to connect to {ws_url}') + self.ws_connect_task = asyncio.create_task(self.ws_connect(ws_url, headers)) + self.ws_connect_task.add_done_callback(self.on_ws_connect_done) + + def on_ws_connect_done(self, task: asyncio.Task): + try: + exception = task.exception() + except asyncio.CancelledError as e: + exception = e + if isinstance(exception, ConnectionClosedOK): + return + + async def ws_connect(self, ws_url, headers): + async with ws_connect(ws_url, extra_headers=headers) as websocket: + log.info(f'ws_connect(): connection established to {ws_url}') + self.websocket = websocket + self.read_loop = self.connection_manager.options.loop.create_task(self.ws_read_loop()) + self.read_loop.add_done_callback(self.on_read_loop_done) + await self.read_loop + + async def ws_read_loop(self): + while True: + if self.websocket is not None: + try: + raw = await self.websocket.recv() + except ConnectionClosedOK: + break + msg = json.loads(raw) + log.info(f'ws_read_loop(): receieved protocol message: {msg}') + if msg['action'] == ProtocolMessageAction.CLOSED: + if self.ws_connect_task: + self.ws_connect_task.cancel() + await self.connection_manager.on_protocol_message(msg) + else: + raise Exception() + + def on_read_loop_done(self, task: asyncio.Task): + try: + exception = task.exception() + except asyncio.CancelledError as e: + exception = e + if isinstance(exception, ConnectionClosedOK): + return + + async def dispose(self): + if self.read_loop: + self.read_loop.cancel() + if self.ws_connect_task: + self.ws_connect_task.cancel() + if self.websocket: + try: + await self.websocket.close() + except asyncio.CancelledError: + return + + async def close(self): + await self.send({'action': ProtocolMessageAction.CLOSE}) + + async def send(self, message: dict): + if self.websocket is None: + raise Exception() + raw_msg = json.dumps(message) + log.info(f'WebSocketTransport.send(): sending {raw_msg}') + await self.websocket.send(raw_msg) From 837ce574136f49f0aff4b7a1be37592ed1b5bd0a Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 29 Nov 2022 11:58:13 +0000 Subject: [PATCH 239/481] refactor: use ProtocolMessageAction from websockettransport module --- ably/realtime/connection.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 805f11c2..ee5d731e 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -1,6 +1,7 @@ import functools import logging import asyncio +from ably.realtime.connectionmanager import ProtocolMessageAction import websockets import json import urllib.parse @@ -8,7 +9,7 @@ from ably.transport.defaults import Defaults from ably.util.exceptions import AblyAuthException, AblyException from ably.util.eventemitter import EventEmitter -from enum import Enum, IntEnum +from enum import Enum from datetime import datetime from ably.util import helper from dataclasses import dataclass @@ -34,19 +35,6 @@ class ConnectionStateChange: reason: Optional[AblyException] = None -class ProtocolMessageAction(IntEnum): - HEARTBEAT = 0 - CONNECTED = 4 - ERROR = 9 - CLOSE = 7 - CLOSED = 8 - ATTACH = 10 - ATTACHED = 11 - DETACH = 12 - DETACHED = 13 - MESSAGE = 15 - - class Connection(EventEmitter): """Ably Realtime Connection From f7a3467cfc4b8ace8c4dd0f2263419c419eecb9b Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 29 Nov 2022 11:58:50 +0000 Subject: [PATCH 240/481] chore: fix styling of protocol_message var --- ably/realtime/connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index ee5d731e..2c515a98 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -212,8 +212,8 @@ async def connect_impl(self): async def send_close_message(self): await self.send_protocol_message({"action": ProtocolMessageAction.CLOSE}) - async def send_protocol_message(self, protocolMessage): - raw_msg = json.dumps(protocolMessage) + async def send_protocol_message(self, protocol_message): + raw_msg = json.dumps(protocol_message) log.info('send_protocol_message(): sending {raw_msg}') await self.__websocket.send(raw_msg) From 7b5079f5923265cb62d679ac51cc2bd8e06997d7 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 29 Nov 2022 12:11:51 +0000 Subject: [PATCH 241/481] refactor: use WebSocketTransport in ConnectionManager --- ably/realtime/connection.py | 200 +++++++++++++-------------- ably/realtime/realtime.py | 1 - ably/realtime/websockettransport.py | 8 ++ test/ably/realtimeconnection_test.py | 10 +- 4 files changed, 110 insertions(+), 109 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 2c515a98..f1e7aa13 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -1,12 +1,7 @@ import functools import logging import asyncio -from ably.realtime.connectionmanager import ProtocolMessageAction -import websockets -import json -import urllib.parse -from ably.http.httputils import HttpUtils -from ably.transport.defaults import Defaults +from ably.realtime.websockettransport import WebSocketTransport, ProtocolMessageAction from ably.util.exceptions import AblyAuthException, AblyException from ably.util.eventemitter import EventEmitter from enum import Enum @@ -133,10 +128,9 @@ def __init__(self, realtime, initial_state): self.__state = initial_state self.__connected_future = asyncio.Future() if initial_state == ConnectionState.CONNECTING else None self.__closed_future = None - self.__websocket = None - self.setup_ws_task = None self.__ping_future = None self.__timeout_in_secs = self.options.realtime_request_timeout / 1000 + self.transport: WebSocketTransport | None = None super().__init__() def enact_state_change(self, state, reason=None): @@ -145,93 +139,96 @@ def enact_state_change(self, state, reason=None): self._emit('connectionstate', ConnectionStateChange(current_state, state, reason)) async def connect(self): + if not self.__connected_future: + self.__connected_future = asyncio.Future() + self.try_connect() + await self.__connected_future + + def try_connect(self): + task = asyncio.create_task(self._connect()) + task.add_done_callback(self.on_connection_attempt_done) + + async def _connect(self): if self.__state == ConnectionState.CONNECTED: return if self.__state == ConnectionState.CONNECTING: - if self.__connected_future is None: - log.fatal('Connection state is CONNECTING but connected_future does not exist') - return try: - print("toh") + if not self.__connected_future: + self.__connected_future = asyncio.Future() await self.__connected_future except asyncio.CancelledError: exception = AblyException( "Connection cancelled due to request timeout. Attempting reconnection...", 504, 50003) - self.enact_state_change(ConnectionState.DISCONNECTED, exception) log.info('Connection cancelled due to request timeout. Attempting reconnection...') - print("cancelled error") raise exception - self.enact_state_change(ConnectionState.CONNECTED) else: self.enact_state_change(ConnectionState.CONNECTING) - self.__connected_future = asyncio.Future() await self.connect_impl() + def on_connection_attempt_done(self, task): + try: + exception = task.exception() + except asyncio.CancelledError: + exception = AblyException( + "Connection cancelled due to request timeout. Attempting reconnection...", 504, 50003) + if exception is None: + return + if self.__state in (ConnectionState.CLOSED, ConnectionState.FAILED): + return + if self.__state != ConnectionState.DISCONNECTED: + if self.__connected_future: + self.__connected_future.set_exception(exception) + self.__connected_future = None + self.enact_state_change(ConnectionState.DISCONNECTED, exception) + async def close(self): + if self.__state in (ConnectionState.CLOSED, ConnectionState.INITIALIZED, ConnectionState.FAILED): + self.enact_state_change(ConnectionState.CLOSED) + return + if self.__state is ConnectionState.DISCONNECTED: + if self.transport: + await self.transport.dispose() + self.transport = None + self.enact_state_change(ConnectionState.CLOSED) + return if self.__state != ConnectionState.CONNECTED: log.warning('Connection.closed called while connection state not connected') + if self.__state == ConnectionState.CONNECTING: + await self.__connected_future self.enact_state_change(ConnectionState.CLOSING) self.__closed_future = asyncio.Future() - if self.__websocket and self.__state != ConnectionState.FAILED: - await self.send_close_message() + if self.transport and self.transport.is_connected: + await self.transport.close() try: await asyncio.wait_for(self.__closed_future, self.__timeout_in_secs) except asyncio.TimeoutError: raise AblyException("Timeout waiting for connection close response", 504, 50003) else: - log.warning('Connection.closed called while connection already closed or not established') + log.warning('ConnectionManager: called close with no connected transport') self.enact_state_change(ConnectionState.CLOSED) - if self.setup_ws_task: - await self.setup_ws_task - - def on_setup_ws_done(self, task): - exception = task.exception() - if exception is not None: - if self.__connected_future and not self.__connected_future.cancelled(): - self.__connected_future.set_exception(exception) - self.enact_state_change(ConnectionState.DISCONNECTED, exception) + if self.transport and self.transport.ws_connect_task is not None: + await self.transport.ws_connect_task async def connect_impl(self): - self.setup_ws_task = self.__ably.options.loop.create_task(self.setup_ws()) - self.setup_ws_task.add_done_callback(self.on_setup_ws_done) + self.transport = WebSocketTransport(self) + await self.transport.connect() try: - await asyncio.wait_for(self.__connected_future, self.__timeout_in_secs) + await asyncio.wait_for(asyncio.shield(self.__connected_future), self.__timeout_in_secs) except asyncio.TimeoutError: exception = AblyException("Timeout waiting for realtime connection", 504, 50003) - self.enact_state_change(ConnectionState.CONNECTING, exception) - await asyncio.sleep(self.options.disconnected_retry_timeout / 1000) - log.info('Attempting reconnection') - self.__connected_future = asyncio.Future() - print("timeout error") - # task.add_done_callback(self.on_setup_ws_done) - # task = self.__ably.options.loop.create_task(self.connect()) - self.__ably.options.loop.create_task(self.connect()) - self.enact_state_change(ConnectionState.CONNECTED) - - async def send_close_message(self): - await self.send_protocol_message({"action": ProtocolMessageAction.CLOSE}) + self.enact_state_change(ConnectionState.DISCONNECTED, exception) + if self.transport: + await self.transport.dispose() + self.tranpsort = None + self.__connected_future.set_exception(exception) + raise exception async def send_protocol_message(self, protocol_message): - raw_msg = json.dumps(protocol_message) - log.info('send_protocol_message(): sending {raw_msg}') - await self.__websocket.send(raw_msg) - - async def setup_ws(self): - headers = HttpUtils.default_headers() - protocol_version = Defaults.protocol_version - params = {"key": self.__ably.key, "v": protocol_version} - query_params = urllib.parse.urlencode(params) - ws_url = (f'wss://{self.options.get_realtime_host()}?{query_params}') - log.info(f'setup_ws(): attempting to connect to {ws_url}') - async with websockets.connect(ws_url, extra_headers=headers) as websocket: - log.info(f'setup_ws(): connection established to {ws_url}') - self.__websocket = websocket - task = self.__ably.options.loop.create_task(self.ws_read_loop()) - try: - await task - except AblyAuthException: - return + if self.transport is not None: + await self.transport.send(protocol_message) + else: + raise Exception() async def ping(self): if self.__ping_future: @@ -258,48 +255,47 @@ async def ping(self): response_time_ms = (ping_end_time - ping_start_time) * 1000 return round(response_time_ms, 2) - async def ws_read_loop(self): - while True: - raw = await self.__websocket.recv() - msg = json.loads(raw) - log.info(f'ws_read_loop(): receieved protocol message: {msg}') - action = msg['action'] - if action == ProtocolMessageAction.CONNECTED: # CONNECTED + async def on_protocol_message(self, msg): + action = msg['action'] + if action == ProtocolMessageAction.CONNECTED: # CONNECTED + if self.transport: + self.transport.is_connected = True + if self.__connected_future: + if not self.__connected_future.cancelled(): + self.__connected_future.set_result(None) + self.__connected_future = None + else: + log.warn('CONNECTED message received but connected_future not set') + self.enact_state_change(ConnectionState.CONNECTED) + if action == ProtocolMessageAction.ERROR: # ERROR + error = msg["error"] + if error['nonfatal'] is False: + exception = AblyAuthException(error["message"], error["statusCode"], error["code"]) + self.enact_state_change(ConnectionState.FAILED, exception) if self.__connected_future: - if not self.__connected_future.cancelled(): - self.__connected_future.set_result(None) + self.__connected_future.set_exception(exception) self.__connected_future = None - else: - log.warn('CONNECTED message received but connected_future not set') - if action == ProtocolMessageAction.ERROR: # ERROR - error = msg["error"] - if error['nonfatal'] is False: - exception = AblyAuthException(error["message"], error["statusCode"], error["code"]) - self.enact_state_change(ConnectionState.FAILED, exception) - if self.__connected_future: - self.__connected_future.set_exception(exception) - self.__connected_future = None - self.__websocket = None - raise exception - if action == ProtocolMessageAction.CLOSED: - await self.__websocket.close() - self.__websocket = None - self.__closed_future.set_result(None) - break - if action == ProtocolMessageAction.HEARTBEAT: - if self.__ping_future: - # Resolve on heartbeat from ping request. - # TODO: Handle Normal heartbeat if required - if self.__ping_id == msg.get("id"): - if not self.__ping_future.cancelled(): - self.__ping_future.set_result(None) - self.__ping_future = None - if action in ( - ProtocolMessageAction.ATTACHED, - ProtocolMessageAction.DETACHED, - ProtocolMessageAction.MESSAGE - ): - self.__ably.channels._on_channel_message(msg) + if self.transport: + await self.transport.dispose() + raise exception + if action == ProtocolMessageAction.CLOSED: + if self.transport: + await self.transport.dispose() + self.__closed_future.set_result(None) + if action == ProtocolMessageAction.HEARTBEAT: + if self.__ping_future: + # Resolve on heartbeat from ping request. + # TODO: Handle Normal heartbeat if required + if self.__ping_id == msg.get("id"): + if not self.__ping_future.cancelled(): + self.__ping_future.set_result(None) + self.__ping_future = None + if action in ( + ProtocolMessageAction.ATTACHED, + ProtocolMessageAction.DETACHED, + ProtocolMessageAction.MESSAGE + ): + self.__ably.channels._on_channel_message(msg) @property def ably(self): diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 4bc0aaa9..c9c73dd4 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -86,7 +86,6 @@ def __init__(self, key=None, loop=None, **kwargs): self.key = key self.__connection = Connection(self) self.__channels = Channels(self) - print(options.auto_connect, "+++") if options.auto_connect: asyncio.ensure_future(self.connection.connection_manager.connect_impl()) diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index 1409e5bd..74ab0e1d 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -4,7 +4,9 @@ from enum import IntEnum import json import logging +import urllib.parse from ably.http.httputils import HttpUtils +from ably.transport.defaults import Defaults from websockets.client import WebSocketClientProtocol, connect as ws_connect from websockets.exceptions import ConnectionClosedOK @@ -37,6 +39,12 @@ def __init__(self, connection_manager: ConnectionManager): self.is_connected = False async def connect(self): + headers = HttpUtils.default_headers() + protocol_version = Defaults.protocol_version + params = {"key": self.connection_manager.ably.key, "v": protocol_version} + query_params = urllib.parse.urlencode(params) + ws_url = (f'wss://{self.connection_manager.options.get_realtime_host()}?{query_params}') + headers = HttpUtils.default_headers() host = self.connection_manager.options.get_realtime_host() key = self.connection_manager.ably.key diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 6d8b25c2..188c614a 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -156,13 +156,11 @@ async def new_send_protocol_message(msg): async def test_realtime_request_timeout_close(self): ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) await ably.connect() - original_send_protocol_message = ably.connection.connection_manager.send_protocol_message - async def new_send_protocol_message(msg): - if msg.get('action') == ProtocolMessageAction.CLOSE: - return - await original_send_protocol_message(msg) - ably.connection.connection_manager.send_protocol_message = new_send_protocol_message + async def new_close_transport(): + pass + + ably.connection.connection_manager.transport.close = new_close_transport with pytest.raises(AblyException) as exception: await ably.close() From 9cf53cd8486f2d6fa2f93404f40da7276be99e2e Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 29 Nov 2022 12:13:47 +0000 Subject: [PATCH 242/481] fix: await calls to ably.close() in tests --- test/ably/realtimeconnection_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 188c614a..2ee39b82 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -134,7 +134,7 @@ async def test_realtime_request_timeout_connect(self): assert exception.value.status_code == 504 assert ably.connection.state == ConnectionState.DISCONNECTED assert ably.connection.error_reason == exception.value - ably.close() + await ably.close() async def test_realtime_request_timeout_ping(self): ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) @@ -188,4 +188,4 @@ def on_state_change(state_change): assert len(state_changes) == 4 assert state_changes[0].previous == ConnectionState.CONNECTING assert state_changes[0].current == ConnectionState.DISCONNECTED - ably.close() + await ably.close() From 16c00ea9acd122b8a866127805f7d6fe3477cce7 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 29 Nov 2022 13:06:37 +0000 Subject: [PATCH 243/481] refactor: transition to DISCONNECTED synchronously on timeout --- ably/realtime/connection.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index f1e7aa13..4336e2aa 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -217,12 +217,13 @@ async def connect_impl(self): await asyncio.wait_for(asyncio.shield(self.__connected_future), self.__timeout_in_secs) except asyncio.TimeoutError: exception = AblyException("Timeout waiting for realtime connection", 504, 50003) - self.enact_state_change(ConnectionState.DISCONNECTED, exception) if self.transport: await self.transport.dispose() self.tranpsort = None self.__connected_future.set_exception(exception) - raise exception + connected_future = self.__connected_future + self.__connected_future = None + self.on_connection_attempt_done(connected_future) async def send_protocol_message(self, protocol_message): if self.transport is not None: From ddf3fd91c6e1188ea35f585542b859b332323116 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 29 Nov 2022 13:07:09 +0000 Subject: [PATCH 244/481] refactor: improve invalid state WebSocketTransport error --- ably/realtime/websockettransport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index 74ab0e1d..6451235f 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -83,7 +83,7 @@ async def ws_read_loop(self): self.ws_connect_task.cancel() await self.connection_manager.on_protocol_message(msg) else: - raise Exception() + raise Exception('ws_read_loop running with no websocket') def on_read_loop_done(self, task: asyncio.Task): try: From f4a18cf9e9c22a1edf3fec0c4e140915be021250 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 29 Nov 2022 13:07:27 +0000 Subject: [PATCH 245/481] feat: reimplement disconnected_retry_timeout --- ably/realtime/connection.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 4336e2aa..b79898da 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -181,6 +181,11 @@ def on_connection_attempt_done(self, task): self.__connected_future.set_exception(exception) self.__connected_future = None self.enact_state_change(ConnectionState.DISCONNECTED, exception) + asyncio.create_task(self.retry_connection_attempt()) + + async def retry_connection_attempt(self): + await asyncio.sleep(self.ably.options.disconnected_retry_timeout / 1000) + self.try_connect() async def close(self): if self.__state in (ConnectionState.CLOSED, ConnectionState.INITIALIZED, ConnectionState.FAILED): From f513941c09075a4d2ceef8f4db5a107cec96af94 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 29 Nov 2022 13:07:48 +0000 Subject: [PATCH 246/481] test: update test for disconnected_retry_timeout --- test/ably/realtimeconnection_test.py | 42 +++++++++++++++++----------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 2ee39b82..f495093a 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -168,24 +168,32 @@ async def new_close_transport(): assert exception.value.status_code == 504 async def test_disconnected_retry_timeout(self): - ably = await RestSetup.get_ably_realtime(realtime_request_timeout=0.001, - disconnected_retry_timeout=2000, auto_connect=False) - state_changes = [] + ably = await RestSetup.get_ably_realtime(disconnected_retry_timeout=2000, auto_connect=False) + original_connect = ably.connection.connection_manager._connect + call_count = 0 + test_future = asyncio.Future() + test_exception = Exception() + + # intercept the library connection mechanism to fail the first two connection attempts + async def new_connect(): + nonlocal call_count + if call_count < 2: + call_count += 1 + raise test_exception + else: + await original_connect() + test_future.set_result(None) + + ably.connection.connection_manager._connect = new_connect + + with pytest.raises(Exception) as exception: + await ably.connect() - def on_state_change(state_change): - state_changes.append(state_change) + assert ably.connection.state == ConnectionState.DISCONNECTED + assert exception.value == test_exception - ably.connection.on(on_state_change) + await test_future + + assert ably.connection.state == ConnectionState.CONNECTED - with pytest.raises(AblyException) as exception: - await ably.connect() - assert exception.value.code == 50003 - assert exception.value.status_code == 504 - assert ably.connection.state == ConnectionState.DISCONNECTED - # 2 state changes happens per retry. - # Retry timeout of 2 secs, will retry connection twice in 3 and/or 4 seconds, resulting in 4 state changes - await asyncio.sleep(4) - assert len(state_changes) == 4 - assert state_changes[0].previous == ConnectionState.CONNECTING - assert state_changes[0].current == ConnectionState.DISCONNECTED await ably.close() From 262a4899db18d28709bb6fe1e6860cb18a2a1a1d Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 6 Dec 2022 10:35:19 +0000 Subject: [PATCH 247/481] test: add fixture for connection to unroutable host --- test/ably/realtimeconnection_test.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index f495093a..1c8ec292 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -197,3 +197,12 @@ async def new_connect(): assert ably.connection.state == ConnectionState.CONNECTED await ably.close() + + async def test_unroutable_host(self): + ably = await RestSetup.get_ably_realtime(realtime_host="10.255.255.1") + with pytest.raises(AblyException) as exception: + await ably.connect() + assert exception.value.code == 50003 + assert exception.value.status_code == 504 + assert ably.connection.state == ConnectionState.DISCONNECTED + assert ably.connection.error_reason == exception.value From 85ec2c541103f44f4d60ddd148642a0f9c7f066a Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 6 Dec 2022 10:36:38 +0000 Subject: [PATCH 248/481] fix: remove errant variable shadowing for ws_url --- ably/realtime/websockettransport.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index 6451235f..96adc617 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -44,11 +44,6 @@ async def connect(self): params = {"key": self.connection_manager.ably.key, "v": protocol_version} query_params = urllib.parse.urlencode(params) ws_url = (f'wss://{self.connection_manager.options.get_realtime_host()}?{query_params}') - - headers = HttpUtils.default_headers() - host = self.connection_manager.options.get_realtime_host() - key = self.connection_manager.ably.key - ws_url = f'wss://{host}?key={key}' log.info(f'connect(): attempting to connect to {ws_url}') self.ws_connect_task = asyncio.create_task(self.ws_connect(ws_url, headers)) self.ws_connect_task.add_done_callback(self.on_ws_connect_done) From 3d82928577043f8bf249833092e6834b9befc7f9 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 6 Dec 2022 10:38:58 +0000 Subject: [PATCH 249/481] refactor: wrap websocket opening errors in AblyExceptions --- ably/realtime/websockettransport.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index 96adc617..7832ed7d 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -7,8 +7,9 @@ import urllib.parse from ably.http.httputils import HttpUtils from ably.transport.defaults import Defaults +from ably.util.exceptions import AblyException from websockets.client import WebSocketClientProtocol, connect as ws_connect -from websockets.exceptions import ConnectionClosedOK +from websockets.exceptions import ConnectionClosedOK, WebSocketException if TYPE_CHECKING: from ably.realtime.connection import ConnectionManager @@ -57,12 +58,15 @@ def on_ws_connect_done(self, task: asyncio.Task): return async def ws_connect(self, ws_url, headers): - async with ws_connect(ws_url, extra_headers=headers) as websocket: - log.info(f'ws_connect(): connection established to {ws_url}') - self.websocket = websocket - self.read_loop = self.connection_manager.options.loop.create_task(self.ws_read_loop()) - self.read_loop.add_done_callback(self.on_read_loop_done) - await self.read_loop + try: + async with ws_connect(ws_url, extra_headers=headers) as websocket: + log.info(f'ws_connect(): connection established to {ws_url}') + self.websocket = websocket + self.read_loop = self.connection_manager.options.loop.create_task(self.ws_read_loop()) + self.read_loop.add_done_callback(self.on_read_loop_done) + await self.read_loop + except WebSocketException as e: + raise AblyException(f'Error opening websocket connection: {e.message}', 400, 40000) async def ws_read_loop(self): while True: From a2308eed502e4c2c79df08c709635e2765e362ae Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 6 Dec 2022 10:39:25 +0000 Subject: [PATCH 250/481] refactor: ProtocolMessageAction enum ascending order --- ably/realtime/websockettransport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index 7832ed7d..a6b33000 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -20,9 +20,9 @@ class ProtocolMessageAction(IntEnum): HEARTBEAT = 0 CONNECTED = 4 - ERROR = 9 CLOSE = 7 CLOSED = 8 + ERROR = 9 ATTACH = 10 ATTACHED = 11 DETACH = 12 From 7e7476ad8284731c751836533fa774a291ea4190 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 6 Dec 2022 10:41:02 +0000 Subject: [PATCH 251/481] refactor: handle socket.gaierror from websocket connection --- ably/realtime/websockettransport.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index a6b33000..485480b6 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -4,6 +4,7 @@ from enum import IntEnum import json import logging +import socket import urllib.parse from ably.http.httputils import HttpUtils from ably.transport.defaults import Defaults @@ -65,7 +66,7 @@ async def ws_connect(self, ws_url, headers): self.read_loop = self.connection_manager.options.loop.create_task(self.ws_read_loop()) self.read_loop.add_done_callback(self.on_read_loop_done) await self.read_loop - except WebSocketException as e: + except (WebSocketException, socket.gaierror) as e: raise AblyException(f'Error opening websocket connection: {e.message}', 400, 40000) async def ws_read_loop(self): From d1422fdac7bd85fce7333c5aed23e801db2e2008 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 6 Dec 2022 10:54:41 +0000 Subject: [PATCH 252/481] refactor: finish connection attempt on ws opening failure --- ably/realtime/websockettransport.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index 485480b6..3aeafccf 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -55,8 +55,11 @@ def on_ws_connect_done(self, task: asyncio.Task): exception = task.exception() except asyncio.CancelledError as e: exception = e - if isinstance(exception, ConnectionClosedOK): + if exception is None or isinstance(exception, ConnectionClosedOK): return + connected_future = asyncio.Future() + connected_future.set_exception(exception) + self.connection_manager.on_connection_attempt_done(connected_future) async def ws_connect(self, ws_url, headers): try: @@ -67,7 +70,7 @@ async def ws_connect(self, ws_url, headers): self.read_loop.add_done_callback(self.on_read_loop_done) await self.read_loop except (WebSocketException, socket.gaierror) as e: - raise AblyException(f'Error opening websocket connection: {e.message}', 400, 40000) + raise AblyException(f'Error opening websocket connection: {e}', 400, 40000) async def ws_read_loop(self): while True: From 02e9cefd12bc83c019569d6752888b2aea0e42f3 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 6 Dec 2022 10:55:02 +0000 Subject: [PATCH 253/481] test: add test fixture for connection with invalid host --- test/ably/realtimeconnection_test.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 1c8ec292..86883f25 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -206,3 +206,12 @@ async def test_unroutable_host(self): assert exception.value.status_code == 504 assert ably.connection.state == ConnectionState.DISCONNECTED assert ably.connection.error_reason == exception.value + + async def test_invalid_host(self): + ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost") + with pytest.raises(AblyException) as exception: + await ably.connect() + assert exception.value.code == 40000 + assert exception.value.status_code == 400 + assert ably.connection.state == ConnectionState.DISCONNECTED + assert ably.connection.error_reason == exception.value From d47e8460bf8fd7cfcacae6eef3476daacacad4ce Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Fri, 16 Dec 2022 10:21:29 +0000 Subject: [PATCH 254/481] add realtime_hosts option to realtime client --- ably/realtime/realtime.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index c9c73dd4..75e3270a 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -61,6 +61,9 @@ def __init__(self, key=None, loop=None, **kwargs): disconnected_retry_timeout: float If the connection is still in the DISCONNECTED state after this delay, the client library will attempt to reconnect automatically. The default is 15 seconds. + fallback_hosts: list[str] + An array of fallback hosts to be used in the case of an error necessitating the use of an alternative host. + If you have been provided a set of custom fallback hosts by Ably, please specify them here. Raises ------ ValueError From 65d3ff1cd7595cfaa1f3e953d294eccc9abd8f56 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 6 Dec 2022 13:37:39 +0000 Subject: [PATCH 255/481] implement connection_state_ttl --- ably/realtime/connection.py | 31 +++++++++++++++++++++++++++++-- ably/transport/defaults.py | 4 +++- ably/types/options.py | 25 ++++++++++++++++++++----- 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index b79898da..4082d5e0 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -21,6 +21,7 @@ class ConnectionState(str, Enum): CLOSING = 'closing' CLOSED = 'closed' FAILED = 'failed' + SUSPENDED = "suspended" @dataclass @@ -131,13 +132,27 @@ def __init__(self, realtime, initial_state): self.__ping_future = None self.__timeout_in_secs = self.options.realtime_request_timeout / 1000 self.transport: WebSocketTransport | None = None + self.__ttl_task = None + self.__retry_task = None super().__init__() def enact_state_change(self, state, reason=None): current_state = self.__state self.__state = state + print(self.state, "enact") + if self.state == ConnectionState.DISCONNECTED: + if not self.__ttl_task or self.__ttl_task.done(): + self.__ttl_task = asyncio.create_task(self.__connection_state_ttl()) self._emit('connectionstate', ConnectionStateChange(current_state, state, reason)) + async def __connection_state_ttl(self): + await asyncio.sleep(self.ably.options.connection_state_ttl) + exception = AblyException("Exceeded connectionStateTtl while in DISCONNECTED state", 504, 50003) + self.enact_state_change(ConnectionState.SUSPENDED, exception) + if self.__retry_task: + self.__retry_task.cancel() + asyncio.create_task(self.retry_connection_attempt()) + async def connect(self): if not self.__connected_future: self.__connected_future = asyncio.Future() @@ -145,11 +160,13 @@ async def connect(self): await self.__connected_future def try_connect(self): + print("erm", self.__state) task = asyncio.create_task(self._connect()) task.add_done_callback(self.on_connection_attempt_done) async def _connect(self): if self.__state == ConnectionState.CONNECTED: + self.__ttl_task.cancel() return if self.__state == ConnectionState.CONNECTING: @@ -177,14 +194,22 @@ def on_connection_attempt_done(self, task): if self.__state in (ConnectionState.CLOSED, ConnectionState.FAILED): return if self.__state != ConnectionState.DISCONNECTED: + print("howdy") if self.__connected_future: self.__connected_future.set_exception(exception) self.__connected_future = None self.enact_state_change(ConnectionState.DISCONNECTED, exception) - asyncio.create_task(self.retry_connection_attempt()) + self.__retry_task = asyncio.create_task(self.retry_connection_attempt()) async def retry_connection_attempt(self): - await asyncio.sleep(self.ably.options.disconnected_retry_timeout / 1000) + print("retrying", self.__state) + if self.state == ConnectionState.SUSPENDED: + print("suspended") + retry_timeout = self.ably.options.suspended_retry_timeout / 1000 + else: + print("not yet") + retry_timeout = self.ably.options.disconnected_retry_timeout / 1000 + await asyncio.sleep(retry_timeout) self.try_connect() async def close(self): @@ -272,6 +297,8 @@ async def on_protocol_message(self, msg): self.__connected_future = None else: log.warn('CONNECTED message received but connected_future not set') + if self.__ttl_task: + self.__ttl_task.cancel() self.enact_state_change(ConnectionState.CONNECTED) if action == ProtocolMessageAction.ERROR: # ERROR error = msg["error"] diff --git a/ably/transport/defaults.py b/ably/transport/defaults.py index 6b0fec88..915d3ef8 100644 --- a/ably/transport/defaults.py +++ b/ably/transport/defaults.py @@ -20,7 +20,9 @@ class Defaults: comet_recv_timeout = 90000 comet_send_timeout = 10000 realtime_request_timeout = 10000 - disconnected_retry_timeout = 1500 + disconnected_retry_timeout = 15000 + connection_state_ttl = 120000 + suspended_retry_timeout = 30000 transports = [] # ["web_socket", "comet"] diff --git a/ably/types/options.py b/ably/types/options.py index 0a926992..e4d8aef1 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -9,14 +9,13 @@ class Options(AuthOptions): - def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, - realtime_host=None, port=0, tls_port=0, use_binary_protocol=True, - queue_messages=False, recover=False, environment=None, + def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realtime_host=None, port=0, + tls_port=0, use_binary_protocol=True, queue_messages=False, recover=False, environment=None, http_open_timeout=None, http_request_timeout=None, realtime_request_timeout=None, http_max_retry_count=None, http_max_retry_duration=None, fallback_hosts=None, fallback_hosts_use_default=None, fallback_retry_timeout=None, disconnected_retry_timeout=None, - idempotent_rest_publishing=None, loop=None, auto_connect=True, - **kwargs): + idempotent_rest_publishing=None, loop=None, auto_connect=True, connection_state_ttl=None, + suspended_retry_timeout=None, **kwargs): super().__init__(**kwargs) # TODO check these defaults @@ -29,6 +28,12 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, if disconnected_retry_timeout is None: disconnected_retry_timeout = Defaults.disconnected_retry_timeout + if connection_state_ttl is None: + connection_state_ttl = Defaults.connection_state_ttl + + if suspended_retry_timeout is None: + suspended_retry_timeout = Defaults.suspended_retry_timeout + if environment is not None and rest_host is not None: raise ValueError('specify rest_host or environment, not both') @@ -62,6 +67,8 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, self.__idempotent_rest_publishing = idempotent_rest_publishing self.__loop = loop self.__auto_connect = auto_connect + self.__connection_state_ttl = connection_state_ttl + self.__suspended_retry_timeout = suspended_retry_timeout self.__rest_hosts = self.__get_rest_hosts() self.__realtime_hosts = self.__get_realtime_hosts() @@ -214,6 +221,14 @@ def loop(self): def auto_connect(self): return self.__auto_connect + @property + def connection_state_ttl(self): + return self.__connection_state_ttl + + @property + def suspended_retry_timeout(self): + return self.__suspended_retry_timeout + def __get_rest_hosts(self): """ Return the list of hosts as they should be tried. First comes the main From 25ab27a061b6353dc37d8b2200d4a7ad38bb7587 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 7 Dec 2022 15:06:42 +0000 Subject: [PATCH 256/481] override ttl with connection details ttl --- ably/realtime/connection.py | 16 +++++++++------- ably/types/options.py | 4 ++++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 4082d5e0..a56ea69b 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -134,19 +134,21 @@ def __init__(self, realtime, initial_state): self.transport: WebSocketTransport | None = None self.__ttl_task = None self.__retry_task = None + self.__connection_details = None super().__init__() def enact_state_change(self, state, reason=None): current_state = self.__state self.__state = state - print(self.state, "enact") if self.state == ConnectionState.DISCONNECTED: if not self.__ttl_task or self.__ttl_task.done(): self.__ttl_task = asyncio.create_task(self.__connection_state_ttl()) self._emit('connectionstate', ConnectionStateChange(current_state, state, reason)) async def __connection_state_ttl(self): - await asyncio.sleep(self.ably.options.connection_state_ttl) + if self.__connection_details: + self.ably.options.connection_state_ttl = self.__connection_details["connectionStateTtl"] + await asyncio.sleep(self.ably.options.connection_state_ttl / 1000) exception = AblyException("Exceeded connectionStateTtl while in DISCONNECTED state", 504, 50003) self.enact_state_change(ConnectionState.SUSPENDED, exception) if self.__retry_task: @@ -160,7 +162,6 @@ async def connect(self): await self.__connected_future def try_connect(self): - print("erm", self.__state) task = asyncio.create_task(self._connect()) task.add_done_callback(self.on_connection_attempt_done) @@ -194,7 +195,6 @@ def on_connection_attempt_done(self, task): if self.__state in (ConnectionState.CLOSED, ConnectionState.FAILED): return if self.__state != ConnectionState.DISCONNECTED: - print("howdy") if self.__connected_future: self.__connected_future.set_exception(exception) self.__connected_future = None @@ -202,12 +202,9 @@ def on_connection_attempt_done(self, task): self.__retry_task = asyncio.create_task(self.retry_connection_attempt()) async def retry_connection_attempt(self): - print("retrying", self.__state) if self.state == ConnectionState.SUSPENDED: - print("suspended") retry_timeout = self.ably.options.suspended_retry_timeout / 1000 else: - print("not yet") retry_timeout = self.ably.options.disconnected_retry_timeout / 1000 await asyncio.sleep(retry_timeout) self.try_connect() @@ -299,6 +296,7 @@ async def on_protocol_message(self, msg): log.warn('CONNECTED message received but connected_future not set') if self.__ttl_task: self.__ttl_task.cancel() + self.__connection_details = msg['connectionDetails'] self.enact_state_change(ConnectionState.CONNECTED) if action == ProtocolMessageAction.ERROR: # ERROR error = msg["error"] @@ -337,3 +335,7 @@ def ably(self): @property def state(self): return self.__state + + @property + def connection_details(self): + return self.__connection_details diff --git a/ably/types/options.py b/ably/types/options.py index e4d8aef1..70b79b40 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -225,6 +225,10 @@ def auto_connect(self): def connection_state_ttl(self): return self.__connection_state_ttl + @connection_state_ttl.setter + def connection_state_ttl(self, value): + self.__connection_state_ttl = value + @property def suspended_retry_timeout(self): return self.__suspended_retry_timeout From 29439b9c67182710ba670e08dadca99b71ae9c2d Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Thu, 8 Dec 2022 12:45:07 +0000 Subject: [PATCH 257/481] update suspended state behaviour --- ably/realtime/connection.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index a56ea69b..ec4a647b 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -135,12 +135,13 @@ def __init__(self, realtime, initial_state): self.__ttl_task = None self.__retry_task = None self.__connection_details = None + self.__in_suspended_state = False super().__init__() def enact_state_change(self, state, reason=None): current_state = self.__state self.__state = state - if self.state == ConnectionState.DISCONNECTED: + if self.__state == ConnectionState.DISCONNECTED: if not self.__ttl_task or self.__ttl_task.done(): self.__ttl_task = asyncio.create_task(self.__connection_state_ttl()) self._emit('connectionstate', ConnectionStateChange(current_state, state, reason)) @@ -151,9 +152,10 @@ async def __connection_state_ttl(self): await asyncio.sleep(self.ably.options.connection_state_ttl / 1000) exception = AblyException("Exceeded connectionStateTtl while in DISCONNECTED state", 504, 50003) self.enact_state_change(ConnectionState.SUSPENDED, exception) + self.__in_suspended_state = True if self.__retry_task: self.__retry_task.cancel() - asyncio.create_task(self.retry_connection_attempt()) + self.__retry_task = asyncio.create_task(self.retry_connection_attempt()) async def connect(self): if not self.__connected_future: @@ -167,7 +169,8 @@ def try_connect(self): async def _connect(self): if self.__state == ConnectionState.CONNECTED: - self.__ttl_task.cancel() + if self.__ttl_task: + self.__ttl_task.cancel() return if self.__state == ConnectionState.CONNECTING: @@ -198,14 +201,18 @@ def on_connection_attempt_done(self, task): if self.__connected_future: self.__connected_future.set_exception(exception) self.__connected_future = None - self.enact_state_change(ConnectionState.DISCONNECTED, exception) + if self.__in_suspended_state: + self.enact_state_change(ConnectionState.SUSPENDED, exception) + else: + self.enact_state_change(ConnectionState.DISCONNECTED, exception) self.__retry_task = asyncio.create_task(self.retry_connection_attempt()) async def retry_connection_attempt(self): - if self.state == ConnectionState.SUSPENDED: + if self.__in_suspended_state: retry_timeout = self.ably.options.suspended_retry_timeout / 1000 else: retry_timeout = self.ably.options.disconnected_retry_timeout / 1000 + await asyncio.sleep(retry_timeout) self.try_connect() @@ -294,6 +301,7 @@ async def on_protocol_message(self, msg): self.__connected_future = None else: log.warn('CONNECTED message received but connected_future not set') + self.__in_suspended_state = False if self.__ttl_task: self.__ttl_task.cancel() self.__connection_details = msg['connectionDetails'] From b9400c782d0dcd9d298b6eb23bac3cc74fe96e82 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Fri, 9 Dec 2022 14:15:46 +0000 Subject: [PATCH 258/481] add test for connection state ttl --- ably/realtime/connection.py | 6 ++++-- ably/realtime/realtime.py | 11 +++++++++-- test/ably/realtimeconnection_test.py | 23 +++++++++++++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index ec4a647b..613c954c 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -212,7 +212,6 @@ async def retry_connection_attempt(self): retry_timeout = self.ably.options.suspended_retry_timeout / 1000 else: retry_timeout = self.ably.options.disconnected_retry_timeout / 1000 - await asyncio.sleep(retry_timeout) self.try_connect() @@ -242,7 +241,10 @@ async def close(self): log.warning('ConnectionManager: called close with no connected transport') self.enact_state_change(ConnectionState.CLOSED) if self.transport and self.transport.ws_connect_task is not None: - await self.transport.ws_connect_task + try: + await self.transport.ws_connect_task + except AblyException as e: + log.warning(f'Connection error encountered while closing: {e}') async def connect_impl(self): self.transport = WebSocketTransport(self) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 75e3270a..9b744217 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -62,8 +62,15 @@ def __init__(self, key=None, loop=None, **kwargs): If the connection is still in the DISCONNECTED state after this delay, the client library will attempt to reconnect automatically. The default is 15 seconds. fallback_hosts: list[str] - An array of fallback hosts to be used in the case of an error necessitating the use of an alternative host. - If you have been provided a set of custom fallback hosts by Ably, please specify them here. + An array of fallback hosts to be used in the case of an error necessitating the use of an + alternative host. If you have been provided a set of custom fallback hosts by Ably, please specify + them here. + connection_state_ttl: float + The duration that Ably will persist the connection state for when a Realtime client is abruptly + disconnected. + suspended_retry_timeout: float + When the connection enters the SUSPENDED state, after this delay, if the state is still SUSPENDED, + the client library attempts to reconnect automatically. The default is 30 seconds. Raises ------ ValueError diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 86883f25..3521e6bb 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -206,6 +206,7 @@ async def test_unroutable_host(self): assert exception.value.status_code == 504 assert ably.connection.state == ConnectionState.DISCONNECTED assert ably.connection.error_reason == exception.value + await ably.close() async def test_invalid_host(self): ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost") @@ -215,3 +216,25 @@ async def test_invalid_host(self): assert exception.value.status_code == 400 assert ably.connection.state == ConnectionState.DISCONNECTED assert ably.connection.error_reason == exception.value + await ably.close() + + async def test_connection_state_ttl(self): + ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost", connection_state_ttl=2000) + changes = [] + suspended_future = asyncio.Future() + + def on_state_change(state_change): + changes.append(state_change) + if state_change.current == ConnectionState.SUSPENDED: + suspended_future.set_result(None) + with pytest.raises(AblyException) as exception: + await ably.connect() + ably.connection.on(on_state_change) + assert exception.value.code == 40000 + assert exception.value.status_code == 400 + assert ably.connection.state == ConnectionState.DISCONNECTED + await suspended_future + assert ably.connection.state == changes[-1].current + assert ably.connection.state == ConnectionState.SUSPENDED + assert ably.connection.error_reason == changes[-1].reason + await ably.close() From 10fe9878a6b7a78cd275847ab7885002216cffc2 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Thu, 5 Jan 2023 15:33:48 +0000 Subject: [PATCH 259/481] implememt review --- ably/realtime/connection.py | 11 ++++++++--- test/ably/realtimeconnection_test.py | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 613c954c..a0fc0b75 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -121,6 +121,10 @@ def state(self, value): def connection_manager(self): return self.__connection_manager + @property + def connection_details(self): + return self.__connection_manager.connection_details + class ConnectionManager(EventEmitter): def __init__(self, realtime, initial_state): @@ -143,15 +147,16 @@ def enact_state_change(self, state, reason=None): self.__state = state if self.__state == ConnectionState.DISCONNECTED: if not self.__ttl_task or self.__ttl_task.done(): - self.__ttl_task = asyncio.create_task(self.__connection_state_ttl()) + self.__ttl_task = asyncio.create_task(self.__start_suspended_timer()) self._emit('connectionstate', ConnectionStateChange(current_state, state, reason)) - async def __connection_state_ttl(self): + async def __start_suspended_timer(self): if self.__connection_details: self.ably.options.connection_state_ttl = self.__connection_details["connectionStateTtl"] await asyncio.sleep(self.ably.options.connection_state_ttl / 1000) exception = AblyException("Exceeded connectionStateTtl while in DISCONNECTED state", 504, 50003) self.enact_state_change(ConnectionState.SUSPENDED, exception) + self.__connection_details = None self.__in_suspended_state = True if self.__retry_task: self.__retry_task.cancel() @@ -306,7 +311,7 @@ async def on_protocol_message(self, msg): self.__in_suspended_state = False if self.__ttl_task: self.__ttl_task.cancel() - self.__connection_details = msg['connectionDetails'] + self.__connection_details = msg.get('connectionDetails') self.enact_state_change(ConnectionState.CONNECTED) if action == ProtocolMessageAction.ERROR: # ERROR error = msg["error"] diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 3521e6bb..806f0097 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -236,5 +236,6 @@ def on_state_change(state_change): await suspended_future assert ably.connection.state == changes[-1].current assert ably.connection.state == ConnectionState.SUSPENDED + assert ably.connection.connection_details is None assert ably.connection.error_reason == changes[-1].reason await ably.close() From d9743e85b823f7fa2ac47ba0bd8c10c7dfdf91c2 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 9 Jan 2023 17:08:45 +0000 Subject: [PATCH 260/481] review: refactor connection details --- ably/realtime/connection.py | 28 +++++++++++++++++++--------- ably/types/options.py | 7 +++---- test/ably/realtimeconnection_test.py | 2 +- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index a0fc0b75..bd1c1d09 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -31,6 +31,18 @@ class ConnectionStateChange: reason: Optional[AblyException] = None +@dataclass +class ConnectionDetails: + connectionStateTtl: int + + def __init__(self, connection_state_ttl: int): + self.connectionStateTtl = connection_state_ttl + + @staticmethod + def from_dict(json_dict: dict): + return ConnectionDetails(json_dict.get('connectionStateTtl')) + + class Connection(EventEmitter): """Ably Realtime Connection @@ -139,7 +151,7 @@ def __init__(self, realtime, initial_state): self.__ttl_task = None self.__retry_task = None self.__connection_details = None - self.__in_suspended_state = False + self.__fail_state = ConnectionState.DISCONNECTED super().__init__() def enact_state_change(self, state, reason=None): @@ -152,12 +164,12 @@ def enact_state_change(self, state, reason=None): async def __start_suspended_timer(self): if self.__connection_details: - self.ably.options.connection_state_ttl = self.__connection_details["connectionStateTtl"] + self.ably.options.connection_state_ttl = self.__connection_details.connectionStateTtl await asyncio.sleep(self.ably.options.connection_state_ttl / 1000) exception = AblyException("Exceeded connectionStateTtl while in DISCONNECTED state", 504, 50003) self.enact_state_change(ConnectionState.SUSPENDED, exception) self.__connection_details = None - self.__in_suspended_state = True + self.__fail_state = ConnectionState.SUSPENDED if self.__retry_task: self.__retry_task.cancel() self.__retry_task = asyncio.create_task(self.retry_connection_attempt()) @@ -174,8 +186,6 @@ def try_connect(self): async def _connect(self): if self.__state == ConnectionState.CONNECTED: - if self.__ttl_task: - self.__ttl_task.cancel() return if self.__state == ConnectionState.CONNECTING: @@ -206,14 +216,14 @@ def on_connection_attempt_done(self, task): if self.__connected_future: self.__connected_future.set_exception(exception) self.__connected_future = None - if self.__in_suspended_state: + if self.__fail_state == ConnectionState.SUSPENDED: self.enact_state_change(ConnectionState.SUSPENDED, exception) else: self.enact_state_change(ConnectionState.DISCONNECTED, exception) self.__retry_task = asyncio.create_task(self.retry_connection_attempt()) async def retry_connection_attempt(self): - if self.__in_suspended_state: + if self.__fail_state == ConnectionState.SUSPENDED: retry_timeout = self.ably.options.suspended_retry_timeout / 1000 else: retry_timeout = self.ably.options.disconnected_retry_timeout / 1000 @@ -308,10 +318,10 @@ async def on_protocol_message(self, msg): self.__connected_future = None else: log.warn('CONNECTED message received but connected_future not set') - self.__in_suspended_state = False + self.__fail_state == ConnectionState.DISCONNECTED if self.__ttl_task: self.__ttl_task.cancel() - self.__connection_details = msg.get('connectionDetails') + self.__connection_details = ConnectionDetails.from_dict(msg["connectionDetails"]) self.enact_state_change(ConnectionState.CONNECTED) if action == ProtocolMessageAction.ERROR: # ERROR error = msg["error"] diff --git a/ably/types/options.py b/ably/types/options.py index 70b79b40..c85f1c05 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -14,8 +14,8 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti http_open_timeout=None, http_request_timeout=None, realtime_request_timeout=None, http_max_retry_count=None, http_max_retry_duration=None, fallback_hosts=None, fallback_hosts_use_default=None, fallback_retry_timeout=None, disconnected_retry_timeout=None, - idempotent_rest_publishing=None, loop=None, auto_connect=True, connection_state_ttl=None, - suspended_retry_timeout=None, **kwargs): + idempotent_rest_publishing=None, loop=None, auto_connect=True, suspended_retry_timeout=None, + **kwargs): super().__init__(**kwargs) # TODO check these defaults @@ -28,8 +28,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti if disconnected_retry_timeout is None: disconnected_retry_timeout = Defaults.disconnected_retry_timeout - if connection_state_ttl is None: - connection_state_ttl = Defaults.connection_state_ttl + connection_state_ttl = Defaults.connection_state_ttl if suspended_retry_timeout is None: suspended_retry_timeout = Defaults.suspended_retry_timeout diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 806f0097..6045d7f7 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -219,7 +219,7 @@ async def test_invalid_host(self): await ably.close() async def test_connection_state_ttl(self): - ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost", connection_state_ttl=2000) + ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost") changes = [] suspended_future = asyncio.Future() From d20d05b831d8f21d13b88e5fe961a4558dd3a526 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 11 Jan 2023 12:48:27 +0000 Subject: [PATCH 261/481] refactor and update test --- ably/realtime/connection.py | 7 +++---- test/ably/realtimeconnection_test.py | 3 +++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index bd1c1d09..b19458e7 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -216,10 +216,7 @@ def on_connection_attempt_done(self, task): if self.__connected_future: self.__connected_future.set_exception(exception) self.__connected_future = None - if self.__fail_state == ConnectionState.SUSPENDED: - self.enact_state_change(ConnectionState.SUSPENDED, exception) - else: - self.enact_state_change(ConnectionState.DISCONNECTED, exception) + self.enact_state_change(self.__fail_state, exception) self.__retry_task = asyncio.create_task(self.retry_connection_attempt()) async def retry_connection_attempt(self): @@ -255,6 +252,8 @@ async def close(self): else: log.warning('ConnectionManager: called close with no connected transport') self.enact_state_change(ConnectionState.CLOSED) + if self.__ttl_task and not self.__ttl_task.done(): + self.__ttl_task.cancel() if self.transport and self.transport.ws_connect_task is not None: try: await self.transport.ws_connect_task diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 6045d7f7..f2c785b3 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -4,6 +4,7 @@ from ably.util.exceptions import AblyAuthException, AblyException from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase +from ably.transport.defaults import Defaults class TestRealtimeAuth(BaseAsyncTestCase): @@ -219,6 +220,7 @@ async def test_invalid_host(self): await ably.close() async def test_connection_state_ttl(self): + Defaults.connection_state_ttl = 100 ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost") changes = [] suspended_future = asyncio.Future() @@ -239,3 +241,4 @@ def on_state_change(state_change): assert ably.connection.connection_details is None assert ably.connection.error_reason == changes[-1].reason await ably.close() + Defaults.connection_state_ttl = 120000 From 4414f015a02c888a060dfb4266c8b7c1b11ed08a Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Thu, 22 Dec 2022 15:27:50 +0000 Subject: [PATCH 262/481] add connection check function --- ably/realtime/connection.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index b19458e7..38fc5fef 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -1,6 +1,7 @@ import functools import logging import asyncio +import httpx from ably.realtime.websockettransport import WebSocketTransport, ProtocolMessageAction from ably.util.exceptions import AblyAuthException, AblyException from ably.util.eventemitter import EventEmitter @@ -202,6 +203,13 @@ async def _connect(self): self.enact_state_change(ConnectionState.CONNECTING) await self.connect_impl() + def check_connection(self): + try: + response = httpx.get("https://internet-up.ably-realtime.com/is-the-internet-up.txt") + return response.status_code == 200 and response.text == "yes" + finally: + return False + def on_connection_attempt_done(self, task): try: exception = task.exception() From 7f9c27abf5bb0a330df63d00a52cf4fc9db5005c Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Wed, 4 Jan 2023 13:14:44 +0000 Subject: [PATCH 263/481] fix missing newline at the end of the internet up check response --- ably/realtime/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 38fc5fef..0ecc1a4f 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -206,7 +206,7 @@ async def _connect(self): def check_connection(self): try: response = httpx.get("https://internet-up.ably-realtime.com/is-the-internet-up.txt") - return response.status_code == 200 and response.text == "yes" + return response.status_code == 200 and response.text == "yes\n" finally: return False From 5d7d371d5b72351f7fe17999ad9bbc533b88b59e Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Fri, 6 Jan 2023 10:40:54 +0000 Subject: [PATCH 264/481] change to FAILED state when unable to connect --- ably/realtime/connection.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 0ecc1a4f..59e12d13 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -206,7 +206,7 @@ async def _connect(self): def check_connection(self): try: response = httpx.get("https://internet-up.ably-realtime.com/is-the-internet-up.txt") - return response.status_code == 200 and response.text == "yes\n" + return response.status_code == 200 and "yes" in response.text finally: return False @@ -233,7 +233,11 @@ async def retry_connection_attempt(self): else: retry_timeout = self.ably.options.disconnected_retry_timeout / 1000 await asyncio.sleep(retry_timeout) - self.try_connect() + if self.check_connection(): + self.try_connect() + else: + exception = AblyException("Unable to connect (network unreachable)", 80003, 404) + self.enact_state_change(ConnectionState.FAILED, exception) async def close(self): if self.__state in (ConnectionState.CLOSED, ConnectionState.INITIALIZED, ConnectionState.FAILED): From 023a301e14d7d5a5b3a47f773ceeb196096d8d32 Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Fri, 6 Jan 2023 14:20:39 +0000 Subject: [PATCH 265/481] fix disconnected retry timeout test hanging --- ably/realtime/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 59e12d13..b2dc050f 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -207,7 +207,7 @@ def check_connection(self): try: response = httpx.get("https://internet-up.ably-realtime.com/is-the-internet-up.txt") return response.status_code == 200 and "yes" in response.text - finally: + except httpx.HTTPError: return False def on_connection_attempt_done(self, task): From 415b2395dc1a2bee6887433e6f3f73c038649b31 Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Wed, 11 Jan 2023 12:06:04 +0000 Subject: [PATCH 266/481] transition to fail state when network connection check fails --- ably/realtime/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index b2dc050f..6f2b80e0 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -237,7 +237,7 @@ async def retry_connection_attempt(self): self.try_connect() else: exception = AblyException("Unable to connect (network unreachable)", 80003, 404) - self.enact_state_change(ConnectionState.FAILED, exception) + self.enact_state_change(self.__fail_state, exception) async def close(self): if self.__state in (ConnectionState.CLOSED, ConnectionState.INITIALIZED, ConnectionState.FAILED): From 5522671ed3b1a10bc90f0290fe3a9563d3fcda37 Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Wed, 11 Jan 2023 12:48:56 +0000 Subject: [PATCH 267/481] add connectivity_check_url option and default --- ably/realtime/connection.py | 6 ++++-- ably/transport/defaults.py | 1 + ably/types/options.py | 11 ++++++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 6f2b80e0..9a2ce0cd 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -3,6 +3,7 @@ import asyncio import httpx from ably.realtime.websockettransport import WebSocketTransport, ProtocolMessageAction +from ably.transport.defaults import Defaults from ably.util.exceptions import AblyAuthException, AblyException from ably.util.eventemitter import EventEmitter from enum import Enum @@ -205,8 +206,9 @@ async def _connect(self): def check_connection(self): try: - response = httpx.get("https://internet-up.ably-realtime.com/is-the-internet-up.txt") - return response.status_code == 200 and "yes" in response.text + response = httpx.get(self.options.connectivity_check_url) + return response.status_code == 200 and \ + (self.options.connectivity_check_url != Defaults.connectivity_check_url or "yes" in response.text) except httpx.HTTPError: return False diff --git a/ably/transport/defaults.py b/ably/transport/defaults.py index 915d3ef8..04c57031 100644 --- a/ably/transport/defaults.py +++ b/ably/transport/defaults.py @@ -10,6 +10,7 @@ class Defaults: rest_host = "rest.ably.io" realtime_host = "realtime.ably.io" + connectivity_check_url = "https://internet-up.ably-realtime.com/is-the-internet-up.txt" environment = 'production' port = 80 diff --git a/ably/types/options.py b/ably/types/options.py index c85f1c05..7aaab5eb 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -15,7 +15,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti http_max_retry_count=None, http_max_retry_duration=None, fallback_hosts=None, fallback_hosts_use_default=None, fallback_retry_timeout=None, disconnected_retry_timeout=None, idempotent_rest_publishing=None, loop=None, auto_connect=True, suspended_retry_timeout=None, - **kwargs): + connectivity_check_url=None, **kwargs): super().__init__(**kwargs) # TODO check these defaults @@ -28,6 +28,9 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti if disconnected_retry_timeout is None: disconnected_retry_timeout = Defaults.disconnected_retry_timeout + if connectivity_check_url is None: + connectivity_check_url = Defaults.connectivity_check_url + connection_state_ttl = Defaults.connection_state_ttl if suspended_retry_timeout is None: @@ -68,10 +71,12 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti self.__auto_connect = auto_connect self.__connection_state_ttl = connection_state_ttl self.__suspended_retry_timeout = suspended_retry_timeout + self.__connectivity_check_url = connectivity_check_url self.__rest_hosts = self.__get_rest_hosts() self.__realtime_hosts = self.__get_realtime_hosts() + @property def client_id(self): return self.__client_id @@ -232,6 +237,10 @@ def connection_state_ttl(self, value): def suspended_retry_timeout(self): return self.__suspended_retry_timeout + @property + def connectivity_check_url(self): + return self.__connectivity_check_url + def __get_rest_hosts(self): """ Return the list of hosts as they should be tried. First comes the main From ba6bf0a5aad59bc3484c46dc30b8b7023526159e Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Wed, 11 Jan 2023 12:49:01 +0000 Subject: [PATCH 268/481] add retry_connection_attempt tests --- test/ably/realtimeconnection_test.py | 33 ++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index f2c785b3..d98e5ce4 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -199,6 +199,39 @@ async def new_connect(): await ably.close() + async def test_connectivity_check_default(self): + ably = await RestSetup.get_ably_realtime() + # The default connectivity check should return True + assert ably.connection.connection_manager.check_connection() is True + + async def test_connectivity_check_non_default(self): + ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://httpbin.org/status/200") + # A non-default URL should return True with a HTTP OK despite not returning "Yes" in the body + assert ably.connection.connection_manager.check_connection() is True + + async def test_connectivity_check_bad_status(self): + ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://httpbin.org/status/400") + # Should return False when the URL returns a non-2xx response code + assert ably.connection.connection_manager.check_connection() is False + + async def test_retry_connection_attempt(self): + ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://httpbin.org/status/400", + disconnected_retry_timeout=1, auto_connect=False) + test_future = asyncio.Future() + + def on_state_change(change): + if change.current == ConnectionState.DISCONNECTED: + test_future.set_result(change) + + ably.connection.connection_manager.on('connectionstate', on_state_change) + + asyncio.create_task(ably.connection.connection_manager.retry_connection_attempt()) + + state_change = await test_future + + assert state_change.reason.status_code == 80003 + assert state_change.reason.message == "Unable to connect (network unreachable)" + async def test_unroutable_host(self): ably = await RestSetup.get_ably_realtime(realtime_host="10.255.255.1") with pytest.raises(AblyException) as exception: From 1679a8b34dc4596e2be88bdbbebcad21e2d3ea25 Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Wed, 11 Jan 2023 13:23:51 +0000 Subject: [PATCH 269/481] check for all 2xx status codes in check_connection --- ably/realtime/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 9a2ce0cd..f1f9ed08 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -207,7 +207,7 @@ async def _connect(self): def check_connection(self): try: response = httpx.get(self.options.connectivity_check_url) - return response.status_code == 200 and \ + return 200 <= response.status_code < 300 and \ (self.options.connectivity_check_url != Defaults.connectivity_check_url or "yes" in response.text) except httpx.HTTPError: return False From 10406b6f3aff1fad6f9a72ec78a4886e2128401d Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Wed, 11 Jan 2023 13:24:12 +0000 Subject: [PATCH 270/481] add documentation for connectivity_check_url option --- ably/realtime/realtime.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 9b744217..f3a6a71f 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -71,6 +71,11 @@ def __init__(self, key=None, loop=None, **kwargs): suspended_retry_timeout: float When the connection enters the SUSPENDED state, after this delay, if the state is still SUSPENDED, the client library attempts to reconnect automatically. The default is 30 seconds. + connectivity_check_url: string + Override the URL used by the realtime client to check if the internet is available. + In the event of a failure to connect to the primary endpoint, the client will send a + GET request to this URL to check if the internet is available. If this request returns + a success response the client will attempt to connect to a fallback host. Raises ------ ValueError From 82137beb638bd4342da908ecb57dd58c9593252e Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Wed, 11 Jan 2023 13:24:53 +0000 Subject: [PATCH 271/481] remove errant newline --- ably/types/options.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ably/types/options.py b/ably/types/options.py index 7aaab5eb..4d7edfc4 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -76,7 +76,6 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti self.__rest_hosts = self.__get_rest_hosts() self.__realtime_hosts = self.__get_realtime_hosts() - @property def client_id(self): return self.__client_id From 30165fbfa7e48ef08558805c5727d5e07bad3d6b Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Wed, 11 Jan 2023 13:26:00 +0000 Subject: [PATCH 272/481] use echo.ably.io for connectivity url tests --- test/ably/realtimeconnection_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index d98e5ce4..e65e8e85 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -205,17 +205,17 @@ async def test_connectivity_check_default(self): assert ably.connection.connection_manager.check_connection() is True async def test_connectivity_check_non_default(self): - ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://httpbin.org/status/200") + ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://echo.ably.io/respondWith?status=200") # A non-default URL should return True with a HTTP OK despite not returning "Yes" in the body assert ably.connection.connection_manager.check_connection() is True async def test_connectivity_check_bad_status(self): - ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://httpbin.org/status/400") + ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://echo.ably.io/respondWith?status=400") # Should return False when the URL returns a non-2xx response code assert ably.connection.connection_manager.check_connection() is False async def test_retry_connection_attempt(self): - ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://httpbin.org/status/400", + ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://echo.ably.io/respondWith?status=400", disconnected_retry_timeout=1, auto_connect=False) test_future = asyncio.Future() From f7d7512cb9baefa038fcfcad1872f00580f8e4e3 Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Wed, 11 Jan 2023 13:33:47 +0000 Subject: [PATCH 273/481] fix line too long linting on realtimeconnection_test.py --- test/ably/realtimeconnection_test.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index e65e8e85..c9e59360 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -205,18 +205,21 @@ async def test_connectivity_check_default(self): assert ably.connection.connection_manager.check_connection() is True async def test_connectivity_check_non_default(self): - ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://echo.ably.io/respondWith?status=200") + ably = await RestSetup.get_ably_realtime( + connectivity_check_url="https://echo.ably.io/respondWith?status=200") # A non-default URL should return True with a HTTP OK despite not returning "Yes" in the body assert ably.connection.connection_manager.check_connection() is True async def test_connectivity_check_bad_status(self): - ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://echo.ably.io/respondWith?status=400") + ably = await RestSetup.get_ably_realtime( + connectivity_check_url="https://echo.ably.io/respondWith?status=400") # Should return False when the URL returns a non-2xx response code assert ably.connection.connection_manager.check_connection() is False async def test_retry_connection_attempt(self): - ably = await RestSetup.get_ably_realtime(connectivity_check_url="https://echo.ably.io/respondWith?status=400", - disconnected_retry_timeout=1, auto_connect=False) + ably = await RestSetup.get_ably_realtime( + connectivity_check_url="https://echo.ably.io/respondWith?status=400", disconnected_retry_timeout=1, + auto_connect=False) test_future = asyncio.Future() def on_state_change(change): From 7f18bee73d888489c4c7e3c26dbcb0777a812273 Mon Sep 17 00:00:00 2001 From: Peter Maguire <peter.maguire@ably.com> Date: Thu, 12 Jan 2023 18:12:40 +0000 Subject: [PATCH 274/481] update for rebase --- poetry.lock | 13 ++++++++----- test/ably/realtimeconnection_test.py | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7c26bd22..74181ccc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -33,10 +33,10 @@ optional = false python-versions = ">=3.5" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] -docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "cloudpickle"] +dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] +docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] +tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] +tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] [[package]] name = "certifi" @@ -525,7 +525,7 @@ oldcrypto = ["pycrypto"] [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "e8dcc51a079609cb656121cc7cb0134c432190bd3f879748a04c62f55c1c67f4" +content-hash = "2ed8bc1953862545c5c388fe654b9841f99045749193bd2f8ea3cff38001ef74" [metadata.files] anyio = [ @@ -743,6 +743,7 @@ pycryptodome = [ {file = "pycryptodome-3.15.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:7c9ed8aa31c146bef65d89a1b655f5f4eab5e1120f55fc297713c89c9e56ff0b"}, {file = "pycryptodome-3.15.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:5099c9ca345b2f252f0c28e96904643153bae9258647585e5e6f649bb7a1844a"}, {file = "pycryptodome-3.15.0-cp27-cp27m-manylinux2014_aarch64.whl", hash = "sha256:2ec709b0a58b539a4f9d33fb8508264c3678d7edb33a68b8906ba914f71e8c13"}, + {file = "pycryptodome-3.15.0-cp27-cp27m-musllinux_1_1_aarch64.whl", hash = "sha256:2ae53125de5b0d2c95194d957db9bb2681da8c24d0fb0fe3b056de2bcaf5d837"}, {file = "pycryptodome-3.15.0-cp27-cp27m-win32.whl", hash = "sha256:fd2184aae6ee2a944aaa49113e6f5787cdc5e4db1eb8edb1aea914bd75f33a0c"}, {file = "pycryptodome-3.15.0-cp27-cp27m-win_amd64.whl", hash = "sha256:7e3a8f6ee405b3bd1c4da371b93c31f7027944b2bcce0697022801db93120d83"}, {file = "pycryptodome-3.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:b9c5b1a1977491533dfd31e01550ee36ae0249d78aae7f632590db833a5012b8"}, @@ -750,12 +751,14 @@ pycryptodome = [ {file = "pycryptodome-3.15.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:2aa55aae81f935a08d5a3c2042eb81741a43e044bd8a81ea7239448ad751f763"}, {file = "pycryptodome-3.15.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:c3640deff4197fa064295aaac10ab49a0d55ef3d6a54ae1499c40d646655c89f"}, {file = "pycryptodome-3.15.0-cp27-cp27mu-manylinux2014_aarch64.whl", hash = "sha256:045d75527241d17e6ef13636d845a12e54660aa82e823b3b3341bcf5af03fa79"}, + {file = "pycryptodome-3.15.0-cp27-cp27mu-musllinux_1_1_aarch64.whl", hash = "sha256:eb6fce570869e70cc8ebe68eaa1c26bed56d40ad0f93431ee61d400525433c54"}, {file = "pycryptodome-3.15.0-cp35-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9ee40e2168f1348ae476676a2e938ca80a2f57b14a249d8fe0d3cdf803e5a676"}, {file = "pycryptodome-3.15.0-cp35-abi3-manylinux1_i686.whl", hash = "sha256:4c3ccad74eeb7b001f3538643c4225eac398c77d617ebb3e57571a897943c667"}, {file = "pycryptodome-3.15.0-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:1b22bcd9ec55e9c74927f6b1f69843cb256fb5a465088ce62837f793d9ffea88"}, {file = "pycryptodome-3.15.0-cp35-abi3-manylinux2010_i686.whl", hash = "sha256:57f565acd2f0cf6fb3e1ba553d0cb1f33405ec1f9c5ded9b9a0a5320f2c0bd3d"}, {file = "pycryptodome-3.15.0-cp35-abi3-manylinux2010_x86_64.whl", hash = "sha256:4b52cb18b0ad46087caeb37a15e08040f3b4c2d444d58371b6f5d786d95534c2"}, {file = "pycryptodome-3.15.0-cp35-abi3-manylinux2014_aarch64.whl", hash = "sha256:092a26e78b73f2530b8bd6b3898e7453ab2f36e42fd85097d705d6aba2ec3e5e"}, + {file = "pycryptodome-3.15.0-cp35-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:50ca7e587b8e541eb6c192acf92449d95377d1f88908c0a32ac5ac2703ebe28b"}, {file = "pycryptodome-3.15.0-cp35-abi3-win32.whl", hash = "sha256:e244ab85c422260de91cda6379e8e986405b4f13dc97d2876497178707f87fc1"}, {file = "pycryptodome-3.15.0-cp35-abi3-win_amd64.whl", hash = "sha256:c77126899c4b9c9827ddf50565e93955cb3996813c18900c16b2ea0474e130e9"}, {file = "pycryptodome-3.15.0-pp27-pypy_73-macosx_10_9_x86_64.whl", hash = "sha256:9eaadc058106344a566dc51d3d3a758ab07f8edde013712bc8d22032a86b264f"}, diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index c9e59360..01a34180 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -8,7 +8,7 @@ class TestRealtimeAuth(BaseAsyncTestCase): - async def setUp(self): + async def asyncSetUp(self): self.test_vars = await RestSetup.get_test_vars() self.valid_key_format = "api:key" From 00bff671d6efe529522e627fc21fec0face5a915 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Fri, 13 Jan 2023 11:02:33 +0000 Subject: [PATCH 275/481] update handle connected implementation --- ably/realtime/connection.py | 34 +++++++++++++++------------- test/ably/realtimeconnection_test.py | 20 +++++++++++++++- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 489682a3..8ca92985 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -85,6 +85,7 @@ def __init__(self, realtime): self.__state = ConnectionState.CONNECTING if realtime.options.auto_connect else ConnectionState.INITIALIZED self.__connection_manager = ConnectionManager(self.__realtime, self.state) self.__connection_manager.on('connectionstate', self._on_state_update) + self.__connection_manager.on('update', self._on_connection_update) super().__init__() async def connect(self): @@ -128,6 +129,9 @@ def _on_state_update(self, state_change): self.__error_reason = state_change.reason self.__realtime.options.loop.call_soon(functools.partial(self._emit, state_change.current, state_change)) + def _on_connection_update(self, state_change): + self.__realtime.options.loop.call_soon(functools.partial(self._emit, ConnectionEvent.UPDATE, state_change)) + @property def state(self): """The current connection state of the connection""" @@ -165,26 +169,24 @@ def __init__(self, realtime, initial_state): self.__retry_task = None self.__connection_details = None self.__fail_state = ConnectionState.DISCONNECTED - self.__fail_event = ConnectionEvent.DISCONNECTED super().__init__() - def enact_state_change(self, state, event, reason=None): + def enact_state_change(self, state, reason=None): current_state = self.__state self.__state = state if self.__state == ConnectionState.DISCONNECTED: if not self.__ttl_task or self.__ttl_task.done(): self.__ttl_task = asyncio.create_task(self.__start_suspended_timer()) - self._emit('connectionstate', ConnectionStateChange(current_state, state, event, reason)) + self._emit('connectionstate', ConnectionStateChange(current_state, state, state, reason)) async def __start_suspended_timer(self): if self.__connection_details: self.ably.options.connection_state_ttl = self.__connection_details.connectionStateTtl await asyncio.sleep(self.ably.options.connection_state_ttl / 1000) exception = AblyException("Exceeded connectionStateTtl while in DISCONNECTED state", 504, 50003) - self.enact_state_change(ConnectionState.SUSPENDED, ConnectionEvent.SUSPENDED, exception) + self.enact_state_change(ConnectionState.SUSPENDED, exception) self.__connection_details = None self.__fail_state = ConnectionState.SUSPENDED - self.__fail_event = ConnectionEvent.SUSPENDED if self.__retry_task: self.__retry_task.cancel() self.__retry_task = asyncio.create_task(self.retry_connection_attempt()) @@ -214,7 +216,7 @@ async def _connect(self): log.info('Connection cancelled due to request timeout. Attempting reconnection...') raise exception else: - self.enact_state_change(ConnectionState.CONNECTING, ConnectionEvent.CONNECTING) + self.enact_state_change(ConnectionState.CONNECTING) await self.connect_impl() def on_connection_attempt_done(self, task): @@ -231,7 +233,7 @@ def on_connection_attempt_done(self, task): if self.__connected_future: self.__connected_future.set_exception(exception) self.__connected_future = None - self.enact_state_change(self.__fail_state, self.__fail_event, exception) + self.enact_state_change(self.__fail_state, exception) self.__retry_task = asyncio.create_task(self.retry_connection_attempt()) async def retry_connection_attempt(self): @@ -244,19 +246,19 @@ async def retry_connection_attempt(self): async def close(self): if self.__state in (ConnectionState.CLOSED, ConnectionState.INITIALIZED, ConnectionState.FAILED): - self.enact_state_change(ConnectionState.CLOSED, ConnectionEvent.CLOSED) + self.enact_state_change(ConnectionState.CLOSED) return if self.__state is ConnectionState.DISCONNECTED: if self.transport: await self.transport.dispose() self.transport = None - self.enact_state_change(ConnectionState.CLOSED, ConnectionEvent.CLOSED) + self.enact_state_change(ConnectionState.CLOSED) return if self.__state != ConnectionState.CONNECTED: log.warning('Connection.closed called while connection state not connected') if self.__state == ConnectionState.CONNECTING: await self.__connected_future - self.enact_state_change(ConnectionState.CLOSING, ConnectionEvent.CLOSING) + self.enact_state_change(ConnectionState.CLOSING) self.__closed_future = asyncio.Future() if self.transport and self.transport.is_connected: await self.transport.close() @@ -266,7 +268,7 @@ async def close(self): raise AblyException("Timeout waiting for connection close response", 504, 50003) else: log.warning('ConnectionManager: called close with no connected transport') - self.enact_state_change(ConnectionState.CLOSED, ConnectionEvent.CLOSED) + self.enact_state_change(ConnectionState.CLOSED) if self.__ttl_task and not self.__ttl_task.done(): self.__ttl_task.cancel() if self.transport and self.transport.ws_connect_task is not None: @@ -324,7 +326,6 @@ async def ping(self): async def on_protocol_message(self, msg): action = msg['action'] if action == ProtocolMessageAction.CONNECTED: # CONNECTED - msg_error = msg.get("error") if self.transport: self.transport.is_connected = True if self.__connected_future: @@ -334,19 +335,20 @@ async def on_protocol_message(self, msg): else: log.warn('CONNECTED message received but connected_future not set') self.__fail_state == ConnectionState.DISCONNECTED - self.__fail_event == ConnectionEvent.DISCONNECTED if self.__ttl_task: self.__ttl_task.cancel() self.__connection_details = ConnectionDetails.from_dict(msg["connectionDetails"]) if self.__state == ConnectionState.CONNECTED: - self.enact_state_change(ConnectionState.CONNECTED, ConnectionEvent.UPDATE, msg_error) + state_change = ConnectionStateChange(ConnectionState.CONNECTED, ConnectionState.CONNECTED, + ConnectionEvent.UPDATE) + self._emit(ConnectionEvent.UPDATE, state_change) else: - self.enact_state_change(ConnectionState.CONNECTED, ConnectionEvent.CONNECTED) + self.enact_state_change(ConnectionState.CONNECTED) if action == ProtocolMessageAction.ERROR: # ERROR error = msg["error"] if error['nonfatal'] is False: exception = AblyAuthException(error["message"], error["statusCode"], error["code"]) - self.enact_state_change(ConnectionState.FAILED, ConnectionEvent.FAILED, exception) + self.enact_state_change(ConnectionState.FAILED, exception) if self.__connected_future: self.__connected_future.set_exception(exception) self.__connected_future = None diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index f2c785b3..f32bee2b 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -1,5 +1,5 @@ import asyncio -from ably.realtime.connection import ConnectionState, ProtocolMessageAction +from ably.realtime.connection import ConnectionEvent, ConnectionState, ProtocolMessageAction import pytest from ably.util.exceptions import AblyAuthException, AblyException from test.ably.restsetup import RestSetup @@ -242,3 +242,21 @@ def on_state_change(state_change): assert ably.connection.error_reason == changes[-1].reason await ably.close() Defaults.connection_state_ttl = 120000 + + async def test_handle_connected(self): + ably = await RestSetup.get_ably_realtime() + test_future = asyncio.Future() + + def on_update(connection_state): + if connection_state.event == ConnectionEvent.UPDATE: + test_future.set_result(connection_state) + + ably.connection.on(ConnectionEvent.UPDATE, on_update) + await ably.connection.connection_manager.on_protocol_message({'action': 4, "connectionDetails": + {"connectionStateTtl": 200}}) + state_change = await test_future + + assert state_change.previous == ConnectionState.CONNECTED + assert state_change.current == ConnectionState.CONNECTED + assert state_change.event == ConnectionEvent.UPDATE + await ably.close() From dd3e9fad57aaa503a3b36891e35e69bde7c75369 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 16 Jan 2023 11:33:49 +0000 Subject: [PATCH 276/481] test: fix connection test naming --- test/ably/realtimeconnection_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 01a34180..9996a030 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -7,12 +7,12 @@ from ably.transport.defaults import Defaults -class TestRealtimeAuth(BaseAsyncTestCase): +class TestRealtimeConnection(BaseAsyncTestCase): async def asyncSetUp(self): self.test_vars = await RestSetup.get_test_vars() self.valid_key_format = "api:key" - async def test_auth_connection(self): + async def test_connection_state(self): ably = await RestSetup.get_ably_realtime(auto_connect=False) assert ably.connection.state == ConnectionState.INITIALIZED await ably.connect() From 15ba46d30cc974ac2b53bc76fde3a85ecb5db835 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 16 Jan 2023 11:34:46 +0000 Subject: [PATCH 277/481] test: fix pyright naming mismatch --- test/ably/realtimeconnection_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 9996a030..05bfda29 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -142,10 +142,10 @@ async def test_realtime_request_timeout_ping(self): await ably.connect() original_send_protocol_message = ably.connection.connection_manager.send_protocol_message - async def new_send_protocol_message(msg): - if msg.get('action') == ProtocolMessageAction.HEARTBEAT: + async def new_send_protocol_message(protocol_message): + if protocol_message.get('action') == ProtocolMessageAction.HEARTBEAT: return - await original_send_protocol_message(msg) + await original_send_protocol_message(protocol_message) ably.connection.connection_manager.send_protocol_message = new_send_protocol_message with pytest.raises(AblyException) as exception: From 4bd8b5ce02cf3a3c9a498ada2daefd7608dc05eb Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 16 Jan 2023 11:35:44 +0000 Subject: [PATCH 278/481] test: fix realtimeinit setup fixture name --- test/ably/realtimeinit_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ably/realtimeinit_test.py b/test/ably/realtimeinit_test.py index fdb99a8e..c6cef00c 100644 --- a/test/ably/realtimeinit_test.py +++ b/test/ably/realtimeinit_test.py @@ -7,7 +7,7 @@ class TestRealtimeAuth(BaseAsyncTestCase): - async def setUp(self): + async def asyncSetUp(self): self.test_vars = await RestSetup.get_test_vars() self.valid_key_format = "api:key" From 41d12d7603eb0ab3c068afd81b0398a122f9d2dc Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 16 Jan 2023 11:37:23 +0000 Subject: [PATCH 279/481] test: fix realtimeinit test names --- test/ably/realtimeinit_test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/ably/realtimeinit_test.py b/test/ably/realtimeinit_test.py index c6cef00c..e97069a0 100644 --- a/test/ably/realtimeinit_test.py +++ b/test/ably/realtimeinit_test.py @@ -6,29 +6,29 @@ from test.ably.utils import BaseAsyncTestCase -class TestRealtimeAuth(BaseAsyncTestCase): +class TestRealtimeInit(BaseAsyncTestCase): async def asyncSetUp(self): self.test_vars = await RestSetup.get_test_vars() self.valid_key_format = "api:key" - async def test_auth_with_valid_key(self): + async def test_init_with_valid_key(self): ably = await RestSetup.get_ably_realtime(key=self.test_vars["keys"][0]["key_str"], auto_connect=False) assert Auth.Method.BASIC == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" assert ably.auth.auth_options.key_name == self.test_vars["keys"][0]['key_name'] assert ably.auth.auth_options.key_secret == self.test_vars["keys"][0]['key_secret'] - async def test_auth_incorrect_key(self): + async def test_init_with_incorrect_key(self): with pytest.raises(AblyAuthException): await RestSetup.get_ably_realtime(key="some invalid key", auto_connect=False) - async def test_auth_with_valid_key_format(self): + async def test_init_with_valid_key_format(self): key = self.valid_key_format.split(":") ably = await RestSetup.get_ably_realtime(key=self.valid_key_format, auto_connect=False) assert Auth.Method.BASIC == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" assert ably.auth.auth_options.key_name == key[0] assert ably.auth.auth_options.key_secret == key[1] - async def test_auth_connection(self): + async def test_init_without_autoconnect(self): ably = await RestSetup.get_ably_realtime(auto_connect=False) assert ably.connection.state == ConnectionState.INITIALIZED await ably.connect() From e231af90dba0a55027c6d067265c5402f07bea9e Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 16 Jan 2023 11:37:54 +0000 Subject: [PATCH 280/481] test: remove duplicate test fixture --- test/ably/realtimeinit_test.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/ably/realtimeinit_test.py b/test/ably/realtimeinit_test.py index e97069a0..5521ae9a 100644 --- a/test/ably/realtimeinit_test.py +++ b/test/ably/realtimeinit_test.py @@ -35,9 +35,3 @@ async def test_init_without_autoconnect(self): assert ably.connection.state == ConnectionState.CONNECTED await ably.close() assert ably.connection.state == ConnectionState.CLOSED - - async def test_auth_invalid_key(self): - ably = await RestSetup.get_ably_realtime(key=self.valid_key_format, auto_connect=False) - with pytest.raises(AblyAuthException): - await ably.connect() - await ably.close() From 1b397997e18ff9e2aa45b8089e801412512eb934 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 9 Jan 2023 12:18:39 +0000 Subject: [PATCH 281/481] chore: add `Timer` utility class --- ably/util/helper.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ably/util/helper.py b/ably/util/helper.py index cead99d9..7cbcdc4c 100644 --- a/ably/util/helper.py +++ b/ably/util/helper.py @@ -2,6 +2,7 @@ import random import string import asyncio +from typing import Callable def get_random_id(): @@ -13,3 +14,17 @@ def get_random_id(): def is_callable_or_coroutine(value): return asyncio.iscoroutinefunction(value) or inspect.isfunction(value) or inspect.ismethod(value) + + +class Timer: + def __init__(self, timeout: float, callback: Callable): + self._timeout = timeout + self._callback = callback + self._task = asyncio.create_task(self._job()) + + async def _job(self): + await asyncio.sleep(self._timeout / 1000) + self._callback() + + def cancel(self): + self._task.cancel() From e31ac1a4fee2ea3fcd961e6148805924267de8e9 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 9 Jan 2023 12:19:45 +0000 Subject: [PATCH 282/481] chore: add `unix_time_ms` helper function --- ably/util/helper.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ably/util/helper.py b/ably/util/helper.py index 7cbcdc4c..25e29407 100644 --- a/ably/util/helper.py +++ b/ably/util/helper.py @@ -2,6 +2,7 @@ import random import string import asyncio +import time from typing import Callable @@ -16,6 +17,10 @@ def is_callable_or_coroutine(value): return asyncio.iscoroutinefunction(value) or inspect.isfunction(value) or inspect.ismethod(value) +def unix_time_ms(): + return round(time.time_ns() / 1_000_000) + + class Timer: def __init__(self, timeout: float, callback: Callable): self._timeout = timeout From 2ca706018798de9da34f00d6d57534ff27a94599 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 10 Jan 2023 12:31:22 +0000 Subject: [PATCH 283/481] fix: ensure no duplicate connection attempt tasks --- ably/realtime/connection.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index b26ab9b1..4bca55ec 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -166,9 +166,10 @@ def __init__(self, realtime, initial_state): self.__closed_future = None self.__ping_future = None self.__timeout_in_secs = self.options.realtime_request_timeout / 1000 + self.retry_connection_attempt_task = None + self.connection_attempt_task = None self.transport: WebSocketTransport | None = None self.__ttl_task = None - self.__retry_task = None self.__connection_details = None self.__fail_state = ConnectionState.DISCONNECTED super().__init__() @@ -189,9 +190,6 @@ async def __start_suspended_timer(self): self.enact_state_change(ConnectionState.SUSPENDED, exception) self.__connection_details = None self.__fail_state = ConnectionState.SUSPENDED - if self.__retry_task: - self.__retry_task.cancel() - self.__retry_task = asyncio.create_task(self.retry_connection_attempt()) async def connect(self): if not self.__connected_future: @@ -200,8 +198,8 @@ async def connect(self): await self.__connected_future def try_connect(self): - task = asyncio.create_task(self._connect()) - task.add_done_callback(self.on_connection_attempt_done) + self.connection_attempt_task = asyncio.create_task(self._connect()) + self.connection_attempt_task.add_done_callback(self.on_connection_attempt_done) async def _connect(self): if self.__state == ConnectionState.CONNECTED: @@ -230,6 +228,14 @@ def check_connection(self): return False def on_connection_attempt_done(self, task): + if self.connection_attempt_task: + if not self.connection_attempt_task.done(): + self.connection_attempt_task.cancel() + self.connection_attempt_task = None + if self.retry_connection_attempt_task: + if not self.retry_connection_attempt_task.done(): + self.retry_connection_attempt_task.cancel() + self.retry_connection_attempt_task = None try: exception = task.exception() except asyncio.CancelledError: @@ -243,8 +249,8 @@ def on_connection_attempt_done(self, task): if self.__connected_future: self.__connected_future.set_exception(exception) self.__connected_future = None - self.enact_state_change(self.__fail_state, exception) - self.__retry_task = asyncio.create_task(self.retry_connection_attempt()) + self.enact_state_change(ConnectionState.DISCONNECTED, exception) + self.retry_connection_attempt_task = asyncio.create_task(self.retry_connection_attempt()) async def retry_connection_attempt(self): if self.__fail_state == ConnectionState.SUSPENDED: From 0798d9afa6a85bec1193a35862855eae54371862 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 10 Jan 2023 12:31:38 +0000 Subject: [PATCH 284/481] refactor(ConnectionManager): emit 'transport.pending' event --- ably/realtime/connection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 4bca55ec..e2deea5c 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -299,6 +299,7 @@ async def close(self): async def connect_impl(self): self.transport = WebSocketTransport(self) + self._emit('transport.pending', self.transport) await self.transport.connect() try: await asyncio.wait_for(asyncio.shield(self.__connected_future), self.__timeout_in_secs) From 68efddfc2f046903591aa33a9c1822fd84ba0740 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 10 Jan 2023 12:34:03 +0000 Subject: [PATCH 285/481] refactor(Timer): allow coroutine Timer callback --- ably/util/helper.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ably/util/helper.py b/ably/util/helper.py index 25e29407..e221d1b8 100644 --- a/ably/util/helper.py +++ b/ably/util/helper.py @@ -29,7 +29,10 @@ def __init__(self, timeout: float, callback: Callable): async def _job(self): await asyncio.sleep(self._timeout / 1000) - self._callback() + if asyncio.iscoroutinefunction(self._callback): + await self._callback() + else: + self._callback() def cancel(self): self._task.cancel() From f703e34b2032cc01913f89272e711b836d80ef00 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 10 Jan 2023 12:41:20 +0000 Subject: [PATCH 286/481] refactor: add WebSocketTransport.on_protocol_message() --- ably/realtime/websockettransport.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index 3aeafccf..4a3c994b 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -72,6 +72,13 @@ async def ws_connect(self, ws_url, headers): except (WebSocketException, socket.gaierror) as e: raise AblyException(f'Error opening websocket connection: {e}', 400, 40000) + async def on_protocol_message(self, msg): + log.info(f'WebSocketTransport.on_protocol_message(): receieved protocol message: {msg}') + if msg['action'] == ProtocolMessageAction.CLOSED: + if self.ws_connect_task: + self.ws_connect_task.cancel() + await self.connection_manager.on_protocol_message(msg) + async def ws_read_loop(self): while True: if self.websocket is not None: @@ -80,11 +87,7 @@ async def ws_read_loop(self): except ConnectionClosedOK: break msg = json.loads(raw) - log.info(f'ws_read_loop(): receieved protocol message: {msg}') - if msg['action'] == ProtocolMessageAction.CLOSED: - if self.ws_connect_task: - self.ws_connect_task.cancel() - await self.connection_manager.on_protocol_message(msg) + await self.on_protocol_message(msg) else: raise Exception('ws_read_loop running with no websocket') From 0bd76ea125755b2a99783f6c354f11469c1538a3 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 10 Jan 2023 12:43:41 +0000 Subject: [PATCH 287/481] feat: implement max_idle_interval --- ably/realtime/connection.py | 4 +++ ably/realtime/websockettransport.py | 43 ++++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index e2deea5c..56370006 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -395,6 +395,10 @@ async def on_protocol_message(self, msg): ): self.__ably.channels._on_channel_message(msg) + def deactivate_transport(self, reason=None): + self.transport = None + self.enact_state_change(ConnectionState.DISCONNECTED, reason) + @property def ably(self): return self.__ably diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index 4a3c994b..553a4190 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -9,6 +9,7 @@ from ably.http.httputils import HttpUtils from ably.transport.defaults import Defaults from ably.util.exceptions import AblyException +from ably.util.helper import Timer, unix_time_ms from websockets.client import WebSocketClientProtocol, connect as ws_connect from websockets.exceptions import ConnectionClosedOK, WebSocketException @@ -38,7 +39,11 @@ def __init__(self, connection_manager: ConnectionManager): self.connect_task: asyncio.Task | None = None self.ws_connect_task: asyncio.Task | None = None self.connection_manager = connection_manager + self.options = self.connection_manager.options self.is_connected = False + self.idle_timer = None + self.last_activity = None + self.max_idle_interval = None async def connect(self): headers = HttpUtils.default_headers() @@ -73,8 +78,17 @@ async def ws_connect(self, ws_url, headers): raise AblyException(f'Error opening websocket connection: {e}', 400, 40000) async def on_protocol_message(self, msg): + self.on_activity() log.info(f'WebSocketTransport.on_protocol_message(): receieved protocol message: {msg}') - if msg['action'] == ProtocolMessageAction.CLOSED: + if msg['action'] == ProtocolMessageAction.CONNECTED: + connection_details = msg.get('connectionDetails') + if not connection_details: + raise NotImplementedError + max_idle_interval = connection_details.get('maxIdleInterval') + if max_idle_interval: + self.max_idle_interval = max_idle_interval + self.options.realtime_request_timeout + self.on_activity() + elif msg['action'] == ProtocolMessageAction.CLOSED: if self.ws_connect_task: self.ws_connect_task.cancel() await self.connection_manager.on_protocol_message(msg) @@ -104,6 +118,8 @@ async def dispose(self): self.read_loop.cancel() if self.ws_connect_task: self.ws_connect_task.cancel() + if self.idle_timer: + self.idle_timer.cancel() if self.websocket: try: await self.websocket.close() @@ -119,3 +135,28 @@ async def send(self, message: dict): raw_msg = json.dumps(message) log.info(f'WebSocketTransport.send(): sending {raw_msg}') await self.websocket.send(raw_msg) + + def set_idle_timer(self, timeout: float): + if not self.idle_timer: + self.idle_timer = Timer(timeout, self.on_idle_timer_expire) + + async def on_idle_timer_expire(self): + self.idle_timer = None + since_last = unix_time_ms() - self.last_activity + time_remaining = self.max_idle_interval - since_last + msg = f"No activity seen from realtime in {since_last} ms; assuming connection has dropped" + if time_remaining <= 0: + log.error(msg) + await self.disconnect(AblyException(msg, 408, 80003)) + else: + self.set_idle_timer(time_remaining + 100) + + def on_activity(self): + if not self.max_idle_interval: + return + self.last_activity = unix_time_ms() + self.set_idle_timer(self.max_idle_interval + 100) + + async def disconnect(self, reason=None): + await self.dispose() + self.connection_manager.deactivate_transport(reason) From 9389c647c2fb5a972dd3fdf68cade0927284d5a5 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 10 Jan 2023 12:43:54 +0000 Subject: [PATCH 288/481] test: add max_idle_interval test --- test/ably/realtimeconnection_test.py | 32 ++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index f666a632..c958c062 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -296,3 +296,35 @@ def on_update(connection_state): assert state_change.current == ConnectionState.CONNECTED assert state_change.event == ConnectionEvent.UPDATE await ably.close() + + async def test_max_idle_interval(self): + ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) + + test_future = asyncio.Future() + + def on_transport_pending(transport): + original_on_protocol_message = transport.on_protocol_message + + async def on_protocol_message(msg): + if msg["action"] == ProtocolMessageAction.CONNECTED: + msg["connectionDetails"]["maxIdleInterval"] = 100 + + await original_on_protocol_message(msg) + + transport.on_protocol_message = on_protocol_message + + ably.connection.connection_manager.on('transport.pending', on_transport_pending) + + def once_disconnected(state_change): + test_future.set_result(state_change) + + ably.connection.once(ConnectionState.DISCONNECTED, once_disconnected) + + state_change = await test_future + + assert state_change.previous == ConnectionState.CONNECTED + assert state_change.current == ConnectionState.DISCONNECTED + assert state_change.reason.code == 80003 + assert state_change.reason.status_code == 408 + + await ably.close() From f83af88a98360b14e136202a66a8564d9444df31 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 16 Jan 2023 22:36:05 +0000 Subject: [PATCH 289/481] fix: fail_state not set correctly on CONNECTED --- ably/realtime/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 56370006..c5be61ae 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -355,7 +355,7 @@ async def on_protocol_message(self, msg): self.__connected_future = None else: log.warn('CONNECTED message received but connected_future not set') - self.__fail_state == ConnectionState.DISCONNECTED + self.__fail_state = ConnectionState.DISCONNECTED if self.__ttl_task: self.__ttl_task.cancel() self.__connection_details = ConnectionDetails.from_dict(msg["connectionDetails"]) From 5ed817fb42683660d6b8030144597d1b24980586 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 16 Jan 2023 22:38:38 +0000 Subject: [PATCH 290/481] fix: ConnectionDetails.connection_state_ttl snake_casing --- ably/realtime/connection.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index c5be61ae..19f336ab 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -48,10 +48,10 @@ class ConnectionStateChange: @dataclass class ConnectionDetails: - connectionStateTtl: int + connection_state_ttl: int def __init__(self, connection_state_ttl: int): - self.connectionStateTtl = connection_state_ttl + self.connection_state_ttl = connection_state_ttl @staticmethod def from_dict(json_dict: dict): @@ -184,7 +184,7 @@ def enact_state_change(self, state, reason=None): async def __start_suspended_timer(self): if self.__connection_details: - self.ably.options.connection_state_ttl = self.__connection_details.connectionStateTtl + self.ably.options.connection_state_ttl = self.__connection_details.connection_state_ttl await asyncio.sleep(self.ably.options.connection_state_ttl / 1000) exception = AblyException("Exceeded connectionStateTtl while in DISCONNECTED state", 504, 50003) self.enact_state_change(ConnectionState.SUSPENDED, exception) From e25992bf48c47bcfe2e911708c6aef30435ce633 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 16 Jan 2023 22:40:01 +0000 Subject: [PATCH 291/481] refactor(ConnectionDetails): add max_idle_interval property --- ably/realtime/connection.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 19f336ab..ff7373ec 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -49,13 +49,15 @@ class ConnectionStateChange: @dataclass class ConnectionDetails: connection_state_ttl: int + max_idle_interval: int - def __init__(self, connection_state_ttl: int): + def __init__(self, connection_state_ttl: int, max_idle_interval: int): self.connection_state_ttl = connection_state_ttl + self.max_idle_interval = max_idle_interval @staticmethod def from_dict(json_dict: dict): - return ConnectionDetails(json_dict.get('connectionStateTtl')) + return ConnectionDetails(json_dict.get('connectionStateTtl'), json_dict.get('maxIdleInterval')) class Connection(EventEmitter): From ec40b1c4b8ad7c77c2f0ac653abcfc606ffbb90a Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 16 Jan 2023 22:57:25 +0000 Subject: [PATCH 292/481] refactor: move ConnectionDetails to types dir This is necessary to prevent a circular import when using the ConnectionDetails class from WebSocketTransport. --- ably/realtime/connection.py | 14 -------------- ably/types/connectiondetails.py | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 14 deletions(-) create mode 100644 ably/types/connectiondetails.py diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index ff7373ec..70ccd385 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -46,20 +46,6 @@ class ConnectionStateChange: reason: Optional[AblyException] = None -@dataclass -class ConnectionDetails: - connection_state_ttl: int - max_idle_interval: int - - def __init__(self, connection_state_ttl: int, max_idle_interval: int): - self.connection_state_ttl = connection_state_ttl - self.max_idle_interval = max_idle_interval - - @staticmethod - def from_dict(json_dict: dict): - return ConnectionDetails(json_dict.get('connectionStateTtl'), json_dict.get('maxIdleInterval')) - - class Connection(EventEmitter): """Ably Realtime Connection diff --git a/ably/types/connectiondetails.py b/ably/types/connectiondetails.py new file mode 100644 index 00000000..c338f6ea --- /dev/null +++ b/ably/types/connectiondetails.py @@ -0,0 +1,15 @@ +from dataclasses import dataclass + + +@dataclass() +class ConnectionDetails: + connection_state_ttl: int + max_idle_interval: int + + def __init__(self, connection_state_ttl: int, max_idle_interval: int): + self.connection_state_ttl = connection_state_ttl + self.max_idle_interval = max_idle_interval + + @staticmethod + def from_dict(json_dict: dict): + return ConnectionDetails(json_dict.get('connectionStateTtl'), json_dict.get('maxIdleInterval')) From 96ce5ab447ac150e5cbd6134277da1b8fb4bec98 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 16 Jan 2023 23:09:00 +0000 Subject: [PATCH 293/481] test: use `asyncSetup` in EventEmitter tests --- test/ably/eventemitter_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ably/eventemitter_test.py b/test/ably/eventemitter_test.py index deda7626..d981785e 100644 --- a/test/ably/eventemitter_test.py +++ b/test/ably/eventemitter_test.py @@ -5,7 +5,7 @@ class TestEventEmitter(BaseAsyncTestCase): - async def setUp(self): + async def asyncSetUp(self): self.test_vars = await RestSetup.get_test_vars() async def test_connection_events(self): From a22330d5fe9dae4a1e31a740f7f788f8945b8387 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 16 Jan 2023 23:29:55 +0000 Subject: [PATCH 294/481] refactor(ConnectionManager): move on_protocol_message into WebSocketTransport --- ably/realtime/connection.py | 89 ++++++++++++++-------------- ably/realtime/websockettransport.py | 28 ++++++--- test/ably/realtimeconnection_test.py | 16 +++-- 3 files changed, 74 insertions(+), 59 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 70ccd385..f2e7aef4 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -4,13 +4,14 @@ import httpx from ably.realtime.websockettransport import WebSocketTransport, ProtocolMessageAction from ably.transport.defaults import Defaults -from ably.util.exceptions import AblyAuthException, AblyException +from ably.util.exceptions import AblyException from ably.util.eventemitter import EventEmitter from enum import Enum from datetime import datetime from ably.util import helper from dataclasses import dataclass from typing import Optional +from ably.types.connectiondetails import ConnectionDetails log = logging.getLogger(__name__) @@ -332,56 +333,52 @@ async def ping(self): response_time_ms = (ping_end_time - ping_start_time) * 1000 return round(response_time_ms, 2) - async def on_protocol_message(self, msg): - action = msg['action'] - if action == ProtocolMessageAction.CONNECTED: # CONNECTED - if self.transport: - self.transport.is_connected = True + def on_connected(self, connection_details: ConnectionDetails): + if self.transport: + self.transport.is_connected = True + if self.__connected_future: + if not self.__connected_future.cancelled(): + self.__connected_future.set_result(None) + self.__connected_future = None + else: + log.warn('CONNECTED message received but connected_future not set') + self.__fail_state = ConnectionState.DISCONNECTED + if self.__ttl_task: + self.__ttl_task.cancel() + self.__connection_details = connection_details + if self.__state == ConnectionState.CONNECTED: + state_change = ConnectionStateChange(ConnectionState.CONNECTED, ConnectionState.CONNECTED, + ConnectionEvent.UPDATE) + self._emit(ConnectionEvent.UPDATE, state_change) + else: + self.enact_state_change(ConnectionState.CONNECTED) + + async def on_error(self, msg: dict, exception: AblyException): + if msg.get('channel') is None: + self.enact_state_change(ConnectionState.FAILED, exception) if self.__connected_future: - if not self.__connected_future.cancelled(): - self.__connected_future.set_result(None) + self.__connected_future.set_exception(exception) self.__connected_future = None - else: - log.warn('CONNECTED message received but connected_future not set') - self.__fail_state = ConnectionState.DISCONNECTED - if self.__ttl_task: - self.__ttl_task.cancel() - self.__connection_details = ConnectionDetails.from_dict(msg["connectionDetails"]) - if self.__state == ConnectionState.CONNECTED: - state_change = ConnectionStateChange(ConnectionState.CONNECTED, ConnectionState.CONNECTED, - ConnectionEvent.UPDATE) - self._emit(ConnectionEvent.UPDATE, state_change) - else: - self.enact_state_change(ConnectionState.CONNECTED) - if action == ProtocolMessageAction.ERROR: # ERROR - error = msg["error"] - if error['nonfatal'] is False: - exception = AblyAuthException(error["message"], error["statusCode"], error["code"]) - self.enact_state_change(ConnectionState.FAILED, exception) - if self.__connected_future: - self.__connected_future.set_exception(exception) - self.__connected_future = None - if self.transport: - await self.transport.dispose() - raise exception - if action == ProtocolMessageAction.CLOSED: if self.transport: await self.transport.dispose() + raise exception + + async def on_closed(self): + if self.transport: + await self.transport.dispose() + if self.__closed_future and not self.__closed_future.done(): self.__closed_future.set_result(None) - if action == ProtocolMessageAction.HEARTBEAT: - if self.__ping_future: - # Resolve on heartbeat from ping request. - # TODO: Handle Normal heartbeat if required - if self.__ping_id == msg.get("id"): - if not self.__ping_future.cancelled(): - self.__ping_future.set_result(None) - self.__ping_future = None - if action in ( - ProtocolMessageAction.ATTACHED, - ProtocolMessageAction.DETACHED, - ProtocolMessageAction.MESSAGE - ): - self.__ably.channels._on_channel_message(msg) + + def on_channel_message(self, msg: dict): + self.__ably.channels._on_channel_message(msg) + + def on_heartbeat(self, id: Optional[str]): + if self.__ping_future: + # Resolve on heartbeat from ping request. + if self.__ping_id == id: + if not self.__ping_future.cancelled(): + self.__ping_future.set_result(None) + self.__ping_future = None def deactivate_transport(self, reason=None): self.transport = None diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index 553a4190..513a2acb 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -8,6 +8,7 @@ import urllib.parse from ably.http.httputils import HttpUtils from ably.transport.defaults import Defaults +from ably.types.connectiondetails import ConnectionDetails from ably.util.exceptions import AblyException from ably.util.helper import Timer, unix_time_ms from websockets.client import WebSocketClientProtocol, connect as ws_connect @@ -80,18 +81,31 @@ async def ws_connect(self, ws_url, headers): async def on_protocol_message(self, msg): self.on_activity() log.info(f'WebSocketTransport.on_protocol_message(): receieved protocol message: {msg}') - if msg['action'] == ProtocolMessageAction.CONNECTED: - connection_details = msg.get('connectionDetails') - if not connection_details: - raise NotImplementedError - max_idle_interval = connection_details.get('maxIdleInterval') + action = msg.get('action') + if action == ProtocolMessageAction.CONNECTED: + connection_details = ConnectionDetails.from_dict(msg.get('connectionDetails')) + max_idle_interval = connection_details.max_idle_interval if max_idle_interval: self.max_idle_interval = max_idle_interval + self.options.realtime_request_timeout self.on_activity() - elif msg['action'] == ProtocolMessageAction.CLOSED: + self.connection_manager.on_connected(connection_details) + elif action == ProtocolMessageAction.CLOSED: if self.ws_connect_task: self.ws_connect_task.cancel() - await self.connection_manager.on_protocol_message(msg) + await self.connection_manager.on_closed() + elif action == ProtocolMessageAction.ERROR: + error = msg.get('error') + exception = AblyException(error.get('message'), error.get('statusCode'), error.get('code')) + await self.connection_manager.on_error(msg, exception) + elif action == ProtocolMessageAction.HEARTBEAT: + id = msg.get('id') + self.connection_manager.on_heartbeat(id) + elif action in ( + ProtocolMessageAction.ATTACHED, + ProtocolMessageAction.DETACHED, + ProtocolMessageAction.MESSAGE + ): + self.connection_manager.on_channel_message(msg) async def ws_read_loop(self): while True: diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index c958c062..af7f0f24 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -1,7 +1,7 @@ import asyncio from ably.realtime.connection import ConnectionEvent, ConnectionState, ProtocolMessageAction import pytest -from ably.util.exceptions import AblyAuthException, AblyException +from ably.util.exceptions import AblyException from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase from ably.transport.defaults import Defaults @@ -38,7 +38,7 @@ async def test_closing_state(self): async def test_auth_invalid_key(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) - with pytest.raises(AblyAuthException) as exception: + with pytest.raises(AblyException) as exception: await ably.connect() assert ably.connection.state == ConnectionState.FAILED assert ably.connection.error_reason == exception.value @@ -62,7 +62,7 @@ async def test_connection_ping_initialized(self): async def test_connection_ping_failed(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) - with pytest.raises(AblyAuthException) as exception: + with pytest.raises(AblyException) as exception: await ably.connect() assert ably.connection.state == ConnectionState.FAILED assert ably.connection.error_reason == exception.value @@ -115,7 +115,7 @@ def on_state_change(change): ably.connection.on(ConnectionState.FAILED, on_state_change) - with pytest.raises(AblyAuthException) as exception: + with pytest.raises(AblyException) as exception: await ably.connect() assert len(failed_changes) == 1 @@ -288,8 +288,12 @@ def on_update(connection_state): test_future.set_result(connection_state) ably.connection.on(ConnectionEvent.UPDATE, on_update) - await ably.connection.connection_manager.on_protocol_message({'action': 4, "connectionDetails": - {"connectionStateTtl": 200}}) + + async def on_transport_pending(transport): + await transport.on_protocol_message({'action': 4, "connectionDetails": {"connectionStateTtl": 200}}) + + ably.connection.connection_manager.on('transport.pending', on_transport_pending) + state_change = await test_future assert state_change.previous == ConnectionState.CONNECTED From cd9946cbcc9a32f1887aaeff51a064ec2acfa2ee Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 16 Jan 2023 23:30:57 +0000 Subject: [PATCH 295/481] fix: remove unneeded log.warn for CONNECTED while not connecting Recieving a CONNECT message outside of the client-initiated connect sequence is normal behaviour and doesn't need a warning message --- ably/realtime/connection.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index f2e7aef4..951eda0b 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -340,8 +340,6 @@ def on_connected(self, connection_details: ConnectionDetails): if not self.__connected_future.cancelled(): self.__connected_future.set_result(None) self.__connected_future = None - else: - log.warn('CONNECTED message received but connected_future not set') self.__fail_state = ConnectionState.DISCONNECTED if self.__ttl_task: self.__ttl_task.cancel() From f2cf13ff0db783a35c1d32f6261045ad4d55e22c Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 16 Jan 2023 23:32:18 +0000 Subject: [PATCH 296/481] test: use `asyncSetup` in RealtimeChannel tests --- test/ably/realtimechannel_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py index c95488cf..78810880 100644 --- a/test/ably/realtimechannel_test.py +++ b/test/ably/realtimechannel_test.py @@ -9,7 +9,7 @@ class TestRealtimeChannel(BaseAsyncTestCase): - async def setUp(self): + async def asyncSetUp(self): self.test_vars = await RestSetup.get_test_vars() self.valid_key_format = "api:key" From 4a04b6ab2871747bea858119f504041d032563f8 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Fri, 13 Jan 2023 16:06:43 +0000 Subject: [PATCH 297/481] doc: copy in feature manifest from ably/features --- .ably/capabilities.yaml | 76 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 .ably/capabilities.yaml diff --git a/.ably/capabilities.yaml b/.ably/capabilities.yaml new file mode 100644 index 00000000..f16884aa --- /dev/null +++ b/.ably/capabilities.yaml @@ -0,0 +1,76 @@ +%YAML 1.2 +--- +common-version: 1.2.0-alpha.1 +compliance: + Agent Identifier: + Agents: + Authentication: + API Key: + Token: + Callback: + Literal: + URL: + Query Time: + Debugging: + Error Information: + Logs: + Protocol: + JSON: + MessagePack: + REST: + Authentication: + Authorize: + Create Token Request: + Get Client Identifier: + Request Token: + Channel: + Encryption: + Existence Check: + Get: + History: + Iterate: + Name: + Presence: + History: + Member List: + Publish: + Idempotence: + Push Notifications: + List Subscriptions: + Subscribe: + Release: + Status: + Channel Details: # https://github.com/ably/ably-python/pull/276 + Opaque Request: + Push Notifications Administration: + Channel Subscription: + List: + List Channels: + Remove: + Save: + Device Registration: + Get: + List: + Remove: + Save: + Publish: + Request Timeout: + Service: + Get Time: + Statistics: + Query: + Service: + Environment: + Fallbacks: + Hosts: + Retry Count: + Retry Duration: + Retry Timeout: + Host: + Testing: + Disable TLS: + TCP Insecure Port: + TCP Secure Port: + Transport: + Connection Open Timeout: + HTTP/2: From 4c3b6b798118dee138db28c44e9c3642a804c5f7 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Fri, 13 Jan 2023 16:11:58 +0000 Subject: [PATCH 298/481] doc: update feature manifest with realtime client progress --- .ably/capabilities.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.ably/capabilities.yaml b/.ably/capabilities.yaml index f16884aa..965c2e74 100644 --- a/.ably/capabilities.yaml +++ b/.ably/capabilities.yaml @@ -17,6 +17,18 @@ compliance: Protocol: JSON: MessagePack: + .caveats: 'Not supported for realtime' + Realtime: + Channel: + Attach: + Subscribe: + State Events: + Connection: + Disconnected Retry Timeout: + Lifecycle control: + Ping: + State Events: + Suspended Retry Timeout: REST: Authentication: Authorize: @@ -40,7 +52,7 @@ compliance: Subscribe: Release: Status: - Channel Details: # https://github.com/ably/ably-python/pull/276 + Channel Details: Opaque Request: Push Notifications Administration: Channel Subscription: From 18cf05ee87e5a5ad7a4db5f04a994bae2a51d6c1 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 16 Jan 2023 12:52:31 +0000 Subject: [PATCH 299/481] doc: add RTC spec point comments --- ably/realtime/realtime.py | 8 ++++++++ ably/types/options.py | 2 ++ 2 files changed, 10 insertions(+) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index f3a6a71f..ba46450a 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -81,6 +81,7 @@ def __init__(self, key=None, loop=None, **kwargs): ValueError If no authentication key is not provided """ + # RTC1 super().__init__(key, **kwargs) if loop is None: @@ -104,6 +105,7 @@ def __init__(self, key=None, loop=None, **kwargs): if options.auto_connect: asyncio.ensure_future(self.connection.connection_manager.connect_impl()) + # RTC15 async def connect(self): """Establishes a realtime connection. @@ -112,17 +114,21 @@ async def connect(self): CONNECTING state. """ log.info('Realtime.connect() called') + # RTC15a await self.connection.connect() + # RTC16 async def close(self): """Causes the connection to close, entering the closing state. Once closed, the library will not attempt to re-establish the connection without an explicit call to connect() """ log.info('Realtime.close() called') + # RTC16a await self.connection.close() await super().close() + # RTC4 @property def auth(self): """Returns the auth object""" @@ -133,11 +139,13 @@ def options(self): """Returns the auth options object""" return self.__options + # RTC2 @property def connection(self): """Returns the realtime connection object""" return self.__connection + # RTC3 @property def channels(self): """Returns the realtime channel object""" diff --git a/ably/types/options.py b/ably/types/options.py index 4d7edfc4..90d112ce 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -108,6 +108,7 @@ def rest_host(self): def rest_host(self, value): self.__rest_host = value + # RTC1d @property def realtime_host(self): return self.__realtime_host @@ -220,6 +221,7 @@ def idempotent_rest_publishing(self): def loop(self): return self.__loop + # RTC1b @property def auto_connect(self): return self.__auto_connect From 97d6fd07a041b920e785d9680dc5b56bbd24b44e Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 16 Jan 2023 13:22:13 +0000 Subject: [PATCH 300/481] doc: add RTN spec point comments --- ably/realtime/connection.py | 26 +++++++++++++++----------- ably/realtime/realtime.py | 2 ++ ably/transport/defaults.py | 2 +- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 951eda0b..10c386a8 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -44,10 +44,10 @@ class ConnectionStateChange: previous: ConnectionState current: ConnectionState event: ConnectionEvent - reason: Optional[AblyException] = None + reason: Optional[AblyException] = None # RTN4f -class Connection(EventEmitter): +class Connection(EventEmitter): # RTN4 """Ably Realtime Connection Enables the management of a connection to Ably @@ -75,10 +75,11 @@ def __init__(self, realtime): self.__error_reason = None self.__state = ConnectionState.CONNECTING if realtime.options.auto_connect else ConnectionState.INITIALIZED self.__connection_manager = ConnectionManager(self.__realtime, self.state) - self.__connection_manager.on('connectionstate', self._on_state_update) - self.__connection_manager.on('update', self._on_connection_update) + self.__connection_manager.on('connectionstate', self._on_state_update) # RTN4a + self.__connection_manager.on('update', self._on_connection_update) # RTN4h super().__init__() + # RTN11 async def connect(self): """Establishes a realtime connection. @@ -95,6 +96,7 @@ async def close(self): """ await self.__connection_manager.close() + # RTN13 async def ping(self): """Send a ping to the realtime connection @@ -123,11 +125,13 @@ def _on_state_update(self, state_change): def _on_connection_update(self, state_change): self.__realtime.options.loop.call_soon(functools.partial(self._emit, ConnectionEvent.UPDATE, state_change)) + # RTN4d @property def state(self): """The current connection state of the connection""" return self.__state + # RTN25 @property def error_reason(self): """An object describing the last error which occurred on the channel, if any.""" @@ -175,7 +179,7 @@ async def __start_suspended_timer(self): if self.__connection_details: self.ably.options.connection_state_ttl = self.__connection_details.connection_state_ttl await asyncio.sleep(self.ably.options.connection_state_ttl / 1000) - exception = AblyException("Exceeded connectionStateTtl while in DISCONNECTED state", 504, 50003) + exception = AblyException("Exceeded connectionStateTtl while in DISCONNECTED state", 504, 50003) # RTN14e self.enact_state_change(ConnectionState.SUSPENDED, exception) self.__connection_details = None self.__fail_state = ConnectionState.SUSPENDED @@ -238,7 +242,7 @@ def on_connection_attempt_done(self, task): if self.__connected_future: self.__connected_future.set_exception(exception) self.__connected_future = None - self.enact_state_change(ConnectionState.DISCONNECTED, exception) + self.enact_state_change(ConnectionState.DISCONNECTED, exception) # RTN14d self.retry_connection_attempt_task = asyncio.create_task(self.retry_connection_attempt()) async def retry_connection_attempt(self): @@ -287,13 +291,13 @@ async def close(self): log.warning(f'Connection error encountered while closing: {e}') async def connect_impl(self): - self.transport = WebSocketTransport(self) + self.transport = WebSocketTransport(self) # RTN1 self._emit('transport.pending', self.transport) await self.transport.connect() try: await asyncio.wait_for(asyncio.shield(self.__connected_future), self.__timeout_in_secs) except asyncio.TimeoutError: - exception = AblyException("Timeout waiting for realtime connection", 504, 50003) + exception = AblyException("Timeout waiting for realtime connection", 504, 50003) # RTN14c if self.transport: await self.transport.dispose() self.tranpsort = None @@ -343,8 +347,8 @@ def on_connected(self, connection_details: ConnectionDetails): self.__fail_state = ConnectionState.DISCONNECTED if self.__ttl_task: self.__ttl_task.cancel() - self.__connection_details = connection_details - if self.__state == ConnectionState.CONNECTED: + self.__connection_details = connection_details # RTN21 + if self.__state == ConnectionState.CONNECTED: # RTN24 state_change = ConnectionStateChange(ConnectionState.CONNECTED, ConnectionState.CONNECTED, ConnectionEvent.UPDATE) self._emit(ConnectionEvent.UPDATE, state_change) @@ -352,7 +356,7 @@ def on_connected(self, connection_details: ConnectionDetails): self.enact_state_change(ConnectionState.CONNECTED) async def on_error(self, msg: dict, exception: AblyException): - if msg.get('channel') is None: + if msg.get('channel') is None: # RTN15i self.enact_state_change(ConnectionState.FAILED, exception) if self.__connected_future: self.__connected_future.set_exception(exception) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index ba46450a..b3fd802c 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -102,6 +102,8 @@ def __init__(self, key=None, loop=None, **kwargs): self.key = key self.__connection = Connection(self) self.__channels = Channels(self) + + # RTN3 if options.auto_connect: asyncio.ensure_future(self.connection.connection_manager.connect_impl()) diff --git a/ably/transport/defaults.py b/ably/transport/defaults.py index 04c57031..d4960f65 100644 --- a/ably/transport/defaults.py +++ b/ably/transport/defaults.py @@ -9,7 +9,7 @@ class Defaults: ] rest_host = "rest.ably.io" - realtime_host = "realtime.ably.io" + realtime_host = "realtime.ably.io" # RTN2 connectivity_check_url = "https://internet-up.ably-realtime.com/is-the-internet-up.txt" environment = 'production' From a27395c6d562c89be9479fe1ce11944c423468cd Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 16 Jan 2023 13:24:05 +0000 Subject: [PATCH 301/481] doc: add RTS spec point comments --- ably/realtime/realtime.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index b3fd802c..c194f9b6 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -147,7 +147,7 @@ def connection(self): """Returns the realtime connection object""" return self.__connection - # RTC3 + # RTC3, RTS1 @property def channels(self): """Returns the realtime channel object""" @@ -169,6 +169,7 @@ def __init__(self, realtime): self.all = {} self.__realtime = realtime + # RTS3 def get(self, name): """Creates a new RealtimeChannel object, or returns the existing channel object. @@ -182,6 +183,7 @@ def get(self, name): self.all[name] = RealtimeChannel(self.__realtime, name) return self.all[name] + # RTS4 def release(self, name): """Releases a RealtimeChannel object, deleting it, and enabling it to be garbage collected From 74d682458f575a34dfc04b3a74acdefd80a36a1a Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 16 Jan 2023 14:02:07 +0000 Subject: [PATCH 302/481] doc: add RTL spec point comments --- ably/realtime/realtime_channel.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 36cc6703..1538c11e 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -55,6 +55,7 @@ def __init__(self, realtime, name): self.__timeout_in_secs = self.__realtime.options.realtime_request_timeout / 1000 Channel.__init__(self, realtime, name, {}) + # RTL4 async def attach(self): """Attach to channel @@ -102,6 +103,7 @@ async def attach(self): await self.__realtime.connect() self.__attach_future = asyncio.Future() + # RTL4c await self.__realtime.connection.connection_manager.send_protocol_message( { "action": ProtocolMessageAction.ATTACH, @@ -109,11 +111,12 @@ async def attach(self): } ) try: - await asyncio.wait_for(self.__attach_future, self.__timeout_in_secs) + await asyncio.wait_for(self.__attach_future, self.__timeout_in_secs) # RTL4f except asyncio.TimeoutError: raise AblyException("Timeout waiting for channel attach", 504, 50003) self.set_state(ChannelState.ATTACHED) + # RTL5 async def detach(self): """Detach from channel @@ -129,7 +132,7 @@ async def detach(self): log.info(f'RealtimeChannel.detach() called, channel = {self.name}') - # RTL5g - raise exception if state invalid + # RTL5g, RTL5b - raise exception if state invalid if self.__realtime.connection.state in [ConnectionState.CLOSING, ConnectionState.FAILED]: raise AblyException( message=f"Unable to detach; channel state = {self.state}", @@ -161,6 +164,7 @@ async def detach(self): await self.__realtime.connect() self.__detach_future = asyncio.Future() + # RTL5d await self.__realtime.connection.connection_manager.send_protocol_message( { "action": ProtocolMessageAction.DETACH, @@ -168,11 +172,12 @@ async def detach(self): } ) try: - await asyncio.wait_for(self.__detach_future, self.__timeout_in_secs) + await asyncio.wait_for(self.__detach_future, self.__timeout_in_secs) # RTL5f except asyncio.TimeoutError: raise AblyException("Timeout waiting for channel detach", 504, 50003) self.set_state(ChannelState.DETACHED) + # RTL7 async def subscribe(self, *args): """Subscribe to a channel @@ -226,16 +231,20 @@ async def subscribe(self, *args): 40000 ) + # RTL7c if self.state in (ChannelState.INITIALIZED, ChannelState.ATTACHING, ChannelState.DETACHED): await self.attach() if event is not None: + # RTL7b self.__message_emitter.on(event, listener) else: + # RTL7a self.__message_emitter.on(listener) await self.attach() + # RTL8 def unsubscribe(self, *args): """Unsubscribe from a channel @@ -280,10 +289,13 @@ def unsubscribe(self, *args): log.info(f'RealtimeChannel.unsubscribe called, channel = {self.name}, event = {event}') if listener is None: + # RTL8c self.__message_emitter.off() elif event is not None: + # RTL8b self.__message_emitter.off(event, listener) else: + # RTL8a self.__message_emitter.off(listener) def _on_message(self, msg): @@ -303,13 +315,15 @@ def _on_message(self, msg): def set_state(self, state): self.__state = state - self._emit(state) + self._emit(state) # RTL2a + # RTL23 @property def name(self): """Returns channel name""" return self.__name + # RTL2b @property def state(self): """Returns channel state""" From 637c704d31f117994707fce9eca30b57e7665069 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 23 Jan 2023 12:54:44 +0000 Subject: [PATCH 303/481] refactor(ConnectionManager): add `request_state` method --- ably/realtime/connection.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 10c386a8..ea9eab9f 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -386,6 +386,20 @@ def deactivate_transport(self, reason=None): self.transport = None self.enact_state_change(ConnectionState.DISCONNECTED, reason) + def request_state(self, state: ConnectionState): + log.info(f'ConnectionManager.request_state(): state = {state}') + + if state == self.state: + return + + if state == ConnectionState.CONNECTING and self.__state == ConnectionState.CONNECTED: + return + + if state == ConnectionState.CLOSING and self.__state == ConnectionState.CLOSED: + return + + self.enact_state_change(state) + @property def ably(self): return self.__ably From e8537b67b107755fcd9d36cce869c23ea876e644 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 23 Jan 2023 12:57:03 +0000 Subject: [PATCH 304/481] refactor(ConnectionManager): add `notify_state` method --- ably/realtime/connection.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index ea9eab9f..134fadb0 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -400,6 +400,14 @@ def request_state(self, state: ConnectionState): self.enact_state_change(state) + def notify_state(self, state: ConnectionState, reason=None): + log.info(f'ConnectionManager.notify_state(): new state: {state}') + + if state == self.__state: + return + + self.enact_state_change(state, reason) + @property def ably(self): return self.__ably From c3329b0fc8d3bbfa4e7dcd26d88303bb374bc31e Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 23 Jan 2023 13:06:03 +0000 Subject: [PATCH 305/481] refactor: make WebSocketTransport.connect synchronous --- ably/realtime/connection.py | 2 +- ably/realtime/websockettransport.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 134fadb0..a54d3a77 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -293,7 +293,7 @@ async def close(self): async def connect_impl(self): self.transport = WebSocketTransport(self) # RTN1 self._emit('transport.pending', self.transport) - await self.transport.connect() + self.transport.connect() try: await asyncio.wait_for(asyncio.shield(self.__connected_future), self.__timeout_in_secs) except asyncio.TimeoutError: diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index 513a2acb..cb09e9d3 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -46,7 +46,7 @@ def __init__(self, connection_manager: ConnectionManager): self.last_activity = None self.max_idle_interval = None - async def connect(self): + def connect(self): headers = HttpUtils.default_headers() protocol_version = Defaults.protocol_version params = {"key": self.connection_manager.ably.key, "v": protocol_version} From 400486d8edddcec1b2ffa532554e549d7db19419 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 23 Jan 2023 13:08:42 +0000 Subject: [PATCH 306/481] refactor: emit 'connected' and 'failed' events from WebSocketTransport --- ably/realtime/websockettransport.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index cb09e9d3..222a9b0b 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -9,6 +9,7 @@ from ably.http.httputils import HttpUtils from ably.transport.defaults import Defaults from ably.types.connectiondetails import ConnectionDetails +from ably.util.eventemitter import EventEmitter from ably.util.exceptions import AblyException from ably.util.helper import Timer, unix_time_ms from websockets.client import WebSocketClientProtocol, connect as ws_connect @@ -33,7 +34,7 @@ class ProtocolMessageAction(IntEnum): MESSAGE = 15 -class WebSocketTransport: +class WebSocketTransport(EventEmitter): def __init__(self, connection_manager: ConnectionManager): self.websocket: WebSocketClientProtocol | None = None self.read_loop: asyncio.Task | None = None @@ -45,6 +46,7 @@ def __init__(self, connection_manager: ConnectionManager): self.idle_timer = None self.last_activity = None self.max_idle_interval = None + super().__init__() def connect(self): headers = HttpUtils.default_headers() @@ -71,12 +73,16 @@ async def ws_connect(self, ws_url, headers): try: async with ws_connect(ws_url, extra_headers=headers) as websocket: log.info(f'ws_connect(): connection established to {ws_url}') + self._emit('connected') self.websocket = websocket self.read_loop = self.connection_manager.options.loop.create_task(self.ws_read_loop()) self.read_loop.add_done_callback(self.on_read_loop_done) await self.read_loop except (WebSocketException, socket.gaierror) as e: - raise AblyException(f'Error opening websocket connection: {e}', 400, 40000) + exception = AblyException(f'Error opening websocket connection: {e}', 400, 40000) + log.exception(f'WebSocketTransport.ws_connect(): Error opening websocket connection: {exception}') + self._emit('failed', exception) + raise exception async def on_protocol_message(self, msg): self.on_activity() From e9804b38699377e113b0d55b0d71bd73a19e5f25 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 23 Jan 2023 13:10:15 +0000 Subject: [PATCH 307/481] refactor(ConnectionManager): add `start_connect` method --- ably/realtime/connection.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index a54d3a77..95dcd2d2 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -400,6 +400,37 @@ def request_state(self, state: ConnectionState): self.enact_state_change(state) + if state == ConnectionState.CONNECTING: + self.start_connect() + + def start_connect(self): + self.connect_base_task = asyncio.create_task(self.connect_base()) + + async def connect_base(self): + self.transport = WebSocketTransport(self) + self._emit('transport.pending', self.transport) + self.transport.connect() + + future = asyncio.Future() + + def on_transport_connected(): + log.info('ConnectionManager.try_a_host(): transport connected') + if self.transport: + self.transport.off('failed', on_transport_failed) + future.set_result(None) + + async def on_transport_failed(exception): + log.info('ConnectionManager.try_a_host(): transport failed') + if self.transport: + self.transport.off('connected', on_transport_connected) + await self.transport.dispose() + future.set_exception(exception) + + self.transport.once('connected', on_transport_connected) + self.transport.once('failed', on_transport_failed) + + await future + def notify_state(self, state: ConnectionState, reason=None): log.info(f'ConnectionManager.notify_state(): new state: {state}') From b9c3af3a59b883857f5053a0c43ca3b9413ae2fb Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 23 Jan 2023 13:21:45 +0000 Subject: [PATCH 308/481] refactor(ConnectionManager): add transition_timer methods --- ably/realtime/connection.py | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 95dcd2d2..30b6d66a 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -8,7 +8,7 @@ from ably.util.eventemitter import EventEmitter from enum import Enum from datetime import datetime -from ably.util import helper +from ably.util.helper import get_random_id, Timer from dataclasses import dataclass from typing import Optional from ably.types.connectiondetails import ConnectionDetails @@ -165,6 +165,7 @@ def __init__(self, realtime, initial_state): self.__ttl_task = None self.__connection_details = None self.__fail_state = ConnectionState.DISCONNECTED + self.transition_timer: Timer | None = None super().__init__() def enact_state_change(self, state, reason=None): @@ -322,7 +323,7 @@ async def ping(self): self.__ping_future = asyncio.Future() if self.__state in [ConnectionState.CONNECTED, ConnectionState.CONNECTING]: - self.__ping_id = helper.get_random_id() + self.__ping_id = get_random_id() ping_start_time = datetime.now().timestamp() await self.send_protocol_message({"action": ProtocolMessageAction.HEARTBEAT, "id": self.__ping_id}) @@ -404,6 +405,7 @@ def request_state(self, state: ConnectionState): self.start_connect() def start_connect(self): + self.start_transition_timer(ConnectionState.CONNECTING) self.connect_base_task = asyncio.create_task(self.connect_base()) async def connect_base(self): @@ -437,8 +439,38 @@ def notify_state(self, state: ConnectionState, reason=None): if state == self.__state: return + self.cancel_transition_timer() + self.enact_state_change(state, reason) + def start_transition_timer(self, state: ConnectionState): + log.debug(f'ConnectionManager.start_transition_timer(): transition state = {state}') + + if self.transition_timer: + log.debug('ConnectionManager.start_transition_timer(): clearing already-running timer') + self.transition_timer.cancel() + + timeout = self.options.realtime_request_timeout + + def on_transition_timer_expire(): + if self.transition_timer: + self.transition_timer = None + log.info(f'ConnectionManager {state} timer expired, notifying new state: {self.__fail_state}') + self.notify_state( + self.__fail_state, + AblyException("Connection cancelled due to request timeout", 504, 50003) + ) + + log.debug(f'ConnectionManager.start_transition_timer(): setting timer for {timeout}ms') + + self.transition_timer = Timer(timeout, on_transition_timer_expire) + + def cancel_transition_timer(self): + log.debug('ConnectionManager.cancel_transition_timer()') + if self.transition_timer: + self.transition_timer.cancel() + self.transition_timer = None + @property def ably(self): return self.__ably From 74004f6020fc2087026337d106fa068009aa9817 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 23 Jan 2023 13:24:29 +0000 Subject: [PATCH 309/481] refactor(ConnectionManager): add suspend_timer methods --- ably/realtime/connection.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 30b6d66a..5198ec1b 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -166,6 +166,7 @@ def __init__(self, realtime, initial_state): self.__connection_details = None self.__fail_state = ConnectionState.DISCONNECTED self.transition_timer: Timer | None = None + self.suspend_timer: Timer | None = None super().__init__() def enact_state_change(self, state, reason=None): @@ -405,6 +406,7 @@ def request_state(self, state: ConnectionState): self.start_connect() def start_connect(self): + self.start_suspend_timer() self.start_transition_timer(ConnectionState.CONNECTING) self.connect_base_task = asyncio.create_task(self.connect_base()) @@ -440,6 +442,7 @@ def notify_state(self, state: ConnectionState, reason=None): return self.cancel_transition_timer() + self.check_suspend_timer(state) self.enact_state_change(state, reason) @@ -471,6 +474,39 @@ def cancel_transition_timer(self): self.transition_timer.cancel() self.transition_timer = None + def start_suspend_timer(self): + log.debug('ConnectionManager.start_suspend_timer()') + if self.suspend_timer: + return + + def on_suspend_timer_expire(): + if self.suspend_timer: + self.suspend_timer = None + log.info('ConnectionManager suspend timer expired, requesting new state: suspended') + self.notify_state( + ConnectionState.SUSPENDED, + AblyException("Connection to server unavailable", 400, 80002) + ) + self.__fail_state = ConnectionState.SUSPENDED + self.__connection_details = None + + self.suspend_timer = Timer(Defaults.connection_state_ttl, on_suspend_timer_expire) + + def check_suspend_timer(self, state: ConnectionState): + if state not in ( + ConnectionState.CONNECTING, + ConnectionState.DISCONNECTED, + ConnectionState.SUSPENDED, + ): + self.cancel_suspend_timer() + + def cancel_suspend_timer(self): + log.debug('ConnectionManager.cancel_suspend_timer()') + self.__fail_state = ConnectionState.DISCONNECTED + if self.suspend_timer: + self.suspend_timer.cancel() + self.suspend_timer = None + @property def ably(self): return self.__ably From cc2dbfafedeb4e0940c2e32f5a0f292584729335 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 23 Jan 2023 13:28:37 +0000 Subject: [PATCH 310/481] refactor(ConnectionManager): add retry_timer methods --- ably/realtime/connection.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 5198ec1b..a8afb8d4 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -444,6 +444,11 @@ def notify_state(self, state: ConnectionState, reason=None): self.cancel_transition_timer() self.check_suspend_timer(state) + if state == ConnectionState.DISCONNECTED: + self.start_retry_timer(self.options.disconnected_retry_timeout) + elif state == ConnectionState.SUSPENDED: + self.start_retry_timer(self.options.suspended_retry_timeout) + self.enact_state_change(state, reason) def start_transition_timer(self, state: ConnectionState): @@ -507,6 +512,19 @@ def cancel_suspend_timer(self): self.suspend_timer.cancel() self.suspend_timer = None + def start_retry_timer(self, interval: int): + def on_retry_timeout(): + log.info('ConnectionManager retry timer expired, retrying') + self.retry_timer = None + self.request_state(ConnectionState.CONNECTING) + + self.retry_timer = Timer(interval, on_retry_timeout) + + def cancel_retry_timer(self): + if self.retry_timer: + self.retry_timer.cancel() + self.retry_timer = None + @property def ably(self): return self.__ably From 752a95a04857ef95f46552af8573fdf259e205de Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 23 Jan 2023 13:45:47 +0000 Subject: [PATCH 311/481] refactor(EventEmitter): add `once_async` coroutine method --- ably/util/eventemitter.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ably/util/eventemitter.py b/ably/util/eventemitter.py index 6e737719..39d1713e 100644 --- a/ably/util/eventemitter.py +++ b/ably/util/eventemitter.py @@ -1,3 +1,4 @@ +import asyncio from pyee.asyncio import AsyncIOEventEmitter from ably.util.helper import is_callable_or_coroutine @@ -100,6 +101,21 @@ def off(self, *args): else: raise ValueError("EventEmitter.once(): invalid args") + async def once_async(self, state=None): + future = asyncio.Future() + + def on_state_change(*args): + future.set_result(*args) + + if state is not None: + self.once(state, on_state_change) + else: + self.once(on_state_change) + + state_change = await future + + return state_change + def _emit(self, *args): self.__named_event_emitter.emit(*args) self.__all_event_emitter.emit(_all_event, *args[1:]) From 30d82db3df77e3d7b8b66ed3f8f3c5093fc277a4 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 23 Jan 2023 15:40:51 +0000 Subject: [PATCH 312/481] refactor(EventEmitter): handle listener errors --- ably/util/eventemitter.py | 78 +++++++++++++++++++++++++++++++++++---- 1 file changed, 71 insertions(+), 7 deletions(-) diff --git a/ably/util/eventemitter.py b/ably/util/eventemitter.py index 39d1713e..4d2bfb41 100644 --- a/ably/util/eventemitter.py +++ b/ably/util/eventemitter.py @@ -1,4 +1,5 @@ import asyncio +import logging from pyee.asyncio import AsyncIOEventEmitter from ably.util.helper import is_callable_or_coroutine @@ -9,6 +10,8 @@ # used to emit all events on that listener _all_event = 'all' +log = logging.getLogger(__name__) + def _is_named_event_args(*args): return len(args) == 2 and is_callable_or_coroutine(args[1]) @@ -32,9 +35,11 @@ class EventEmitter: off() Subscribe to messages on a channel """ + def __init__(self): self.__named_event_emitter = AsyncIOEventEmitter() self.__all_event_emitter = AsyncIOEventEmitter() + self.__wrapped_listeners = {} def on(self, *args): """ @@ -51,12 +56,35 @@ def on(self, *args): The event listener. """ if _is_all_event_args(*args): - self.__all_event_emitter.add_listener(_all_event, args[0]) + event = _all_event + listener = args[0] + emitter = self.__all_event_emitter + # self.__all_event_emitter.add_listener(_all_event, args[0]) elif _is_named_event_args(*args): - self.__named_event_emitter.add_listener(args[0], args[1]) + event = args[0] + listener = args[1] + emitter = self.__named_event_emitter + # self.__named_event_emitter.add_listener(args[0], args[1]) else: raise ValueError("EventEmitter.on(): invalid args") + if asyncio.iscoroutinefunction(listener): + async def wrapped_listener(*args, **kwargs): + try: + await listener(*args, **kwargs) + except Exception as err: + log.exception(f'EventEmitter.emit(): uncaught listener exception: {err}') + else: + def wrapped_listener(*args, **kwargs): + try: + listener(*args, **kwargs) + except Exception as err: + log.exception(f'EventEmitter.emit(): uncaught listener exception: {err}') + + self.__wrapped_listeners[listener] = wrapped_listener + + emitter.add_listener(event, wrapped_listener) + def once(self, *args): """ Registers the provided listener for the first event that is emitted. If once() is called more than once @@ -73,11 +101,34 @@ def once(self, *args): The event listener. """ if _is_all_event_args(*args): - self.__all_event_emitter.once(_all_event, args[0]) + event = _all_event + listener = args[0] + emitter = self.__all_event_emitter + # self.__all_event_emitter.add_listener(_all_event, args[0]) elif _is_named_event_args(*args): - self.__named_event_emitter.once(args[0], args[1]) + event = args[0] + listener = args[1] + emitter = self.__named_event_emitter + # self.__named_event_emitter.add_listener(args[0], args[1]) else: - raise ValueError("EventEmitter.once(): invalid args") + raise ValueError("EventEmitter.on(): invalid args") + + if asyncio.iscoroutinefunction(listener): + async def wrapped_listener(*args, **kwargs): + try: + await listener(*args, **kwargs) + except Exception as err: + log.exception(f'EventEmitter.emit(): uncaught listener exception: {err}') + else: + def wrapped_listener(*args, **kwargs): + try: + listener(*args, **kwargs) + except Exception as err: + log.exception(f'EventEmitter.emit(): uncaught listener exception: {err}') + + self.__wrapped_listeners[listener] = wrapped_listener + + emitter.once(event, wrapped_listener) def off(self, *args): """ @@ -94,13 +145,26 @@ def off(self, *args): if len(args) == 0: self.__all_event_emitter.remove_all_listeners() self.__named_event_emitter.remove_all_listeners() + return elif _is_all_event_args(*args): - self.__all_event_emitter.remove_listener(_all_event, args[0]) + event = _all_event + listener = args[0] + emitter = self.__all_event_emitter elif _is_named_event_args(*args): - self.__named_event_emitter.remove_listener(args[0], args[1]) + event = args[0] + listener = args[1] + emitter = self.__named_event_emitter else: raise ValueError("EventEmitter.once(): invalid args") + wrapped_listener = self.__wrapped_listeners.get(listener) + + if wrapped_listener is None: + return + + emitter.remove_listener(event, wrapped_listener) + self.__wrapped_listeners[listener] = None + async def once_async(self, state=None): future = asyncio.Future() From 957e8f59911f5c798ec0bf1aad430005c13fa3b5 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 23 Jan 2023 15:58:04 +0000 Subject: [PATCH 313/481] refactor(Connnection): make connect method synchronous Sorry for the mega-commit, this commit rewrites a lot of the internal connection state logic to use `request_state` and `notify_state`. It also moves the retry/timeout behaviour to use the new `Timeout` class. --- ably/realtime/connection.py | 206 +++++++-------------------- ably/realtime/realtime.py | 8 +- ably/realtime/websockettransport.py | 16 ++- test/ably/eventemitter_test.py | 25 +--- test/ably/realtimechannel_test.py | 22 +-- test/ably/realtimeconnection_test.py | 188 ++++++++---------------- test/ably/realtimeinit_test.py | 3 +- 7 files changed, 144 insertions(+), 324 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index a8afb8d4..b9ffccc6 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -80,13 +80,13 @@ def __init__(self, realtime): super().__init__() # RTN11 - async def connect(self): + def connect(self): """Establishes a realtime connection. Causes the connection to open, entering the connecting state """ self.__error_reason = None - await self.__connection_manager.connect() + self.connection_manager.request_state(ConnectionState.CONNECTING) async def close(self): """Causes the connection to close, entering the closing state. @@ -94,7 +94,8 @@ async def close(self): Once closed, the library will not attempt to re-establish the connection without an explicit call to connect() """ - await self.__connection_manager.close() + self.connection_manager.request_state(ConnectionState.CLOSING) + await self.once_async(ConnectionState.CLOSED) # RTN13 async def ping(self): @@ -155,65 +156,24 @@ def __init__(self, realtime, initial_state): self.options = realtime.options self.__ably = realtime self.__state = initial_state - self.__connected_future = asyncio.Future() if initial_state == ConnectionState.CONNECTING else None - self.__closed_future = None self.__ping_future = None self.__timeout_in_secs = self.options.realtime_request_timeout / 1000 - self.retry_connection_attempt_task = None - self.connection_attempt_task = None self.transport: WebSocketTransport | None = None - self.__ttl_task = None self.__connection_details = None self.__fail_state = ConnectionState.DISCONNECTED self.transition_timer: Timer | None = None self.suspend_timer: Timer | None = None + self.retry_timer: Timer | None = None + self.connect_base_task: asyncio.Task | None = None + self.disconnect_transport_task: asyncio.Task | None = None super().__init__() def enact_state_change(self, state, reason=None): current_state = self.__state + log.info(f'ConnectionManager.enact_state_change(): {current_state} -> {state}') self.__state = state - if self.__state == ConnectionState.DISCONNECTED: - if not self.__ttl_task or self.__ttl_task.done(): - self.__ttl_task = asyncio.create_task(self.__start_suspended_timer()) self._emit('connectionstate', ConnectionStateChange(current_state, state, state, reason)) - async def __start_suspended_timer(self): - if self.__connection_details: - self.ably.options.connection_state_ttl = self.__connection_details.connection_state_ttl - await asyncio.sleep(self.ably.options.connection_state_ttl / 1000) - exception = AblyException("Exceeded connectionStateTtl while in DISCONNECTED state", 504, 50003) # RTN14e - self.enact_state_change(ConnectionState.SUSPENDED, exception) - self.__connection_details = None - self.__fail_state = ConnectionState.SUSPENDED - - async def connect(self): - if not self.__connected_future: - self.__connected_future = asyncio.Future() - self.try_connect() - await self.__connected_future - - def try_connect(self): - self.connection_attempt_task = asyncio.create_task(self._connect()) - self.connection_attempt_task.add_done_callback(self.on_connection_attempt_done) - - async def _connect(self): - if self.__state == ConnectionState.CONNECTED: - return - - if self.__state == ConnectionState.CONNECTING: - try: - if not self.__connected_future: - self.__connected_future = asyncio.Future() - await self.__connected_future - except asyncio.CancelledError: - exception = AblyException( - "Connection cancelled due to request timeout. Attempting reconnection...", 504, 50003) - log.info('Connection cancelled due to request timeout. Attempting reconnection...') - raise exception - else: - self.enact_state_change(ConnectionState.CONNECTING) - await self.connect_impl() - def check_connection(self): try: response = httpx.get(self.options.connectivity_check_url) @@ -222,91 +182,20 @@ def check_connection(self): except httpx.HTTPError: return False - def on_connection_attempt_done(self, task): - if self.connection_attempt_task: - if not self.connection_attempt_task.done(): - self.connection_attempt_task.cancel() - self.connection_attempt_task = None - if self.retry_connection_attempt_task: - if not self.retry_connection_attempt_task.done(): - self.retry_connection_attempt_task.cancel() - self.retry_connection_attempt_task = None - try: - exception = task.exception() - except asyncio.CancelledError: - exception = AblyException( - "Connection cancelled due to request timeout. Attempting reconnection...", 504, 50003) - if exception is None: - return - if self.__state in (ConnectionState.CLOSED, ConnectionState.FAILED): - return - if self.__state != ConnectionState.DISCONNECTED: - if self.__connected_future: - self.__connected_future.set_exception(exception) - self.__connected_future = None - self.enact_state_change(ConnectionState.DISCONNECTED, exception) # RTN14d - self.retry_connection_attempt_task = asyncio.create_task(self.retry_connection_attempt()) - - async def retry_connection_attempt(self): - if self.__fail_state == ConnectionState.SUSPENDED: - retry_timeout = self.ably.options.suspended_retry_timeout / 1000 - else: - retry_timeout = self.ably.options.disconnected_retry_timeout / 1000 - await asyncio.sleep(retry_timeout) - if self.check_connection(): - self.try_connect() - else: - exception = AblyException("Unable to connect (network unreachable)", 80003, 404) - self.enact_state_change(self.__fail_state, exception) + async def close_impl(self): + log.debug('ConnectionManager.close_impl()') - async def close(self): - if self.__state in (ConnectionState.CLOSED, ConnectionState.INITIALIZED, ConnectionState.FAILED): - self.enact_state_change(ConnectionState.CLOSED) - return - if self.__state is ConnectionState.DISCONNECTED: - if self.transport: - await self.transport.dispose() - self.transport = None - self.enact_state_change(ConnectionState.CLOSED) - return - if self.__state != ConnectionState.CONNECTED: - log.warning('Connection.closed called while connection state not connected') - if self.__state == ConnectionState.CONNECTING: - await self.__connected_future - self.enact_state_change(ConnectionState.CLOSING) - self.__closed_future = asyncio.Future() - if self.transport and self.transport.is_connected: - await self.transport.close() - try: - await asyncio.wait_for(self.__closed_future, self.__timeout_in_secs) - except asyncio.TimeoutError: - raise AblyException("Timeout waiting for connection close response", 504, 50003) - else: - log.warning('ConnectionManager: called close with no connected transport') - self.enact_state_change(ConnectionState.CLOSED) - if self.__ttl_task and not self.__ttl_task.done(): - self.__ttl_task.cancel() - if self.transport and self.transport.ws_connect_task is not None: - try: - await self.transport.ws_connect_task - except AblyException as e: - log.warning(f'Connection error encountered while closing: {e}') + self.cancel_suspend_timer() + self.start_transition_timer(ConnectionState.CLOSING, fail_state=ConnectionState.CLOSED) + if self.transport: + await self.transport.dispose() + if self.connect_base_task: + self.connect_base_task.cancel() + if self.disconnect_transport_task: + await self.disconnect_transport_task + self.cancel_retry_timer() - async def connect_impl(self): - self.transport = WebSocketTransport(self) # RTN1 - self._emit('transport.pending', self.transport) - self.transport.connect() - try: - await asyncio.wait_for(asyncio.shield(self.__connected_future), self.__timeout_in_secs) - except asyncio.TimeoutError: - exception = AblyException("Timeout waiting for realtime connection", 504, 50003) # RTN14c - if self.transport: - await self.transport.dispose() - self.tranpsort = None - self.__connected_future.set_exception(exception) - connected_future = self.__connected_future - self.__connected_future = None - self.on_connection_attempt_done(connected_future) + self.notify_state(ConnectionState.CLOSED) async def send_protocol_message(self, protocol_message): if self.transport is not None: @@ -340,29 +229,20 @@ async def ping(self): return round(response_time_ms, 2) def on_connected(self, connection_details: ConnectionDetails): - if self.transport: - self.transport.is_connected = True - if self.__connected_future: - if not self.__connected_future.cancelled(): - self.__connected_future.set_result(None) - self.__connected_future = None self.__fail_state = ConnectionState.DISCONNECTED - if self.__ttl_task: - self.__ttl_task.cancel() - self.__connection_details = connection_details # RTN21 - if self.__state == ConnectionState.CONNECTED: # RTN24 + + self.__connection_details = connection_details + + if self.__state == ConnectionState.CONNECTED: state_change = ConnectionStateChange(ConnectionState.CONNECTED, ConnectionState.CONNECTED, ConnectionEvent.UPDATE) self._emit(ConnectionEvent.UPDATE, state_change) else: - self.enact_state_change(ConnectionState.CONNECTED) + self.notify_state(ConnectionState.CONNECTED) async def on_error(self, msg: dict, exception: AblyException): if msg.get('channel') is None: # RTN15i self.enact_state_change(ConnectionState.FAILED, exception) - if self.__connected_future: - self.__connected_future.set_exception(exception) - self.__connected_future = None if self.transport: await self.transport.dispose() raise exception @@ -370,8 +250,8 @@ async def on_error(self, msg: dict, exception: AblyException): async def on_closed(self): if self.transport: await self.transport.dispose() - if self.__closed_future and not self.__closed_future.done(): - self.__closed_future.set_result(None) + if self.connect_base_task: + self.connect_base_task.cancel() def on_channel_message(self, msg: dict): self.__ably.channels._on_channel_message(msg) @@ -388,10 +268,10 @@ def deactivate_transport(self, reason=None): self.transport = None self.enact_state_change(ConnectionState.DISCONNECTED, reason) - def request_state(self, state: ConnectionState): + def request_state(self, state: ConnectionState, force=False): log.info(f'ConnectionManager.request_state(): state = {state}') - if state == self.state: + if not force and state == self.state: return if state == ConnectionState.CONNECTING and self.__state == ConnectionState.CONNECTED: @@ -400,11 +280,15 @@ def request_state(self, state: ConnectionState): if state == ConnectionState.CLOSING and self.__state == ConnectionState.CLOSED: return - self.enact_state_change(state) + if not force: + self.enact_state_change(state) if state == ConnectionState.CONNECTING: self.start_connect() + if state == ConnectionState.CLOSING: + asyncio.create_task(self.close_impl()) + def start_connect(self): self.start_suspend_timer() self.start_transition_timer(ConnectionState.CONNECTING) @@ -433,7 +317,10 @@ async def on_transport_failed(exception): self.transport.once('connected', on_transport_connected) self.transport.once('failed', on_transport_failed) - await future + try: + await future + except Exception as exception: + self.notify_state(self.__fail_state, reason=exception) def notify_state(self, state: ConnectionState, reason=None): log.info(f'ConnectionManager.notify_state(): new state: {state}') @@ -449,23 +336,29 @@ def notify_state(self, state: ConnectionState, reason=None): elif state == ConnectionState.SUSPENDED: self.start_retry_timer(self.options.suspended_retry_timeout) + if state == ConnectionState.DISCONNECTED or state == ConnectionState.SUSPENDED: + self.disconnect_transport() + self.enact_state_change(state, reason) - def start_transition_timer(self, state: ConnectionState): + def start_transition_timer(self, state: ConnectionState, fail_state=None): log.debug(f'ConnectionManager.start_transition_timer(): transition state = {state}') if self.transition_timer: log.debug('ConnectionManager.start_transition_timer(): clearing already-running timer') self.transition_timer.cancel() + if fail_state is None: + fail_state = self.__fail_state if state != ConnectionState.CLOSING else ConnectionState.CLOSED + timeout = self.options.realtime_request_timeout def on_transition_timer_expire(): if self.transition_timer: self.transition_timer = None - log.info(f'ConnectionManager {state} timer expired, notifying new state: {self.__fail_state}') + log.info(f'ConnectionManager {state} timer expired, notifying new state: {fail_state}') self.notify_state( - self.__fail_state, + fail_state, AblyException("Connection cancelled due to request timeout", 504, 50003) ) @@ -525,6 +418,11 @@ def cancel_retry_timer(self): self.retry_timer.cancel() self.retry_timer = None + def disconnect_transport(self): + log.info('ConnectionManager.disconnect_transport()') + if self.transport: + self.disconnect_transport_task = asyncio.create_task(self.transport.dispose()) + @property def ably(self): return self.__ably diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index c194f9b6..d1499b1f 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -1,6 +1,6 @@ import logging import asyncio -from ably.realtime.connection import Connection +from ably.realtime.connection import Connection, ConnectionState from ably.rest.auth import Auth from ably.rest.rest import AblyRest from ably.types.options import Options @@ -105,10 +105,10 @@ def __init__(self, key=None, loop=None, **kwargs): # RTN3 if options.auto_connect: - asyncio.ensure_future(self.connection.connection_manager.connect_impl()) + self.connection.connection_manager.request_state(ConnectionState.CONNECTING, force=True) # RTC15 - async def connect(self): + def connect(self): """Establishes a realtime connection. Explicitly calling connect() is unnecessary unless the autoConnect attribute of the ClientOptions object @@ -117,7 +117,7 @@ async def connect(self): """ log.info('Realtime.connect() called') # RTC15a - await self.connection.connect() + self.connection.connect() # RTC16 async def close(self): diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index 222a9b0b..e90a57d2 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -46,6 +46,7 @@ def __init__(self, connection_manager: ConnectionManager): self.idle_timer = None self.last_activity = None self.max_idle_interval = None + self.is_disposed = False super().__init__() def connect(self): @@ -65,9 +66,9 @@ def on_ws_connect_done(self, task: asyncio.Task): exception = e if exception is None or isinstance(exception, ConnectionClosedOK): return - connected_future = asyncio.Future() - connected_future.set_exception(exception) - self.connection_manager.on_connection_attempt_done(connected_future) + log.info( + f'WebSocketTransport.on_ws_connect_done(): exception = {exception}' + ) async def ws_connect(self, ws_url, headers): try: @@ -77,7 +78,12 @@ async def ws_connect(self, ws_url, headers): self.websocket = websocket self.read_loop = self.connection_manager.options.loop.create_task(self.ws_read_loop()) self.read_loop.add_done_callback(self.on_read_loop_done) - await self.read_loop + try: + await self.read_loop + except WebSocketException as err: + if not self.is_disposed: + await self.dispose() + self.connection_manager.deactivate_transport(err) except (WebSocketException, socket.gaierror) as e: exception = AblyException(f'Error opening websocket connection: {e}', 400, 40000) log.exception(f'WebSocketTransport.ws_connect(): Error opening websocket connection: {exception}') @@ -94,6 +100,7 @@ async def on_protocol_message(self, msg): if max_idle_interval: self.max_idle_interval = max_idle_interval + self.options.realtime_request_timeout self.on_activity() + self.is_connected = True self.connection_manager.on_connected(connection_details) elif action == ProtocolMessageAction.CLOSED: if self.ws_connect_task: @@ -134,6 +141,7 @@ def on_read_loop_done(self, task: asyncio.Task): return async def dispose(self): + self.is_disposed = True if self.read_loop: self.read_loop.cancel() if self.ws_connect_task: diff --git a/test/ably/eventemitter_test.py b/test/ably/eventemitter_test.py index d981785e..08a236fe 100644 --- a/test/ably/eventemitter_test.py +++ b/test/ably/eventemitter_test.py @@ -8,24 +8,6 @@ class TestEventEmitter(BaseAsyncTestCase): async def asyncSetUp(self): self.test_vars = await RestSetup.get_test_vars() - async def test_connection_events(self): - realtime = await RestSetup.get_ably_realtime() - call_count = 0 - - def listener(_): - nonlocal call_count - call_count += 1 - - realtime.connection.on(ConnectionState.CONNECTED, listener) - - await realtime.connect() - - # Listener is only called once event loop is free - assert call_count == 0 - await asyncio.sleep(0) - assert call_count == 1 - await realtime.close() - async def test_event_listener_error(self): realtime = await RestSetup.get_ably_realtime() call_count = 0 @@ -39,10 +21,9 @@ def listener(_): listener.side_effect = Exception() realtime.connection.on(ConnectionState.CONNECTED, listener) - await realtime.connect() + realtime.connect() + await realtime.connection.once_async(ConnectionState.CONNECTED) - assert call_count == 0 - await asyncio.sleep(0) assert call_count == 1 await realtime.close() @@ -57,7 +38,7 @@ def listener(_): realtime.connection.on(ConnectionState.CONNECTED, listener) realtime.connection.off(ConnectionState.CONNECTED, listener) - await realtime.connect() + await realtime.connection.once_async(ConnectionState.CONNECTED) assert call_count == 0 await asyncio.sleep(0) diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py index 78810880..b887ea38 100644 --- a/test/ably/realtimechannel_test.py +++ b/test/ably/realtimechannel_test.py @@ -4,7 +4,7 @@ from ably.types.message import Message from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase -from ably.realtime.connection import ProtocolMessageAction +from ably.realtime.connection import ConnectionState, ProtocolMessageAction from ably.util.exceptions import AblyException @@ -28,7 +28,7 @@ async def test_channels_release(self): async def test_channel_attach(self): ably = await RestSetup.get_ably_realtime() - await ably.connect() + await ably.connection.once_async(ConnectionState.CONNECTED) channel = ably.channels.get('my_channel') assert channel.state == ChannelState.INITIALIZED await channel.attach() @@ -37,7 +37,7 @@ async def test_channel_attach(self): async def test_channel_detach(self): ably = await RestSetup.get_ably_realtime() - await ably.connect() + await ably.connection.once_async(ConnectionState.CONNECTED) channel = ably.channels.get('my_channel') await channel.attach() await channel.detach() @@ -57,7 +57,7 @@ def listener(message): else: second_message_future.set_result(message) - await ably.connect() + await ably.connection.once_async(ConnectionState.CONNECTED) channel = ably.channels.get('my_channel') await channel.attach() await channel.subscribe('event', listener) @@ -78,7 +78,7 @@ def listener(message): async def test_subscribe_coroutine(self): ably = await RestSetup.get_ably_realtime() - await ably.connect() + await ably.connection.once_async(ConnectionState.CONNECTED) channel = ably.channels.get('my_channel') await channel.attach() @@ -106,7 +106,7 @@ async def listener(msg): # RTL7a async def test_subscribe_all_events(self): ably = await RestSetup.get_ably_realtime() - await ably.connect() + await ably.connection.once_async(ConnectionState.CONNECTED) channel = ably.channels.get('my_channel') await channel.attach() @@ -133,7 +133,7 @@ def listener(msg): # RTL7c async def test_subscribe_auto_attach(self): ably = await RestSetup.get_ably_realtime() - await ably.connect() + await ably.connection.once_async(ConnectionState.CONNECTED) channel = ably.channels.get('my_channel') assert channel.state == ChannelState.INITIALIZED @@ -149,7 +149,7 @@ def listener(_): # RTL8b async def test_unsubscribe(self): ably = await RestSetup.get_ably_realtime() - await ably.connect() + await ably.connection.once_async(ConnectionState.CONNECTED) channel = ably.channels.get('my_channel') await channel.attach() @@ -184,7 +184,7 @@ def listener(msg): # RTL8c async def test_unsubscribe_all(self): ably = await RestSetup.get_ably_realtime() - await ably.connect() + await ably.connection.once_async(ConnectionState.CONNECTED) channel = ably.channels.get('my_channel') await channel.attach() @@ -218,7 +218,7 @@ def listener(msg): async def test_realtime_request_timeout_attach(self): ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) - await ably.connect() + await ably.connection.once_async(ConnectionState.CONNECTED) original_send_protocol_message = ably.connection.connection_manager.send_protocol_message async def new_send_protocol_message(msg): @@ -236,7 +236,7 @@ async def new_send_protocol_message(msg): async def test_realtime_request_timeout_detach(self): ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) - await ably.connect() + await ably.connection.once_async(ConnectionState.CONNECTED) original_send_protocol_message = ably.connection.connection_manager.send_protocol_message async def new_send_protocol_message(msg): diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index af7f0f24..916c1190 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -15,38 +15,34 @@ async def asyncSetUp(self): async def test_connection_state(self): ably = await RestSetup.get_ably_realtime(auto_connect=False) assert ably.connection.state == ConnectionState.INITIALIZED - await ably.connect() + ably.connect() + await ably.connection.once_async() + assert ably.connection.state == ConnectionState.CONNECTING + await ably.connection.once_async() assert ably.connection.state == ConnectionState.CONNECTED await ably.close() assert ably.connection.state == ConnectionState.CLOSED - async def test_connecting_state(self): + async def test_connection_state_is_connecting_on_init(self): ably = await RestSetup.get_ably_realtime() - task = asyncio.create_task(ably.connect()) - await asyncio.sleep(0) assert ably.connection.state == ConnectionState.CONNECTING - await task await ably.close() - async def test_closing_state(self): - ably = await RestSetup.get_ably_realtime() - await ably.connect() - task = asyncio.create_task(ably.close()) - await asyncio.sleep(0) - assert ably.connection.state == ConnectionState.CLOSING - await task - async def test_auth_invalid_key(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) - with pytest.raises(AblyException) as exception: - await ably.connect() + state_change = await ably.connection.once_async() assert ably.connection.state == ConnectionState.FAILED - assert ably.connection.error_reason == exception.value + assert state_change.reason + assert state_change.reason.code == 40005 + assert state_change.reason.status_code == 400 + assert ably.connection.error_reason + assert ably.connection.error_reason.code == 40005 + assert ably.connection.error_reason.status_code == 400 await ably.close() async def test_connection_ping_connected(self): ably = await RestSetup.get_ably_realtime() - await ably.connect() + await ably.connection.once_async(ConnectionState.CONNECTED) response_time_ms = await ably.connection.ping() assert response_time_ms is not None assert type(response_time_ms) is float @@ -62,10 +58,8 @@ async def test_connection_ping_initialized(self): async def test_connection_ping_failed(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) - with pytest.raises(AblyException) as exception: - await ably.connect() + await ably.connection.once_async(ConnectionState.FAILED) assert ably.connection.state == ConnectionState.FAILED - assert ably.connection.error_reason == exception.value with pytest.raises(AblyException) as exception: await ably.connection.ping() assert exception.value.code == 400 @@ -74,8 +68,8 @@ async def test_connection_ping_failed(self): async def test_connection_ping_closed(self): ably = await RestSetup.get_ably_realtime() - await ably.connect() - assert ably.connection.state == ConnectionState.CONNECTED + ably.connect() + await ably.connection.once_async(ConnectionState.CONNECTED) await ably.close() with pytest.raises(AblyException) as exception: await ably.connection.ping() @@ -108,175 +102,120 @@ def on_state_change(change): async def test_connection_state_change_reason(self): ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) - failed_changes = [] - - def on_state_change(change): - failed_changes.append(change) + state_change = await ably.connection.once_async() - ably.connection.on(ConnectionState.FAILED, on_state_change) - - with pytest.raises(AblyException) as exception: - await ably.connect() - - assert len(failed_changes) == 1 - state_change = failed_changes[0] - assert state_change is not None assert state_change.previous == ConnectionState.CONNECTING assert state_change.current == ConnectionState.FAILED - assert state_change.reason == exception.value - assert ably.connection.error_reason == exception.value + assert ably.connection.error_reason is not None + assert ably.connection.error_reason is state_change.reason await ably.close() async def test_realtime_request_timeout_connect(self): ably = await RestSetup.get_ably_realtime(realtime_request_timeout=0.000001) - with pytest.raises(AblyException) as exception: - await ably.connect() - assert exception.value.code == 50003 - assert exception.value.status_code == 504 + state_change = await ably.connection.once_async() + assert state_change.reason is not None + assert state_change.reason.code == 50003 + assert state_change.reason.status_code == 504 assert ably.connection.state == ConnectionState.DISCONNECTED - assert ably.connection.error_reason == exception.value + assert ably.connection.error_reason == state_change.reason await ably.close() async def test_realtime_request_timeout_ping(self): ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) - await ably.connect() + await ably.connection.once_async(ConnectionState.CONNECTED) + original_send_protocol_message = ably.connection.connection_manager.send_protocol_message async def new_send_protocol_message(protocol_message): if protocol_message.get('action') == ProtocolMessageAction.HEARTBEAT: return await original_send_protocol_message(protocol_message) + ably.connection.connection_manager.send_protocol_message = new_send_protocol_message with pytest.raises(AblyException) as exception: await ably.connection.ping() - assert exception.value.code == 50003 - assert exception.value.status_code == 504 - await ably.close() - - async def test_realtime_request_timeout_close(self): - ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) - await ably.connect() - - async def new_close_transport(): - pass - ably.connection.connection_manager.transport.close = new_close_transport - - with pytest.raises(AblyException) as exception: - await ably.close() assert exception.value.code == 50003 assert exception.value.status_code == 504 + await ably.close() async def test_disconnected_retry_timeout(self): ably = await RestSetup.get_ably_realtime(disconnected_retry_timeout=2000, auto_connect=False) - original_connect = ably.connection.connection_manager._connect + original_connect = ably.connection.connection_manager.connect_base call_count = 0 - test_future = asyncio.Future() - test_exception = Exception() # intercept the library connection mechanism to fail the first two connection attempts async def new_connect(): nonlocal call_count if call_count < 2: + ably.connection.connection_manager.notify_state(ConnectionState.DISCONNECTED) call_count += 1 - raise test_exception else: await original_connect() - test_future.set_result(None) - ably.connection.connection_manager._connect = new_connect + ably.connection.connection_manager.connect_base = new_connect - with pytest.raises(Exception) as exception: - await ably.connect() + ably.connect() - assert ably.connection.state == ConnectionState.DISCONNECTED - assert exception.value == test_exception - - await test_future + await ably.connection.once_async(ConnectionState.DISCONNECTED) - assert ably.connection.state == ConnectionState.CONNECTED + # Test that the library eventually connects after two failed attempts + await ably.connection.once_async(ConnectionState.CONNECTED) await ably.close() async def test_connectivity_check_default(self): - ably = await RestSetup.get_ably_realtime() + ably = await RestSetup.get_ably_realtime(auto_connect=False) # The default connectivity check should return True assert ably.connection.connection_manager.check_connection() is True async def test_connectivity_check_non_default(self): ably = await RestSetup.get_ably_realtime( - connectivity_check_url="https://echo.ably.io/respondWith?status=200") + connectivity_check_url="https://echo.ably.io/respondWith?status=200", auto_connect=False) # A non-default URL should return True with a HTTP OK despite not returning "Yes" in the body assert ably.connection.connection_manager.check_connection() is True async def test_connectivity_check_bad_status(self): ably = await RestSetup.get_ably_realtime( - connectivity_check_url="https://echo.ably.io/respondWith?status=400") + connectivity_check_url="https://echo.ably.io/respondWith?status=400", auto_connect=False) # Should return False when the URL returns a non-2xx response code assert ably.connection.connection_manager.check_connection() is False - async def test_retry_connection_attempt(self): - ably = await RestSetup.get_ably_realtime( - connectivity_check_url="https://echo.ably.io/respondWith?status=400", disconnected_retry_timeout=1, - auto_connect=False) - test_future = asyncio.Future() - - def on_state_change(change): - if change.current == ConnectionState.DISCONNECTED: - test_future.set_result(change) - - ably.connection.connection_manager.on('connectionstate', on_state_change) - - asyncio.create_task(ably.connection.connection_manager.retry_connection_attempt()) - - state_change = await test_future - - assert state_change.reason.status_code == 80003 - assert state_change.reason.message == "Unable to connect (network unreachable)" - async def test_unroutable_host(self): - ably = await RestSetup.get_ably_realtime(realtime_host="10.255.255.1") - with pytest.raises(AblyException) as exception: - await ably.connect() - assert exception.value.code == 50003 - assert exception.value.status_code == 504 + ably = await RestSetup.get_ably_realtime(realtime_host="10.255.255.1", realtime_request_timeout=3000) + state_change = await ably.connection.once_async() + assert state_change.reason + assert state_change.reason.code == 50003 + assert state_change.reason.status_code == 504 assert ably.connection.state == ConnectionState.DISCONNECTED - assert ably.connection.error_reason == exception.value + assert ably.connection.error_reason == state_change.reason await ably.close() async def test_invalid_host(self): ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost") - with pytest.raises(AblyException) as exception: - await ably.connect() - assert exception.value.code == 40000 - assert exception.value.status_code == 400 + state_change = await ably.connection.once_async() + assert state_change.reason + assert state_change.reason.code == 40000 + assert state_change.reason.status_code == 400 assert ably.connection.state == ConnectionState.DISCONNECTED - assert ably.connection.error_reason == exception.value + assert ably.connection.error_reason == state_change.reason await ably.close() async def test_connection_state_ttl(self): - Defaults.connection_state_ttl = 100 - ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost") - changes = [] - suspended_future = asyncio.Future() + Defaults.connection_state_ttl = 10 + ably = await RestSetup.get_ably_realtime() - def on_state_change(state_change): - changes.append(state_change) - if state_change.current == ConnectionState.SUSPENDED: - suspended_future.set_result(None) - with pytest.raises(AblyException) as exception: - await ably.connect() - ably.connection.on(on_state_change) - assert exception.value.code == 40000 - assert exception.value.status_code == 400 - assert ably.connection.state == ConnectionState.DISCONNECTED - await suspended_future - assert ably.connection.state == changes[-1].current - assert ably.connection.state == ConnectionState.SUSPENDED + state_change = await ably.connection.once_async() + + assert state_change.previous == ConnectionState.CONNECTING + assert state_change.current == ConnectionState.SUSPENDED + assert state_change.reason + assert state_change.reason.code == 80002 + assert state_change.reason.status_code == 400 assert ably.connection.connection_details is None - assert ably.connection.error_reason == changes[-1].reason await ably.close() + Defaults.connection_state_ttl = 120000 async def test_handle_connected(self): @@ -304,8 +243,6 @@ async def on_transport_pending(transport): async def test_max_idle_interval(self): ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) - test_future = asyncio.Future() - def on_transport_pending(transport): original_on_protocol_message = transport.on_protocol_message @@ -319,12 +256,7 @@ async def on_protocol_message(msg): ably.connection.connection_manager.on('transport.pending', on_transport_pending) - def once_disconnected(state_change): - test_future.set_result(state_change) - - ably.connection.once(ConnectionState.DISCONNECTED, once_disconnected) - - state_change = await test_future + state_change = await ably.connection.once_async(ConnectionState.DISCONNECTED) assert state_change.previous == ConnectionState.CONNECTED assert state_change.current == ConnectionState.DISCONNECTED diff --git a/test/ably/realtimeinit_test.py b/test/ably/realtimeinit_test.py index 5521ae9a..a146ea25 100644 --- a/test/ably/realtimeinit_test.py +++ b/test/ably/realtimeinit_test.py @@ -31,7 +31,8 @@ async def test_init_with_valid_key_format(self): async def test_init_without_autoconnect(self): ably = await RestSetup.get_ably_realtime(auto_connect=False) assert ably.connection.state == ConnectionState.INITIALIZED - await ably.connect() + ably.connect() + await ably.connection.once_async(ConnectionState.CONNECTED) assert ably.connection.state == ConnectionState.CONNECTED await ably.close() assert ably.connection.state == ConnectionState.CLOSED From 022cb52e22392c46bcb7df6e1bb3207860a19c21 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 23 Jan 2023 16:22:06 +0000 Subject: [PATCH 314/481] feat: immediately reattempt connection if disconnected unexpectedly --- ably/realtime/connection.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index b9ffccc6..f00a518f 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -323,7 +323,13 @@ async def on_transport_failed(exception): self.notify_state(self.__fail_state, reason=exception) def notify_state(self, state: ConnectionState, reason=None): - log.info(f'ConnectionManager.notify_state(): new state: {state}') + # RTN15a + retry_immediately = state == ConnectionState.DISCONNECTED and self.__state == ConnectionState.CONNECTED + + log.info( + f'ConnectionManager.notify_state(): new state: {state}' + + ('; will retry immediately' if retry_immediately else '') + ) if state == self.__state: return @@ -331,12 +337,14 @@ def notify_state(self, state: ConnectionState, reason=None): self.cancel_transition_timer() self.check_suspend_timer(state) - if state == ConnectionState.DISCONNECTED: + if retry_immediately: + self.options.loop.call_soon(self.request_state, ConnectionState.CONNECTING) + elif state == ConnectionState.DISCONNECTED: self.start_retry_timer(self.options.disconnected_retry_timeout) elif state == ConnectionState.SUSPENDED: self.start_retry_timer(self.options.suspended_retry_timeout) - if state == ConnectionState.DISCONNECTED or state == ConnectionState.SUSPENDED: + if (state == ConnectionState.DISCONNECTED and not retry_immediately) or state == ConnectionState.SUSPENDED: self.disconnect_transport() self.enact_state_change(state, reason) From a2d182f2ae1d079a009b335e6ec24737d1f7898c Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 23 Jan 2023 16:22:48 +0000 Subject: [PATCH 315/481] test: add test for immediate connection retry --- test/ably/realtimeconnection_test.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 916c1190..daf28f49 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -264,3 +264,23 @@ async def on_protocol_message(msg): assert state_change.reason.status_code == 408 await ably.close() + + # RTN15a + async def test_retry_immediately_upon_unexpected_disconnection(self): + # Set timeouts to 500s so that if the client uses retry delay the test will fail with a timeout + ably = await RestSetup.get_ably_realtime( + disconnected_retry_timeout=500_000, + suspended_retry_timeout=500_000 + ) + + # Wait for the client to connect + await ably.connection.once_async(ConnectionState.CONNECTED) + + # Simulate random loss of connection + assert ably.connection.connection_manager.transport + await ably.connection.connection_manager.transport.dispose() + ably.connection.connection_manager.notify_state(ConnectionState.DISCONNECTED) + assert ably.connection.state == ConnectionState.DISCONNECTED + + # Wait for the client to connect again + await ably.connection.once_async(ConnectionState.CONNECTED) From 5cae4d69207192cc502228adabedf4def1f27cc6 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 25 Jan 2023 13:43:10 +0000 Subject: [PATCH 316/481] refactor: add realtime channel SUSPENDED and FAILED states --- ably/realtime/realtime_channel.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 1538c11e..563a3db0 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -19,6 +19,8 @@ class ChannelState(str, Enum): ATTACHED = 'attached' DETACHING = 'detaching' DETACHED = 'detached' + SUSPENDED = 'suspended' + FAILED = 'failed' class RealtimeChannel(EventEmitter, Channel): From f369b4e3977d3d09f22577324757686a4eae9ec9 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 25 Jan 2023 13:44:04 +0000 Subject: [PATCH 317/481] refactor: add `ChannelStateChange` class --- ably/realtime/realtime_channel.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 563a3db0..eddefe38 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -1,5 +1,7 @@ import asyncio +from dataclasses import dataclass import logging +from typing import Optional from ably.realtime.connection import ConnectionState, ProtocolMessageAction from ably.rest.channel import Channel @@ -23,6 +25,13 @@ class ChannelState(str, Enum): FAILED = 'failed' +@dataclass +class ChannelStateChange: + previous: ChannelState + current: ChannelState + reason: Optional[AblyException] = None + + class RealtimeChannel(EventEmitter, Channel): """ Ably Realtime Channel From 7a119a76ee73bfbde5dfb3175ea965ba4f477e23 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 25 Jan 2023 13:48:41 +0000 Subject: [PATCH 318/481] refactor(RealtimeChannel): add `_notify_state` method --- ably/realtime/realtime_channel.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index eddefe38..700d1045 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -107,7 +107,7 @@ async def attach(self): raise AblyException("Unable to detach channel due to request timeout", 504, 50003) return - self.set_state(ChannelState.ATTACHING) + self._notify_state(ChannelState.ATTACHING) # RTL4i - wait for pending connection if self.__realtime.connection.state == ConnectionState.CONNECTING: @@ -125,7 +125,7 @@ async def attach(self): await asyncio.wait_for(self.__attach_future, self.__timeout_in_secs) # RTL4f except asyncio.TimeoutError: raise AblyException("Timeout waiting for channel attach", 504, 50003) - self.set_state(ChannelState.ATTACHED) + self._notify_state(ChannelState.ATTACHED) # RTL5 async def detach(self): @@ -168,7 +168,7 @@ async def detach(self): except asyncio.CancelledError: raise AblyException("Unable to attach channel due to request timeout", 504, 50003) - self.set_state(ChannelState.DETACHING) + self._notify_state(ChannelState.DETACHING) # RTL5h - wait for pending connection if self.__realtime.connection.state == ConnectionState.CONNECTING: @@ -186,7 +186,7 @@ async def detach(self): await asyncio.wait_for(self.__detach_future, self.__timeout_in_secs) # RTL5f except asyncio.TimeoutError: raise AblyException("Timeout waiting for channel detach", 504, 50003) - self.set_state(ChannelState.DETACHED) + self._notify_state(ChannelState.DETACHED) # RTL7 async def subscribe(self, *args): @@ -324,9 +324,16 @@ def _on_message(self, msg): for message in messages: self.__message_emitter._emit(message.name, message) - def set_state(self, state): + def _notify_state(self, state: ChannelState, reason=None): + log.info(f'RealtimeChannel._notify_state(): state = {state}') + + if state == self.state: + return + + state_change = ChannelStateChange(self.__state, state, reason=reason) + self.__state = state - self._emit(state) # RTL2a + self._emit(state, state_change) # RTL23 @property From 2636b9b0139cc0f9d5d80067ac0d31f11fa7910c Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 25 Jan 2023 14:04:22 +0000 Subject: [PATCH 319/481] refactor(RealtimeChannel): add `_request_state` method --- ably/realtime/realtime_channel.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 700d1045..1de19580 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -107,7 +107,7 @@ async def attach(self): raise AblyException("Unable to detach channel due to request timeout", 504, 50003) return - self._notify_state(ChannelState.ATTACHING) + self._request_state(ChannelState.ATTACHING) # RTL4i - wait for pending connection if self.__realtime.connection.state == ConnectionState.CONNECTING: @@ -125,7 +125,7 @@ async def attach(self): await asyncio.wait_for(self.__attach_future, self.__timeout_in_secs) # RTL4f except asyncio.TimeoutError: raise AblyException("Timeout waiting for channel attach", 504, 50003) - self._notify_state(ChannelState.ATTACHED) + self._request_state(ChannelState.ATTACHED) # RTL5 async def detach(self): @@ -324,6 +324,10 @@ def _on_message(self, msg): for message in messages: self.__message_emitter._emit(message.name, message) + def _request_state(self, state: ChannelState): + log.info(f'RealtimeChannel._request_state(): state = {state}') + self._notify_state(state) + def _notify_state(self, state: ChannelState, reason=None): log.info(f'RealtimeChannel._notify_state(): state = {state}') From 4bf5ecd2de5985366c31892bf9430d9eecc97bd8 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 25 Jan 2023 14:10:30 +0000 Subject: [PATCH 320/481] refactor(RealtimeChannel): add `_send_message` method --- ably/realtime/realtime_channel.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 1de19580..ed8e94bd 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -114,13 +114,14 @@ async def attach(self): await self.__realtime.connect() self.__attach_future = asyncio.Future() + # RTL4c - await self.__realtime.connection.connection_manager.send_protocol_message( - { - "action": ProtocolMessageAction.ATTACH, - "channel": self.name, - } - ) + attach_msg = { + "action": ProtocolMessageAction.ATTACH, + "channel": self.name, + } + self._send_message(attach_msg) + try: await asyncio.wait_for(self.__attach_future, self.__timeout_in_secs) # RTL4f except asyncio.TimeoutError: @@ -175,13 +176,14 @@ async def detach(self): await self.__realtime.connect() self.__detach_future = asyncio.Future() + # RTL5d - await self.__realtime.connection.connection_manager.send_protocol_message( - { - "action": ProtocolMessageAction.DETACH, - "channel": self.name, - } - ) + detach_msg = { + "action": ProtocolMessageAction.DETACH, + "channel": self.name, + } + self._send_message(detach_msg) + try: await asyncio.wait_for(self.__detach_future, self.__timeout_in_secs) # RTL5f except asyncio.TimeoutError: @@ -339,6 +341,9 @@ def _notify_state(self, state: ChannelState, reason=None): self.__state = state self._emit(state, state_change) + def _send_message(self, msg): + asyncio.create_task(self.__realtime.connection.connection_manager.send_protocol_message(msg)) + # RTL23 @property def name(self): From 105bba34e35200d80462abb37b555e3ff4fe0a62 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 25 Jan 2023 16:22:53 +0000 Subject: [PATCH 321/481] refactor(RealtimeChannel): add `attach_impl` method --- ably/realtime/realtime_channel.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index ed8e94bd..d21459c1 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -115,12 +115,7 @@ async def attach(self): self.__attach_future = asyncio.Future() - # RTL4c - attach_msg = { - "action": ProtocolMessageAction.ATTACH, - "channel": self.name, - } - self._send_message(attach_msg) + self._attach_impl() try: await asyncio.wait_for(self.__attach_future, self.__timeout_in_secs) # RTL4f @@ -128,6 +123,16 @@ async def attach(self): raise AblyException("Timeout waiting for channel attach", 504, 50003) self._request_state(ChannelState.ATTACHED) + def _attach_impl(self): + log.info("RealtimeChannel.attach_impl(): sending ATTACH protocol message") + + # RTL4c + attach_msg = { + "action": ProtocolMessageAction.ATTACH, + "channel": self.name, + } + self._send_message(attach_msg) + # RTL5 async def detach(self): """Detach from channel From 3f0ab218afbecd82e17e656bc205c78ac20e1a6c Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 26 Jan 2023 11:22:56 +0000 Subject: [PATCH 322/481] refactor(RealtimeChannel) add `detach_impl` method --- ably/realtime/realtime_channel.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index d21459c1..31c28da5 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -182,12 +182,7 @@ async def detach(self): self.__detach_future = asyncio.Future() - # RTL5d - detach_msg = { - "action": ProtocolMessageAction.DETACH, - "channel": self.name, - } - self._send_message(detach_msg) + self._detach_impl() try: await asyncio.wait_for(self.__detach_future, self.__timeout_in_secs) # RTL5f @@ -195,6 +190,17 @@ async def detach(self): raise AblyException("Timeout waiting for channel detach", 504, 50003) self._notify_state(ChannelState.DETACHED) + def _detach_impl(self): + log.info("RealtimeChannel.detach_impl(): sending DETACH protocol message") + + # RTL5d + detach_msg = { + "action": ProtocolMessageAction.DETACH, + "channel": self.__name, + } + + self._send_message(detach_msg) + # RTL7 async def subscribe(self, *args): """Subscribe to a channel From 851c98c75970b6060206c542bf8ec9953649e578 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 24 Jan 2023 15:47:09 +0000 Subject: [PATCH 323/481] update options with fallback hosts --- ably/types/options.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/ably/types/options.py b/ably/types/options.py index 90d112ce..17beeeee 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -46,6 +46,9 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti from ably import api_version idempotent_rest_publishing = api_version >= '1.2' + if environment is None: + environment = Defaults.environment + self.__client_id = client_id self.__log_level = log_level self.__tls = tls @@ -254,8 +257,6 @@ def __get_rest_hosts(self): host = Defaults.rest_host environment = self.environment - if environment is None: - environment = Defaults.environment http_max_retry_count = self.http_max_retry_count if http_max_retry_count is None: @@ -292,6 +293,7 @@ def __get_rest_hosts(self): # Shuffle fallback_hosts = list(fallback_hosts) random.shuffle(fallback_hosts) + self.__fallback_hosts = fallback_hosts # First main host hosts = [host] + fallback_hosts @@ -300,11 +302,13 @@ def __get_rest_hosts(self): def __get_realtime_hosts(self): if self.realtime_host is not None: - return self.realtime_host - elif self.environment is not None: - return f'{self.environment}-{Defaults.realtime_host}' + host = self.realtime_host + elif self.environment != "production": + host = f'{self.environment}-{Defaults.realtime_host}' else: - return Defaults.realtime_host + host = Defaults.realtime_host + + return [host] + self.__fallback_hosts def get_rest_hosts(self): return self.__rest_hosts @@ -312,8 +316,14 @@ def get_rest_hosts(self): def get_rest_host(self): return self.__rest_hosts[0] - def get_realtime_host(self): + def get_realtime_hosts(self): return self.__realtime_hosts + def get_realtime_host(self): + return self.__realtime_hosts[0] + def get_fallback_rest_hosts(self): return self.__rest_hosts[1:] + + def get_fallback_realtime_hosts(self): + return self.__realtime_hosts[1:] \ No newline at end of file From b0c44069561613c30f33f0423fb1364083022b70 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 24 Jan 2023 15:51:47 +0000 Subject: [PATCH 324/481] refactor transport to take host --- ably/realtime/connection.py | 3 ++- ably/realtime/websockettransport.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index f00a518f..5be8856e 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -295,7 +295,8 @@ def start_connect(self): self.connect_base_task = asyncio.create_task(self.connect_base()) async def connect_base(self): - self.transport = WebSocketTransport(self) + host = self.options.get_realtime_host() + self.transport = WebSocketTransport(self, host) self._emit('transport.pending', self.transport) self.transport.connect() diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index e90a57d2..c7265691 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -35,7 +35,7 @@ class ProtocolMessageAction(IntEnum): class WebSocketTransport(EventEmitter): - def __init__(self, connection_manager: ConnectionManager): + def __init__(self, connection_manager: ConnectionManager, host: str): self.websocket: WebSocketClientProtocol | None = None self.read_loop: asyncio.Task | None = None self.connect_task: asyncio.Task | None = None @@ -47,6 +47,7 @@ def __init__(self, connection_manager: ConnectionManager): self.last_activity = None self.max_idle_interval = None self.is_disposed = False + self.host = host super().__init__() def connect(self): @@ -54,7 +55,7 @@ def connect(self): protocol_version = Defaults.protocol_version params = {"key": self.connection_manager.ably.key, "v": protocol_version} query_params = urllib.parse.urlencode(params) - ws_url = (f'wss://{self.connection_manager.options.get_realtime_host()}?{query_params}') + ws_url = (f'wss://{self.host}?{query_params}') log.info(f'connect(): attempting to connect to {ws_url}') self.ws_connect_task = asyncio.create_task(self.ws_connect(ws_url, headers)) self.ws_connect_task.add_done_callback(self.on_ws_connect_done) From 5eac8a583f500c97bf61c7daf23081d4c5dabd10 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 24 Jan 2023 15:57:29 +0000 Subject: [PATCH 325/481] implement try_host method --- ably/realtime/connection.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 5be8856e..e1b15792 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -296,6 +296,12 @@ def start_connect(self): async def connect_base(self): host = self.options.get_realtime_host() + try: + await self.try_host(host) + except Exception as exception: + self.notify_state(self.__fail_state, reason=exception) + + async def try_host(self, host): self.transport = WebSocketTransport(self, host) self._emit('transport.pending', self.transport) self.transport.connect() @@ -317,11 +323,7 @@ async def on_transport_failed(exception): self.transport.once('connected', on_transport_connected) self.transport.once('failed', on_transport_failed) - - try: - await future - except Exception as exception: - self.notify_state(self.__fail_state, reason=exception) + await future def notify_state(self, state: ConnectionState, reason=None): # RTN15a From 3a1f44a358aff3278a998f5f0cf09119e34e8e48 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 24 Jan 2023 16:12:26 +0000 Subject: [PATCH 326/481] implement use fallback host --- ably/realtime/connection.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index e1b15792..ffe9856b 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -295,11 +295,16 @@ def start_connect(self): self.connect_base_task = asyncio.create_task(self.connect_base()) async def connect_base(self): - host = self.options.get_realtime_host() - try: - await self.try_host(host) - except Exception as exception: - self.notify_state(self.__fail_state, reason=exception) + hosts = self.options.get_realtime_hosts() + for host in hosts: + try: + await self.try_host(host) + return + except Exception as exception: + log.exception(f'Connection to {host} failed, reason={exception}') + + log.exception("No more fallback hosts to try") + self.notify_state(self.__fail_state, reason=exception) async def try_host(self, host): self.transport = WebSocketTransport(self, host) From 022838ef51c5e6e2c7f9bdd894cacd9c3d0b17a7 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 24 Jan 2023 16:13:42 +0000 Subject: [PATCH 327/481] add test for fallback host --- test/ably/realtimeconnection_test.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index daf28f49..96afd546 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -284,3 +284,18 @@ async def test_retry_immediately_upon_unexpected_disconnection(self): # Wait for the client to connect again await ably.connection.once_async(ConnectionState.CONNECTED) + await ably.close() + + async def test_fallback_host(self): + fallback_host = 'sandbox-realtime.ably.io' + ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost", disconnected_retry_timeout=400000, + fallback_hosts=[fallback_host]) + connected_future = asyncio.Future() + + def on_change(connection_state): + if connection_state.current == ConnectionState.CONNECTED: + connected_future.set_result(connection_state) + + await ably.connection.once_async(ConnectionState.CONNECTED) + assert ably.connection.connection_manager.transport.host == fallback_host + await ably.close() From 9c48b4ea5b31717728e60d0c1532810a1185ec5a Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 25 Jan 2023 12:35:34 +0000 Subject: [PATCH 328/481] add connection check --- ably/realtime/connection.py | 25 +++++++++++++++++++------ test/ably/realtimeconnection_test.py | 20 +++++++++++++++----- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index ffe9856b..62b00248 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -296,12 +296,25 @@ def start_connect(self): async def connect_base(self): hosts = self.options.get_realtime_hosts() - for host in hosts: - try: - await self.try_host(host) - return - except Exception as exception: - log.exception(f'Connection to {host} failed, reason={exception}') + primary_host = hosts.pop(0) + try: + await self.try_host(primary_host) + return + except Exception as exception: + log.exception(f'Connection to {primary_host} failed, reason={exception}, attempting fallback hosts') + for host in hosts: + try: + if self.check_connection(): + await self.try_host(host) + return + else: + message = "Unable to connect, network unreachable" + log.exception(message) + exception = AblyException(message, status_code=404, code=80003) + self.notify_state(self.__fail_state, exception) + return + except Exception as exception: + log.exception(f'Connection to {host} failed, reason={exception}') log.exception("No more fallback hosts to try") self.notify_state(self.__fail_state, reason=exception) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 96afd546..4c786011 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -1,6 +1,7 @@ import asyncio from ably.realtime.connection import ConnectionEvent, ConnectionState, ProtocolMessageAction import pytest +import logging from ably.util.exceptions import AblyException from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase @@ -290,12 +291,21 @@ async def test_fallback_host(self): fallback_host = 'sandbox-realtime.ably.io' ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost", disconnected_retry_timeout=400000, fallback_hosts=[fallback_host]) - connected_future = asyncio.Future() - - def on_change(connection_state): - if connection_state.current == ConnectionState.CONNECTED: - connected_future.set_result(connection_state) await ably.connection.once_async(ConnectionState.CONNECTED) assert ably.connection.connection_manager.transport.host == fallback_host await ably.close() + + async def test_fallback_host_no_connectivity(self): + fallback_host = 'sandbox-realtime.ably.io' + ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost", disconnected_retry_timeout=400000, + fallback_hosts=[fallback_host]) + + def check_connection(): + return False + + ably.connection.connection_manager.check_connection = check_connection + + await ably.connection.once_async(ConnectionState.DISCONNECTED) + assert ably.connection.connection_manager.transport.host == "iamnotahost" + await ably.close() From 5dee0e463212e4b88f34e21bf15129c9fc8f7060 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 25 Jan 2023 13:42:52 +0000 Subject: [PATCH 329/481] set realtime fallback host fot http --- ably/http/http.py | 2 +- ably/realtime/connection.py | 39 ++++++++++++++-------------- ably/realtime/websockettransport.py | 2 ++ ably/types/options.py | 11 +++++++- test/ably/realtimeconnection_test.py | 9 +++---- 5 files changed, 37 insertions(+), 26 deletions(-) diff --git a/ably/http/http.py b/ably/http/http.py index e2607ca0..d53b540f 100644 --- a/ably/http/http.py +++ b/ably/http/http.py @@ -145,7 +145,7 @@ async def reauth(self): def get_rest_hosts(self): hosts = self.options.get_rest_hosts() - host = self.__host + host = self.__host or self.options.fallback_realtime_host if host is None: return hosts diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 62b00248..65b6a9fa 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -295,29 +295,30 @@ def start_connect(self): self.connect_base_task = asyncio.create_task(self.connect_base()) async def connect_base(self): - hosts = self.options.get_realtime_hosts() - primary_host = hosts.pop(0) + fallback_hosts = self.options.get_fallback_realtime_hosts() + primary_host = self.options.get_realtime_host() try: await self.try_host(primary_host) return except Exception as exception: - log.exception(f'Connection to {primary_host} failed, reason={exception}, attempting fallback hosts') - for host in hosts: - try: - if self.check_connection(): - await self.try_host(host) - return - else: - message = "Unable to connect, network unreachable" - log.exception(message) - exception = AblyException(message, status_code=404, code=80003) - self.notify_state(self.__fail_state, exception) - return - except Exception as exception: - log.exception(f'Connection to {host} failed, reason={exception}') - - log.exception("No more fallback hosts to try") - self.notify_state(self.__fail_state, reason=exception) + log.exception(f'Connection to {primary_host} failed, reason={exception}') + if len(fallback_hosts) > 0: + log.info("Attempting connection to fallback host(s)") + for host in fallback_hosts: + try: + if self.check_connection(): + await self.try_host(host) + return + else: + message = "Unable to connect, network unreachable" + log.exception(message) + exception = AblyException(message, status_code=404, code=80003) + self.notify_state(self.__fail_state, exception) + return + except Exception as exception: + log.exception(f'Connection to {host} failed, reason={exception}') + log.exception("No more fallback hosts to try") + self.notify_state(self.__fail_state, reason=exception) async def try_host(self, host): self.transport = WebSocketTransport(self, host) diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index c7265691..2e2a954d 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -102,6 +102,8 @@ async def on_protocol_message(self, msg): self.max_idle_interval = max_idle_interval + self.options.realtime_request_timeout self.on_activity() self.is_connected = True + if self.host != self.options.get_realtime_host(): # RTN17e + self.options.fallback_realtime_host = self.host self.connection_manager.on_connected(connection_details) elif action == ProtocolMessageAction.CLOSED: if self.ws_connect_task: diff --git a/ably/types/options.py b/ably/types/options.py index 17beeeee..750b91ac 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -75,6 +75,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti self.__connection_state_ttl = connection_state_ttl self.__suspended_retry_timeout = suspended_retry_timeout self.__connectivity_check_url = connectivity_check_url + self.__fallback_realtime_host = None self.__rest_hosts = self.__get_rest_hosts() self.__realtime_hosts = self.__get_realtime_hosts() @@ -245,6 +246,14 @@ def suspended_retry_timeout(self): def connectivity_check_url(self): return self.__connectivity_check_url + @property + def fallback_realtime_host(self): + return self.__fallback_realtime_host + + @fallback_realtime_host.setter + def fallback_realtime_host(self, value): + self.__fallback_realtime_host = value + def __get_rest_hosts(self): """ Return the list of hosts as they should be tried. First comes the main @@ -326,4 +335,4 @@ def get_fallback_rest_hosts(self): return self.__rest_hosts[1:] def get_fallback_realtime_hosts(self): - return self.__realtime_hosts[1:] \ No newline at end of file + return self.__realtime_hosts[1:] diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 4c786011..ad87bf3d 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -1,7 +1,6 @@ import asyncio from ably.realtime.connection import ConnectionEvent, ConnectionState, ProtocolMessageAction import pytest -import logging from ably.util.exceptions import AblyException from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase @@ -290,16 +289,16 @@ async def test_retry_immediately_upon_unexpected_disconnection(self): async def test_fallback_host(self): fallback_host = 'sandbox-realtime.ably.io' ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost", disconnected_retry_timeout=400000, - fallback_hosts=[fallback_host]) - + fallback_hosts=[fallback_host]) await ably.connection.once_async(ConnectionState.CONNECTED) assert ably.connection.connection_manager.transport.host == fallback_host + assert ably.options.fallback_realtime_host == fallback_host await ably.close() - async def test_fallback_host_no_connectivity(self): + async def test_no_connection_fallback_host(self): fallback_host = 'sandbox-realtime.ably.io' ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost", disconnected_retry_timeout=400000, - fallback_hosts=[fallback_host]) + fallback_hosts=[fallback_host]) def check_connection(): return False From 3a480b105da8f157e01673d09be206965971874a Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 25 Jan 2023 16:11:25 +0000 Subject: [PATCH 330/481] use fallback hosts on disconnected protocol message --- ably/realtime/connection.py | 54 +++++++++++++++++++++-------- ably/realtime/websockettransport.py | 3 ++ 2 files changed, 42 insertions(+), 15 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 65b6a9fa..9ba877f8 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -166,6 +166,7 @@ def __init__(self, realtime, initial_state): self.retry_timer: Timer | None = None self.connect_base_task: asyncio.Task | None = None self.disconnect_transport_task: asyncio.Task | None = None + self.__fallback_hosts = self.options.get_fallback_realtime_hosts() super().__init__() def enact_state_change(self, state, reason=None): @@ -240,6 +241,21 @@ def on_connected(self, connection_details: ConnectionDetails): else: self.notify_state(ConnectionState.CONNECTED) + def on_disconnected(self, msg:dict): + error = msg.get("error") + exception = AblyException(error.get('message'), error.get('statusCode'), error.get('code')) + self.notify_state(ConnectionState.DISCONNECTED, exception) + if error: + error_status_code = error.get("statusCode") + if error_status_code >= 500 or error_status_code <= 504: # RTN17f1 + if len(self.__fallback_hosts) > 0: + res = asyncio.create_task(self.connect_with_fallback_hosts(self.__fallback_hosts)) + if not res: + return + self.notify_state(self.__fail_state, reason=res) + else: + log.info("No fallback host to try for disconnected protocol message") + async def on_error(self, msg: dict, exception: AblyException): if msg.get('channel') is None: # RTN15i self.enact_state_change(ConnectionState.FAILED, exception) @@ -294,8 +310,26 @@ def start_connect(self): self.start_transition_timer(ConnectionState.CONNECTING) self.connect_base_task = asyncio.create_task(self.connect_base()) + async def connect_with_fallback_hosts(self, fallback_hosts): + for host in fallback_hosts: + try: + if self.check_connection(): + await self.try_host(host) + return + else: + message = "Unable to connect, network unreachable" + log.exception(message) + exception = AblyException(message, status_code=404, code=80003) + self.notify_state(self.__fail_state, exception) + return + except Exception as exc: + exception = exc + log.exception(f'Connection to {host}, reason={exception}') + log.exception("No more fallback hosts to try") + return exception + async def connect_base(self): - fallback_hosts = self.options.get_fallback_realtime_hosts() + fallback_hosts = self.__fallback_hosts primary_host = self.options.get_realtime_host() try: await self.try_host(primary_host) @@ -304,20 +338,10 @@ async def connect_base(self): log.exception(f'Connection to {primary_host} failed, reason={exception}') if len(fallback_hosts) > 0: log.info("Attempting connection to fallback host(s)") - for host in fallback_hosts: - try: - if self.check_connection(): - await self.try_host(host) - return - else: - message = "Unable to connect, network unreachable" - log.exception(message) - exception = AblyException(message, status_code=404, code=80003) - self.notify_state(self.__fail_state, exception) - return - except Exception as exception: - log.exception(f'Connection to {host} failed, reason={exception}') - log.exception("No more fallback hosts to try") + resp = await self.connect_with_fallback_hosts(fallback_hosts) + if not resp: + return + exception = resp self.notify_state(self.__fail_state, reason=exception) async def try_host(self, host): diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index 2e2a954d..fef9304d 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -24,6 +24,7 @@ class ProtocolMessageAction(IntEnum): HEARTBEAT = 0 CONNECTED = 4 + DISCONNECTED = 6 CLOSE = 7 CLOSED = 8 ERROR = 9 @@ -105,6 +106,8 @@ async def on_protocol_message(self, msg): if self.host != self.options.get_realtime_host(): # RTN17e self.options.fallback_realtime_host = self.host self.connection_manager.on_connected(connection_details) + elif action == ProtocolMessageAction.DISCONNECTED: + self.connection_manager.on_disconnected(msg) elif action == ProtocolMessageAction.CLOSED: if self.ws_connect_task: self.ws_connect_task.cancel() From 858332207bc4e72800bb5256d7394b17c6d6de33 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 25 Jan 2023 16:12:42 +0000 Subject: [PATCH 331/481] test fallback hosts on disconnected protocol msg --- test/ably/realtimeconnection_test.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index ad87bf3d..ccaa97c4 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -295,7 +295,7 @@ async def test_fallback_host(self): assert ably.options.fallback_realtime_host == fallback_host await ably.close() - async def test_no_connection_fallback_host(self): + async def test_fallback_host_no_connection(self): fallback_host = 'sandbox-realtime.ably.io' ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost", disconnected_retry_timeout=400000, fallback_hosts=[fallback_host]) @@ -308,3 +308,18 @@ def check_connection(): await ably.connection.once_async(ConnectionState.DISCONNECTED) assert ably.connection.connection_manager.transport.host == "iamnotahost" await ably.close() + + async def test_fallback_host_disconnected_protocol_msg(self): + fallback_host = 'sandbox-realtime.ably.io' + ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost", disconnected_retry_timeout=400000, + fallback_hosts=[fallback_host]) + + async def on_transport_pending(transport): + await transport.on_protocol_message({'action': 6, "error": {"statusCode": 500, "code": 500}}) + + ably.connection.connection_manager.on('transport.pending', on_transport_pending) + + + await ably.connection.once_async(ConnectionState.CONNECTED) + assert ably.connection.connection_manager.transport.host == fallback_host + await ably.close() From 0a223f6e12d0e2d1eb3e12e9423cc0f18a1f9d93 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 25 Jan 2023 16:30:25 +0000 Subject: [PATCH 332/481] fix linting and update type --- ably/realtime/connection.py | 4 ++-- test/ably/realtimeconnection_test.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 9ba877f8..811a1883 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -241,7 +241,7 @@ def on_connected(self, connection_details: ConnectionDetails): else: self.notify_state(ConnectionState.CONNECTED) - def on_disconnected(self, msg:dict): + def on_disconnected(self, msg: dict): error = msg.get("error") exception = AblyException(error.get('message'), error.get('statusCode'), error.get('code')) self.notify_state(ConnectionState.DISCONNECTED, exception) @@ -310,7 +310,7 @@ def start_connect(self): self.start_transition_timer(ConnectionState.CONNECTING) self.connect_base_task = asyncio.create_task(self.connect_base()) - async def connect_with_fallback_hosts(self, fallback_hosts): + async def connect_with_fallback_hosts(self, fallback_hosts: list): for host in fallback_hosts: try: if self.check_connection(): diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index ccaa97c4..43dd1d55 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -319,7 +319,6 @@ async def on_transport_pending(transport): ably.connection.connection_manager.on('transport.pending', on_transport_pending) - await ably.connection.once_async(ConnectionState.CONNECTED) assert ably.connection.connection_manager.transport.host == fallback_host await ably.close() From 52f6ddc9c35031dc32b189b94260fed52c02c1a7 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 26 Jan 2023 11:38:57 +0000 Subject: [PATCH 333/481] refactor(RealtimeChannel): use Timers and internal state emitter for channel attach --- ably/realtime/realtime_channel.py | 87 +++++++++++++++++++++---------- test/ably/realtimechannel_test.py | 4 +- 2 files changed, 62 insertions(+), 29 deletions(-) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 31c28da5..18b722c5 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -10,7 +10,7 @@ from ably.util.exceptions import AblyException from enum import Enum -from ably.util.helper import is_callable_or_coroutine +from ably.util.helper import Timer, is_callable_or_coroutine log = logging.getLogger(__name__) @@ -64,6 +64,12 @@ def __init__(self, realtime, name): self.__state = ChannelState.INITIALIZED self.__message_emitter = EventEmitter() self.__timeout_in_secs = self.__realtime.options.realtime_request_timeout / 1000 + self.__state_timer: Timer | None = None + + # Used to listen to state changes internally, if we use the public event emitter interface then internals + # will be disrupted if the user called .off() to remove all listeners + self.__internal_state_emitter = EventEmitter() + Channel.__init__(self, realtime, name, {}) # RTL4 @@ -93,35 +99,17 @@ async def attach(self): status_code=400 ) - # RTL4h - wait for pending attach/detach - if self.state == ChannelState.ATTACHING: - try: - await self.__attach_future - except asyncio.CancelledError: - raise AblyException("Unable to attach channel due to request timeout", 504, 50003) - return - elif self.state == ChannelState.DETACHING: - try: - await self.__detach_future - except asyncio.CancelledError: - raise AblyException("Unable to detach channel due to request timeout", 504, 50003) - return - - self._request_state(ChannelState.ATTACHING) + if self.state != ChannelState.ATTACHING: + self._request_state(ChannelState.ATTACHING) # RTL4i - wait for pending connection if self.__realtime.connection.state == ConnectionState.CONNECTING: await self.__realtime.connect() - self.__attach_future = asyncio.Future() - - self._attach_impl() + state_change = await self.__internal_state_emitter.once_async() - try: - await asyncio.wait_for(self.__attach_future, self.__timeout_in_secs) # RTL4f - except asyncio.TimeoutError: - raise AblyException("Timeout waiting for channel attach", 504, 50003) - self._request_state(ChannelState.ATTACHED) + if state_change.current in (ChannelState.SUSPENDED, ChannelState.FAILED): + raise state_change.reason def _attach_impl(self): log.info("RealtimeChannel.attach_impl(): sending ATTACH protocol message") @@ -325,9 +313,10 @@ def unsubscribe(self, *args): def _on_message(self, msg): action = msg.get('action') if action == ProtocolMessageAction.ATTACHED: - if self.__attach_future: - self.__attach_future.set_result(None) - self.__attach_future = None + if self.state == ChannelState.ATTACHING: + self._notify_state(ChannelState.ATTACHED) + else: + log.warn("RealtimeChannel._on_message(): ATTACHED received while not attaching") elif action == ProtocolMessageAction.DETACHED: if self.__detach_future: self.__detach_future.set_result(None) @@ -340,10 +329,13 @@ def _on_message(self, msg): def _request_state(self, state: ChannelState): log.info(f'RealtimeChannel._request_state(): state = {state}') self._notify_state(state) + self.__check_pending_state() def _notify_state(self, state: ChannelState, reason=None): log.info(f'RealtimeChannel._notify_state(): state = {state}') + self.__clear_state_timer() + if state == self.state: return @@ -351,10 +343,51 @@ def _notify_state(self, state: ChannelState, reason=None): self.__state = state self._emit(state, state_change) + self.__internal_state_emitter._emit(state, state_change) def _send_message(self, msg): asyncio.create_task(self.__realtime.connection.connection_manager.send_protocol_message(msg)) + def __check_pending_state(self): + connection_state = self.__realtime.connection.connection_manager.state + + if connection_state not in ( + ConnectionState.CONNECTING, + ConnectionState.CONNECTED, + ConnectionState.DISCONNECTED, + ): + return + + if self.state == ChannelState.ATTACHING: + self.__start_state_timer() + self._attach_impl() + elif self.state == ChannelState.DETACHING: + self.__start_state_timer() + self._detach_impl() + + def __start_state_timer(self): + if not self.__state_timer: + def on_timeout(): + log.info('RealtimeChannel.start_state_timer(): timer expired') + self.__state_timer = None + self.__timeout_pending_state() + + self.__state_timer = Timer(self.__realtime.options.realtime_request_timeout, on_timeout) + + def __clear_state_timer(self): + if self.__state_timer: + self.__state_timer.cancel() + self.__state_timer = None + + def __timeout_pending_state(self): + if self.state == ChannelState.ATTACHING: + self._notify_state( + ChannelState.SUSPENDED, reason=AblyException("Channel attach timed out", 408, 90007)) + elif self.state == ChannelState.DETACHING: + self._notify_state(ChannelState.ATTACHED, reason=AblyException("Channel detach timed out", 408, 90007)) + else: + self.__check_pending_state() + # RTL23 @property def name(self): diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py index b887ea38..30b38243 100644 --- a/test/ably/realtimechannel_test.py +++ b/test/ably/realtimechannel_test.py @@ -230,8 +230,8 @@ async def new_send_protocol_message(msg): channel = ably.channels.get('channel_name') with pytest.raises(AblyException) as exception: await channel.attach() - assert exception.value.code == 50003 - assert exception.value.status_code == 504 + assert exception.value.code == 90007 + assert exception.value.status_code == 408 await ably.close() async def test_realtime_request_timeout_detach(self): From acfef8a8dd9cb4bf823f21c7ecf946542dbdb398 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 26 Jan 2023 12:03:38 +0000 Subject: [PATCH 334/481] refactor(RealtimeChannel): use Timers and internal state emitter for channel detach --- ably/realtime/realtime_channel.py | 42 +++++++++++++------------------ test/ably/realtimechannel_test.py | 4 +-- 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 18b722c5..f832a80f 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -149,34 +149,27 @@ async def detach(self): if self.state in [ChannelState.INITIALIZED, ChannelState.DETACHED]: return - # RTL5i - wait for pending attach/detach - if self.state == ChannelState.DETACHING: - try: - await self.__detach_future - except asyncio.CancelledError: - raise AblyException("Unable to detach channel due to request timeout", 504, 50003) + if self.state == ChannelState.SUSPENDED: + self._notify_state(ChannelState.DETACHED) return - elif self.state == ChannelState.ATTACHING: - try: - await self.__attach_future - except asyncio.CancelledError: - raise AblyException("Unable to attach channel due to request timeout", 504, 50003) - - self._notify_state(ChannelState.DETACHING) + elif self.state == ChannelState.FAILED: + raise AblyException("Unable to detach; channel state = failed", 90001, 400) + else: + self._request_state(ChannelState.DETACHING) # RTL5h - wait for pending connection if self.__realtime.connection.state == ConnectionState.CONNECTING: await self.__realtime.connect() - self.__detach_future = asyncio.Future() - - self._detach_impl() + state_change = await self.__internal_state_emitter.once_async() + new_state = state_change.current - try: - await asyncio.wait_for(self.__detach_future, self.__timeout_in_secs) # RTL5f - except asyncio.TimeoutError: - raise AblyException("Timeout waiting for channel detach", 504, 50003) - self._notify_state(ChannelState.DETACHED) + if new_state == ChannelState.DETACHED: + return + elif new_state == ChannelState.ATTACHING: + raise AblyException("Detach request superseded by a subsequent attach request", 90000, 409) + else: + raise state_change.reason def _detach_impl(self): log.info("RealtimeChannel.detach_impl(): sending DETACH protocol message") @@ -318,9 +311,10 @@ def _on_message(self, msg): else: log.warn("RealtimeChannel._on_message(): ATTACHED received while not attaching") elif action == ProtocolMessageAction.DETACHED: - if self.__detach_future: - self.__detach_future.set_result(None) - self.__detach_future = None + if self.state == ChannelState.DETACHING: + self._notify_state(ChannelState.DETACHED) + else: + log.warn("RealtimeChannel._on_message(): DETACHED recieved while not detaching") elif action == ProtocolMessageAction.MESSAGE: messages = Message.from_encoded_array(msg.get('messages')) for message in messages: diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py index 30b38243..e171e873 100644 --- a/test/ably/realtimechannel_test.py +++ b/test/ably/realtimechannel_test.py @@ -249,6 +249,6 @@ async def new_send_protocol_message(msg): await channel.attach() with pytest.raises(AblyException) as exception: await channel.detach() - assert exception.value.code == 50003 - assert exception.value.status_code == 504 + assert exception.value.code == 90007 + assert exception.value.status_code == 408 await ably.close() From d5ebee0ef5b2eaa122b16ec87603d8c74b981e6b Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Thu, 26 Jan 2023 18:17:53 +0000 Subject: [PATCH 335/481] fix failing test in python 3.7 --- ably/realtime/connection.py | 8 ++++++-- test/ably/realtimeconnection_test.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 811a1883..db49f98c 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -324,7 +324,7 @@ async def connect_with_fallback_hosts(self, fallback_hosts: list): return except Exception as exc: exception = exc - log.exception(f'Connection to {host}, reason={exception}') + log.exception(f'Connection to {host} failed, reason={exception}') log.exception("No more fallback hosts to try") return exception @@ -366,7 +366,11 @@ async def on_transport_failed(exception): self.transport.once('connected', on_transport_connected) self.transport.once('failed', on_transport_failed) - await future + # Fix asyncio CancelledError in python 3.7 + try: + await future + except asyncio.CancelledError: + return def notify_state(self, state: ConnectionState, reason=None): # RTN15a diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 43dd1d55..8f13f319 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -317,7 +317,7 @@ async def test_fallback_host_disconnected_protocol_msg(self): async def on_transport_pending(transport): await transport.on_protocol_message({'action': 6, "error": {"statusCode": 500, "code": 500}}) - ably.connection.connection_manager.on('transport.pending', on_transport_pending) + ably.connection.connection_manager.once('transport.pending', on_transport_pending) await ably.connection.once_async(ConnectionState.CONNECTED) assert ably.connection.connection_manager.transport.host == fallback_host From e8fdfa1afb73a1e4b9b5fee95accba26271ccfef Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 26 Jan 2023 11:38:57 +0000 Subject: [PATCH 336/481] refactor(RealtimeChannel): use Timers and internal state emitter for channel attach --- ably/realtime/realtime_channel.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index f832a80f..10b88c55 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -58,12 +58,9 @@ class RealtimeChannel(EventEmitter, Channel): def __init__(self, realtime, name): EventEmitter.__init__(self) self.__name = name - self.__attach_future = None - self.__detach_future = None self.__realtime = realtime self.__state = ChannelState.INITIALIZED self.__message_emitter = EventEmitter() - self.__timeout_in_secs = self.__realtime.options.realtime_request_timeout / 1000 self.__state_timer: Timer | None = None # Used to listen to state changes internally, if we use the public event emitter interface then internals From 258db9fb9f11cf7e616254a6c61ec3c74e79e2f1 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 26 Jan 2023 12:14:20 +0000 Subject: [PATCH 337/481] feat: propagate connection interruption to RealtimeChannels --- ably/realtime/connection.py | 8 ++++++++ ably/realtime/realtime.py | 22 +++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index b9ffccc6..69a7aa58 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -341,6 +341,14 @@ def notify_state(self, state: ConnectionState, reason=None): self.enact_state_change(state, reason) + if state in ( + ConnectionState.CLOSING, + ConnectionState.CLOSED, + ConnectionState.SUSPENDED, + ConnectionState.FAILED, + ): + self.ably.channels._propagate_connection_interruption(state, reason) + def start_transition_timer(self, state: ConnectionState, fail_state=None): log.debug(f'ConnectionManager.start_transition_timer(): transition state = {state}') diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index d1499b1f..6fd0bc80 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -4,7 +4,7 @@ from ably.rest.auth import Auth from ably.rest.rest import AblyRest from ably.types.options import Options -from ably.realtime.realtime_channel import RealtimeChannel +from ably.realtime.realtime_channel import ChannelState, RealtimeChannel log = logging.getLogger(__name__) @@ -218,3 +218,23 @@ def _on_channel_message(self, msg): return channel._on_message(msg) + + def _propagate_connection_interruption(self, state: ConnectionState, reason): + from_channel_states = ( + ChannelState.ATTACHING, + ChannelState.ATTACHED, + ChannelState.DETACHING, + ChannelState.SUSPENDED, + ) + + connection_to_channel_state = { + ConnectionState.CLOSING: ChannelState.DETACHED, + ConnectionState.CLOSED: ChannelState.DETACHED, + ConnectionState.FAILED: ChannelState.FAILED, + ConnectionState.SUSPENDED: ChannelState.SUSPENDED, + } + + for name in self.all.keys(): + channel = self.all[name] + if channel.state in from_channel_states: + channel._notify_state(connection_to_channel_state[state], reason) From bd1abeb94129db1b528bbb94ef1d6b5fd4dc3b1f Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 26 Jan 2023 12:26:41 +0000 Subject: [PATCH 338/481] test: add test fixtures for channel state changes upon connection interruption --- test/ably/realtimechannel_test.py | 33 ++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py index e171e873..cbe16d40 100644 --- a/test/ably/realtimechannel_test.py +++ b/test/ably/realtimechannel_test.py @@ -3,7 +3,7 @@ from ably.realtime.realtime_channel import ChannelState from ably.types.message import Message from test.ably.restsetup import RestSetup -from test.ably.utils import BaseAsyncTestCase +from test.ably.utils import BaseAsyncTestCase, random_string from ably.realtime.connection import ConnectionState, ProtocolMessageAction from ably.util.exceptions import AblyException @@ -252,3 +252,34 @@ async def new_send_protocol_message(msg): assert exception.value.code == 90007 assert exception.value.status_code == 408 await ably.close() + + async def test_channel_detached_once_connection_closed(self): + ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) + await ably.connection.once_async(ConnectionState.CONNECTED) + channel = ably.channels.get(random_string(5)) + await channel.attach() + + await ably.close() + assert channel.state == ChannelState.DETACHED + + async def test_channel_failed_once_connection_failed(self): + ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) + await ably.connection.once_async(ConnectionState.CONNECTED) + channel = ably.channels.get(random_string(5)) + await channel.attach() + + ably.connection.connection_manager.notify_state(ConnectionState.SUSPENDED) + assert channel.state == ChannelState.SUSPENDED + + await ably.close() + + async def test_channel_suspended_once_connection_suspended(self): + ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) + await ably.connection.once_async(ConnectionState.CONNECTED) + channel = ably.channels.get(random_string(5)) + await channel.attach() + + ably.connection.connection_manager.notify_state(ConnectionState.FAILED) + assert channel.state == ChannelState.FAILED + + await ably.close() From f9039eb6270b5cf5300bdd44b005f94166d338ff Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 26 Jan 2023 13:06:13 +0000 Subject: [PATCH 339/481] feat: queue messages while CONNECTING or DISCONNECTED --- ably/realtime/connection.py | 42 ++++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 123506c7..1f18a4af 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -12,6 +12,7 @@ from dataclasses import dataclass from typing import Optional from ably.types.connectiondetails import ConnectionDetails +from queue import Queue log = logging.getLogger(__name__) @@ -167,6 +168,7 @@ def __init__(self, realtime, initial_state): self.connect_base_task: asyncio.Task | None = None self.disconnect_transport_task: asyncio.Task | None = None self.__fallback_hosts = self.options.get_fallback_realtime_hosts() + self.queued_messages = Queue() super().__init__() def enact_state_change(self, state, reason=None): @@ -199,10 +201,37 @@ async def close_impl(self): self.notify_state(ConnectionState.CLOSED) async def send_protocol_message(self, protocol_message): - if self.transport is not None: - await self.transport.send(protocol_message) - else: - raise Exception() + if self.state in ( + ConnectionState.DISCONNECTED, + ConnectionState.CONNECTING, + ): + self.queued_messages.put(protocol_message) + return + + if self.state == ConnectionState.CONNECTED: + if self.transport: + await self.transport.send(protocol_message) + else: + log.exception( + "ConnectionManager.send_protocol_message(): can not send message with no active transport" + ) + return + + raise AblyException(f"ConnectionManager.send_protocol_message(): called in {self.state}", 500, 50000) + + def send_queued_messages(self): + log.info(f'ConnectionManager.send_queued_messages(): sending {self.queued_messages.qsize()} message(s)') + while not self.queued_messages.empty(): + asyncio.create_task(self.send_protocol_message(self.queued_messages.get())) + + def fail_queued_messages(self, err): + log.info( + f"ConnectionManager.fail_queued_messages(): discarding {self.queued_messages.qsize()} messages;" + + f" reason = {err}" + ) + while not self.queued_messages.empty(): + msg = self.queued_messages.get() + log.exception(f"ConnectionManager.fail_queued_messages(): Failed to send protocol message: {msg}") async def ping(self): if self.__ping_future: @@ -399,12 +428,15 @@ def notify_state(self, state: ConnectionState, reason=None): self.enact_state_change(state, reason) - if state in ( + if state == ConnectionState.CONNECTED: + self.send_queued_messages() + elif state in ( ConnectionState.CLOSING, ConnectionState.CLOSED, ConnectionState.SUSPENDED, ConnectionState.FAILED, ): + self.fail_queued_messages(reason) self.ably.channels._propagate_connection_interruption(state, reason) def start_transition_timer(self, state: ConnectionState, fail_state=None): From 43b249f6402aa73da9eac967a2510031d6af4db6 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 26 Jan 2023 13:07:55 +0000 Subject: [PATCH 340/481] refactor(RealtimeChannel): queue attach message when CONNECTING/DISCONNECTED --- ably/realtime/realtime_channel.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 10b88c55..e6abb24c 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -89,7 +89,11 @@ async def attach(self): return # RTL4b - if self.__realtime.connection.state not in [ConnectionState.CONNECTING, ConnectionState.CONNECTED]: + if self.__realtime.connection.state not in [ + ConnectionState.CONNECTING, + ConnectionState.CONNECTED, + ConnectionState.DISCONNECTED + ]: raise AblyException( message=f"Unable to attach; channel state = {self.state}", code=90001, @@ -99,10 +103,6 @@ async def attach(self): if self.state != ChannelState.ATTACHING: self._request_state(ChannelState.ATTACHING) - # RTL4i - wait for pending connection - if self.__realtime.connection.state == ConnectionState.CONNECTING: - await self.__realtime.connect() - state_change = await self.__internal_state_emitter.once_async() if state_change.current in (ChannelState.SUSPENDED, ChannelState.FAILED): From 81fc3fd02274f711fe1d4480886587a270994018 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 26 Jan 2023 13:08:25 +0000 Subject: [PATCH 341/481] test: add fixture for attaching whilst CONNECTING --- test/ably/realtimechannel_test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py index cbe16d40..fa737d09 100644 --- a/test/ably/realtimechannel_test.py +++ b/test/ably/realtimechannel_test.py @@ -283,3 +283,10 @@ async def test_channel_suspended_once_connection_suspended(self): assert channel.state == ChannelState.FAILED await ably.close() + + async def test_attach_while_connecting(self): + ably = await RestSetup.get_ably_realtime() + channel = ably.channels.get(random_string(5)) + await channel.attach() + assert channel.state == ChannelState.ATTACHED + await ably.close() From db4a30037c22adac9c2def659a061a8119d105e6 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 30 Jan 2023 15:23:11 +0000 Subject: [PATCH 342/481] refactor websocket transport to accept params --- ably/realtime/connection.py | 7 ++++++- ably/realtime/websockettransport.py | 7 +++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 1f18a4af..9bcdc6f5 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -185,6 +185,10 @@ def check_connection(self): except httpx.HTTPError: return False + def __get_transport_params(self): + protocol_version = Defaults.protocol_version + return {"key": self.__ably.key, "v": protocol_version} + async def close_impl(self): log.debug('ConnectionManager.close_impl()') @@ -374,7 +378,8 @@ async def connect_base(self): self.notify_state(self.__fail_state, reason=exception) async def try_host(self, host): - self.transport = WebSocketTransport(self, host) + params = self.__get_transport_params() + self.transport = WebSocketTransport(self, host, params) self._emit('transport.pending', self.transport) self.transport.connect() diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index fef9304d..5d55c801 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -36,7 +36,7 @@ class ProtocolMessageAction(IntEnum): class WebSocketTransport(EventEmitter): - def __init__(self, connection_manager: ConnectionManager, host: str): + def __init__(self, connection_manager: ConnectionManager, host: str, params: dict): self.websocket: WebSocketClientProtocol | None = None self.read_loop: asyncio.Task | None = None self.connect_task: asyncio.Task | None = None @@ -49,13 +49,12 @@ def __init__(self, connection_manager: ConnectionManager, host: str): self.max_idle_interval = None self.is_disposed = False self.host = host + self.params = params super().__init__() def connect(self): headers = HttpUtils.default_headers() - protocol_version = Defaults.protocol_version - params = {"key": self.connection_manager.ably.key, "v": protocol_version} - query_params = urllib.parse.urlencode(params) + query_params = urllib.parse.urlencode(self.params) ws_url = (f'wss://{self.host}?{query_params}') log.info(f'connect(): attempting to connect to {ws_url}') self.ws_connect_task = asyncio.create_task(self.ws_connect(ws_url, headers)) From e5fb0ffa95ed0de69c2c4c51b384f5311a282bc0 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 30 Jan 2023 15:28:50 +0000 Subject: [PATCH 343/481] update connection details with connection key and id --- ably/types/connectiondetails.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ably/types/connectiondetails.py b/ably/types/connectiondetails.py index c338f6ea..c25c1ccb 100644 --- a/ably/types/connectiondetails.py +++ b/ably/types/connectiondetails.py @@ -5,11 +5,17 @@ class ConnectionDetails: connection_state_ttl: int max_idle_interval: int + connection_key: str + connection_id: str - def __init__(self, connection_state_ttl: int, max_idle_interval: int): + def __init__(self, connection_state_ttl: int, max_idle_interval: int, + connection_key: str, connection_id: str): self.connection_state_ttl = connection_state_ttl self.max_idle_interval = max_idle_interval + self.connection_key = connection_key + self.connection_id = connection_id @staticmethod def from_dict(json_dict: dict): - return ConnectionDetails(json_dict.get('connectionStateTtl'), json_dict.get('maxIdleInterval')) + return ConnectionDetails(json_dict.get('connectionStateTtl'), json_dict.get('maxIdleInterval'), + json_dict.get('connectionKey'), json_dict.get('connectionId')) \ No newline at end of file From f56b00ce38938f32a507a61f98b17eb5aeeb70b6 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 30 Jan 2023 15:37:15 +0000 Subject: [PATCH 344/481] send connection key on resume --- ably/realtime/connection.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 9bcdc6f5..7cf72319 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -187,7 +187,10 @@ def check_connection(self): def __get_transport_params(self): protocol_version = Defaults.protocol_version - return {"key": self.__ably.key, "v": protocol_version} + params = {"key": self.__ably.key, "v": protocol_version} + if self.connection_details: + params["resume"] = self.connection_details.connection_key + return params async def close_impl(self): log.debug('ConnectionManager.close_impl()') From 829bef7b98cf870c5853a7fa514634639c15b391 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 30 Jan 2023 15:57:42 +0000 Subject: [PATCH 345/481] test send resume param --- ably/realtime/websockettransport.py | 1 - ably/types/connectiondetails.py | 2 +- test/ably/realtimeresume_test.py | 25 +++++++++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 test/ably/realtimeresume_test.py diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index 5d55c801..0fac18cb 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -7,7 +7,6 @@ import socket import urllib.parse from ably.http.httputils import HttpUtils -from ably.transport.defaults import Defaults from ably.types.connectiondetails import ConnectionDetails from ably.util.eventemitter import EventEmitter from ably.util.exceptions import AblyException diff --git a/ably/types/connectiondetails.py b/ably/types/connectiondetails.py index c25c1ccb..eceb3968 100644 --- a/ably/types/connectiondetails.py +++ b/ably/types/connectiondetails.py @@ -18,4 +18,4 @@ def __init__(self, connection_state_ttl: int, max_idle_interval: int, @staticmethod def from_dict(json_dict: dict): return ConnectionDetails(json_dict.get('connectionStateTtl'), json_dict.get('maxIdleInterval'), - json_dict.get('connectionKey'), json_dict.get('connectionId')) \ No newline at end of file + json_dict.get('connectionKey'), json_dict.get('connectionId')) diff --git a/test/ably/realtimeresume_test.py b/test/ably/realtimeresume_test.py new file mode 100644 index 00000000..c257ccfb --- /dev/null +++ b/test/ably/realtimeresume_test.py @@ -0,0 +1,25 @@ +from ably.realtime.connection import ConnectionState +from test.ably.restsetup import RestSetup +from test.ably.utils import BaseAsyncTestCase + + +class TestRealtimeResume(BaseAsyncTestCase): + async def asyncSetUp(self): + self.test_vars = await RestSetup.get_test_vars() + self.valid_key_format = "api:key" + + async def test_connection_resume(self): + ably = await RestSetup.get_ably_realtime() + + await ably.connection.once_async(ConnectionState.CONNECTED) + prev_connection_id = ably.connection.connection_details.connection_id + connection_key = ably.connection.connection_details.connection_key + await ably.connection.connection_manager.transport.dispose() + ably.connection.connection_manager.notify_state(ConnectionState.DISCONNECTED) + + await ably.connection.once_async(ConnectionState.CONNECTED) + new_connection_id = ably.connection.connection_details.connection_id + assert ably.connection.connection_manager.transport.params["resume"] == connection_key + assert prev_connection_id == new_connection_id + + await ably.close() From 0359142c11d56fa02c5ec431553e2230fcf06c67 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 30 Jan 2023 22:32:08 +0000 Subject: [PATCH 346/481] fix: don't check channel/connection state in subscribe Fixes a few issues: 1. subscribe was using the old async `connect` signature 2. subscribe wasn't raising exception when DISCONNECTED 3. listeners would not be attached if `attach` raised --- ably/realtime/realtime_channel.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 10b88c55..02823a57 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -224,19 +224,6 @@ async def subscribe(self, *args): log.info(f'RealtimeChannel.subscribe called, channel = {self.name}, event = {event}') - if self.__realtime.connection.state == ConnectionState.CONNECTING: - await self.__realtime.connection.connect() - elif self.__realtime.connection.state != ConnectionState.CONNECTED: - raise AblyException( - 'Cannot subscribe to channel, invalid connection state: {self.__realtime.connection.state}', - 400, - 40000 - ) - - # RTL7c - if self.state in (ChannelState.INITIALIZED, ChannelState.ATTACHING, ChannelState.DETACHED): - await self.attach() - if event is not None: # RTL7b self.__message_emitter.on(event, listener) @@ -244,6 +231,7 @@ async def subscribe(self, *args): # RTL7a self.__message_emitter.on(listener) + # RTL7c await self.attach() # RTL8 From c2bc3c51bd172f27da22513d8c22c190bb1451da Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 31 Jan 2023 11:36:50 +0000 Subject: [PATCH 347/481] fix connection_id error --- ably/realtime/connection.py | 4 +++- ably/realtime/websockettransport.py | 3 ++- ably/types/connectiondetails.py | 6 ++---- test/ably/realtimeresume_test.py | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 7cf72319..73081b33 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -161,6 +161,7 @@ def __init__(self, realtime, initial_state): self.__timeout_in_secs = self.options.realtime_request_timeout / 1000 self.transport: WebSocketTransport | None = None self.__connection_details = None + self.connection_id = None self.__fail_state = ConnectionState.DISCONNECTED self.transition_timer: Timer | None = None self.suspend_timer: Timer | None = None @@ -265,10 +266,11 @@ async def ping(self): response_time_ms = (ping_end_time - ping_start_time) * 1000 return round(response_time_ms, 2) - def on_connected(self, connection_details: ConnectionDetails): + def on_connected(self, connection_details: ConnectionDetails, connection_id: str): self.__fail_state = ConnectionState.DISCONNECTED self.__connection_details = connection_details + self.connection_id = connection_id if self.__state == ConnectionState.CONNECTED: state_change = ConnectionStateChange(ConnectionState.CONNECTED, ConnectionState.CONNECTED, diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index 0fac18cb..4904cbde 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -95,6 +95,7 @@ async def on_protocol_message(self, msg): log.info(f'WebSocketTransport.on_protocol_message(): receieved protocol message: {msg}') action = msg.get('action') if action == ProtocolMessageAction.CONNECTED: + connection_id = msg.get('connectionId') connection_details = ConnectionDetails.from_dict(msg.get('connectionDetails')) max_idle_interval = connection_details.max_idle_interval if max_idle_interval: @@ -103,7 +104,7 @@ async def on_protocol_message(self, msg): self.is_connected = True if self.host != self.options.get_realtime_host(): # RTN17e self.options.fallback_realtime_host = self.host - self.connection_manager.on_connected(connection_details) + self.connection_manager.on_connected(connection_details, connection_id) elif action == ProtocolMessageAction.DISCONNECTED: self.connection_manager.on_disconnected(msg) elif action == ProtocolMessageAction.CLOSED: diff --git a/ably/types/connectiondetails.py b/ably/types/connectiondetails.py index eceb3968..8fc98cf4 100644 --- a/ably/types/connectiondetails.py +++ b/ably/types/connectiondetails.py @@ -6,16 +6,14 @@ class ConnectionDetails: connection_state_ttl: int max_idle_interval: int connection_key: str - connection_id: str def __init__(self, connection_state_ttl: int, max_idle_interval: int, - connection_key: str, connection_id: str): + connection_key: str): self.connection_state_ttl = connection_state_ttl self.max_idle_interval = max_idle_interval self.connection_key = connection_key - self.connection_id = connection_id @staticmethod def from_dict(json_dict: dict): return ConnectionDetails(json_dict.get('connectionStateTtl'), json_dict.get('maxIdleInterval'), - json_dict.get('connectionKey'), json_dict.get('connectionId')) + json_dict.get('connectionKey')) diff --git a/test/ably/realtimeresume_test.py b/test/ably/realtimeresume_test.py index c257ccfb..a4fba059 100644 --- a/test/ably/realtimeresume_test.py +++ b/test/ably/realtimeresume_test.py @@ -12,13 +12,13 @@ async def test_connection_resume(self): ably = await RestSetup.get_ably_realtime() await ably.connection.once_async(ConnectionState.CONNECTED) - prev_connection_id = ably.connection.connection_details.connection_id + prev_connection_id = ably.connection.connection_manager.connection_id connection_key = ably.connection.connection_details.connection_key await ably.connection.connection_manager.transport.dispose() ably.connection.connection_manager.notify_state(ConnectionState.DISCONNECTED) await ably.connection.once_async(ConnectionState.CONNECTED) - new_connection_id = ably.connection.connection_details.connection_id + new_connection_id = ably.connection.connection_manager.connection_id assert ably.connection.connection_manager.transport.params["resume"] == connection_key assert prev_connection_id == new_connection_id From 58bc86118dbd27dadc46dc84401056d0f124bf80 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 30 Jan 2023 16:14:12 +0000 Subject: [PATCH 348/481] add test for fatal resume --- test/ably/realtimeresume_test.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/ably/realtimeresume_test.py b/test/ably/realtimeresume_test.py index a4fba059..cc397a44 100644 --- a/test/ably/realtimeresume_test.py +++ b/test/ably/realtimeresume_test.py @@ -23,3 +23,17 @@ async def test_connection_resume(self): assert prev_connection_id == new_connection_id await ably.close() + + async def test_fatal_resume_error(self): + ably = await RestSetup.get_ably_realtime() + + await ably.connection.once_async(ConnectionState.CONNECTED) + key_name = ably.options.key_name + ably.key = f"{key_name}:wrong-secret" + await ably.connection.connection_manager.transport.dispose() + ably.connection.connection_manager.notify_state(ConnectionState.DISCONNECTED) + + state_change = await ably.connection.once_async(ConnectionState.FAILED) + assert state_change.reason.code == 40101 + assert state_change.reason.status_code == 401 + await ably.close() From df4d78647362de404f8007773cd10e6d906e39ef Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 30 Jan 2023 16:47:09 +0000 Subject: [PATCH 349/481] refactor: emit error reasons from CONNECTED messages --- ably/realtime/connection.py | 4 ++-- ably/realtime/websockettransport.py | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 73081b33..c0ae654b 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -266,7 +266,7 @@ async def ping(self): response_time_ms = (ping_end_time - ping_start_time) * 1000 return round(response_time_ms, 2) - def on_connected(self, connection_details: ConnectionDetails, connection_id: str): + def on_connected(self, connection_details: ConnectionDetails, connection_id: str, reason=None): self.__fail_state = ConnectionState.DISCONNECTED self.__connection_details = connection_details @@ -277,7 +277,7 @@ def on_connected(self, connection_details: ConnectionDetails, connection_id: str ConnectionEvent.UPDATE) self._emit(ConnectionEvent.UPDATE, state_change) else: - self.notify_state(ConnectionState.CONNECTED) + self.notify_state(ConnectionState.CONNECTED, reason=reason) def on_disconnected(self, msg: dict): error = msg.get("error") diff --git a/ably/realtime/websockettransport.py b/ably/realtime/websockettransport.py index 4904cbde..949e06b3 100644 --- a/ably/realtime/websockettransport.py +++ b/ably/realtime/websockettransport.py @@ -97,6 +97,12 @@ async def on_protocol_message(self, msg): if action == ProtocolMessageAction.CONNECTED: connection_id = msg.get('connectionId') connection_details = ConnectionDetails.from_dict(msg.get('connectionDetails')) + + error = msg.get('error') + exception = None + if error: + exception = AblyException(error.get('message'), error.get('statusCode'), error.get('code')) + max_idle_interval = connection_details.max_idle_interval if max_idle_interval: self.max_idle_interval = max_idle_interval + self.options.realtime_request_timeout @@ -104,7 +110,7 @@ async def on_protocol_message(self, msg): self.is_connected = True if self.host != self.options.get_realtime_host(): # RTN17e self.options.fallback_realtime_host = self.host - self.connection_manager.on_connected(connection_details, connection_id) + self.connection_manager.on_connected(connection_details, connection_id, reason=exception) elif action == ProtocolMessageAction.DISCONNECTED: self.connection_manager.on_disconnected(msg) elif action == ProtocolMessageAction.CLOSED: From 5ebb436513d6bfa9873b90f9d52b21ae072d7f30 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 30 Jan 2023 17:11:32 +0000 Subject: [PATCH 350/481] feat: reattach channels upon connection --- ably/realtime/connection.py | 2 ++ ably/realtime/realtime.py | 11 +++++++++++ ably/realtime/realtime_channel.py | 10 +++++++--- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index c0ae654b..5095a79b 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -279,6 +279,8 @@ def on_connected(self, connection_details: ConnectionDetails, connection_id: str else: self.notify_state(ConnectionState.CONNECTED, reason=reason) + self.ably.channels._on_connected() + def on_disconnected(self, msg: dict): error = msg.get("error") exception = AblyException(error.get('message'), error.get('statusCode'), error.get('code')) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 6fd0bc80..a3987ef4 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -238,3 +238,14 @@ def _propagate_connection_interruption(self, state: ConnectionState, reason): channel = self.all[name] if channel.state in from_channel_states: channel._notify_state(connection_to_channel_state[state], reason) + + def _on_connected(self): + for channel_name in self.all.keys(): + channel = self.all[channel_name] + + if channel.state == ChannelState.ATTACHING or channel.state == ChannelState.DETACHING: + channel._check_pending_state() + elif channel.state == ChannelState.SUSPENDED: + asyncio.create_task(channel.attach()) + elif channel.state == ChannelState.ATTACHED: + channel._request_state(ChannelState.ATTACHING) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index e6abb24c..84a7bc8a 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -320,7 +320,7 @@ def _on_message(self, msg): def _request_state(self, state: ChannelState): log.info(f'RealtimeChannel._request_state(): state = {state}') self._notify_state(state) - self.__check_pending_state() + self._check_pending_state() def _notify_state(self, state: ChannelState, reason=None): log.info(f'RealtimeChannel._notify_state(): state = {state}') @@ -339,7 +339,7 @@ def _notify_state(self, state: ChannelState, reason=None): def _send_message(self, msg): asyncio.create_task(self.__realtime.connection.connection_manager.send_protocol_message(msg)) - def __check_pending_state(self): + def _check_pending_state(self): connection_state = self.__realtime.connection.connection_manager.state if connection_state not in ( @@ -377,7 +377,7 @@ def __timeout_pending_state(self): elif self.state == ChannelState.DETACHING: self._notify_state(ChannelState.ATTACHED, reason=AblyException("Channel detach timed out", 408, 90007)) else: - self.__check_pending_state() + self._check_pending_state() # RTL23 @property @@ -390,3 +390,7 @@ def name(self): def state(self): """Returns channel state""" return self.__state + + @state.setter + def state(self, state: ChannelState): + self.__state = state From 9d4cfc15e14b7a96b325e7216569d1bb123b4c91 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 30 Jan 2023 17:15:31 +0000 Subject: [PATCH 351/481] test: add tests for invalid resume response --- test/ably/realtimeresume_test.py | 70 +++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/test/ably/realtimeresume_test.py b/test/ably/realtimeresume_test.py index cc397a44..af0bacf8 100644 --- a/test/ably/realtimeresume_test.py +++ b/test/ably/realtimeresume_test.py @@ -1,6 +1,7 @@ from ably.realtime.connection import ConnectionState +from ably.realtime.realtime_channel import ChannelState from test.ably.restsetup import RestSetup -from test.ably.utils import BaseAsyncTestCase +from test.ably.utils import BaseAsyncTestCase, random_string class TestRealtimeResume(BaseAsyncTestCase): @@ -37,3 +38,70 @@ async def test_fatal_resume_error(self): assert state_change.reason.code == 40101 assert state_change.reason.status_code == 401 await ably.close() + + async def test_invalid_resume_response(self): + ably = await RestSetup.get_ably_realtime() + + await ably.connection.once_async(ConnectionState.CONNECTED) + + assert ably.connection.connection_manager.connection_details + ably.connection.connection_manager.connection_details.connection_key = 'ably-python-fake-key' + + assert ably.connection.connection_manager.transport + await ably.connection.connection_manager.transport.dispose() + ably.connection.connection_manager.notify_state(ConnectionState.DISCONNECTED) + + state_change = await ably.connection.once_async(ConnectionState.CONNECTED) + + assert state_change.reason.code == 80018 + assert state_change.reason.status_code == 400 + assert ably.connection.error_reason == state_change.reason + + await ably.close() + + async def test_attached_channel_reattaches_on_invalid_resume(self): + ably = await RestSetup.get_ably_realtime() + + await ably.connection.once_async(ConnectionState.CONNECTED) + + channel = ably.channels.get(random_string(5)) + + await channel.attach() + + assert ably.connection.connection_manager.connection_details + ably.connection.connection_manager.connection_details.connection_key = 'ably-python-fake-key' + + assert ably.connection.connection_manager.transport + await ably.connection.connection_manager.transport.dispose() + ably.connection.connection_manager.notify_state(ConnectionState.DISCONNECTED) + + await ably.connection.once_async(ConnectionState.CONNECTED) + + assert channel.state == ChannelState.ATTACHING + + await channel.once_async(ChannelState.ATTACHED) + + await ably.close() + + async def test_suspended_channel_reattaches_on_invalid_resume(self): + ably = await RestSetup.get_ably_realtime() + + await ably.connection.once_async(ConnectionState.CONNECTED) + + channel = ably.channels.get(random_string(5)) + channel.state = ChannelState.SUSPENDED + + assert ably.connection.connection_manager.connection_details + ably.connection.connection_manager.connection_details.connection_key = 'ably-python-fake-key' + + assert ably.connection.connection_manager.transport + await ably.connection.connection_manager.transport.dispose() + ably.connection.connection_manager.notify_state(ConnectionState.DISCONNECTED) + + await ably.connection.once_async(ConnectionState.CONNECTED) + + assert channel.state == ChannelState.ATTACHING + + await channel.once_async(ChannelState.ATTACHED) + + await ably.close() From 81ebff3f85a2d2f398dc77d57b2b30a080539a8e Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 30 Jan 2023 17:20:00 +0000 Subject: [PATCH 352/481] doc: add spec point annotations to resume tests --- test/ably/realtimeresume_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/ably/realtimeresume_test.py b/test/ably/realtimeresume_test.py index af0bacf8..7aba71f4 100644 --- a/test/ably/realtimeresume_test.py +++ b/test/ably/realtimeresume_test.py @@ -9,6 +9,7 @@ async def asyncSetUp(self): self.test_vars = await RestSetup.get_test_vars() self.valid_key_format = "api:key" + # RTN15c6 - valid resume response async def test_connection_resume(self): ably = await RestSetup.get_ably_realtime() @@ -25,6 +26,7 @@ async def test_connection_resume(self): await ably.close() + # RTN15c4 - fatal resume error async def test_fatal_resume_error(self): ably = await RestSetup.get_ably_realtime() @@ -39,6 +41,7 @@ async def test_fatal_resume_error(self): assert state_change.reason.status_code == 401 await ably.close() + # RTN15c7 - invalid resume response async def test_invalid_resume_response(self): ably = await RestSetup.get_ably_realtime() From aa686d842764d15791be6083946a50a60218116e Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 31 Jan 2023 00:05:38 +0000 Subject: [PATCH 353/481] fix: don't queue pending channel messages when DISCONNECTED/CONNECTING this is now handled by `Channels._on_connect()` so no longer needed --- ably/realtime/realtime_channel.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 84a7bc8a..660ff02b 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -342,11 +342,8 @@ def _send_message(self, msg): def _check_pending_state(self): connection_state = self.__realtime.connection.connection_manager.state - if connection_state not in ( - ConnectionState.CONNECTING, - ConnectionState.CONNECTED, - ConnectionState.DISCONNECTED, - ): + if connection_state is not ConnectionState.CONNECTED: + log.info(f"RealtimeChannel._check_pending_state(): connection state = {connection_state}") return if self.state == ChannelState.ATTACHING: From b2e4191751cddaed9d7b4ac90fdf9374a92abd89 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 31 Jan 2023 15:57:02 +0000 Subject: [PATCH 354/481] refactor: add retry_immediately kwarg to notify_state --- ably/realtime/connection.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 5095a79b..c5f78d06 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -413,9 +413,10 @@ async def on_transport_failed(exception): except asyncio.CancelledError: return - def notify_state(self, state: ConnectionState, reason=None): + def notify_state(self, state: ConnectionState, reason=None, retry_immediately=None): # RTN15a - retry_immediately = state == ConnectionState.DISCONNECTED and self.__state == ConnectionState.CONNECTED + retry_immediately = (retry_immediately is not False) and ( + state == ConnectionState.DISCONNECTED and self.__state == ConnectionState.CONNECTED) log.info( f'ConnectionManager.notify_state(): new state: {state}' From 020a10acead75c836a80b462711678b0f73b0a55 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 31 Jan 2023 15:58:49 +0000 Subject: [PATCH 355/481] refactor: add ChannelStateChange.resumed field --- ably/realtime/realtime_channel.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index e53b4d59..10a12981 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -29,6 +29,7 @@ class ChannelState(str, Enum): class ChannelStateChange: previous: ChannelState current: ChannelState + resumed: bool reason: Optional[AblyException] = None @@ -310,7 +311,7 @@ def _request_state(self, state: ChannelState): self._notify_state(state) self._check_pending_state() - def _notify_state(self, state: ChannelState, reason=None): + def _notify_state(self, state: ChannelState, reason=None, resumed=False): log.info(f'RealtimeChannel._notify_state(): state = {state}') self.__clear_state_timer() @@ -318,7 +319,7 @@ def _notify_state(self, state: ChannelState, reason=None): if state == self.state: return - state_change = ChannelStateChange(self.__state, state, reason=reason) + state_change = ChannelStateChange(self.__state, state, resumed, reason=reason) self.__state = state self._emit(state, state_change) From 13a2deca6dea6526bf0dd8e7ad3cb07b02b85b22 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 31 Jan 2023 16:12:49 +0000 Subject: [PATCH 356/481] refactor: add Flags enum --- ably/realtime/realtime_channel.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 10a12981..c0f1f9f1 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -25,6 +25,20 @@ class ChannelState(str, Enum): FAILED = 'failed' +class Flags(int, Enum): + # Channel attach state flags + HAS_PRESENCE = 1 << 0 + HAS_BACKLOG = 1 << 1 + RESUMED = 1 << 2 + TRANSIENT = 1 << 4 + ATTACH_RESUME = 1 << 5 + # Channel mode flags + PRESENCE = 1 << 16 + PUBLISH = 1 << 17 + SUBSCRIBE = 1 << 18 + PRESENCE_SUBSCRIBE = 1 << 19 + + @dataclass class ChannelStateChange: previous: ChannelState From 74daa8de6b39385b168aefb4be208619bd53d687 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 31 Jan 2023 16:31:54 +0000 Subject: [PATCH 357/481] feat: send ATTACH_RESUME flag on unclean attach --- ably/realtime/realtime_channel.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index c0f1f9f1..396d57d9 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -25,7 +25,7 @@ class ChannelState(str, Enum): FAILED = 'failed' -class Flags(int, Enum): +class Flag(int, Enum): # Channel attach state flags HAS_PRESENCE = 1 << 0 HAS_BACKLOG = 1 << 1 @@ -39,6 +39,10 @@ class Flags(int, Enum): PRESENCE_SUBSCRIBE = 1 << 19 +def has_flag(message_flags: int, flag: Flag): + return message_flags & flag > 0 + + @dataclass class ChannelStateChange: previous: ChannelState @@ -77,6 +81,7 @@ def __init__(self, realtime, name): self.__state = ChannelState.INITIALIZED self.__message_emitter = EventEmitter() self.__state_timer: Timer | None = None + self.__attach_resume = False # Used to listen to state changes internally, if we use the public event emitter interface then internals # will be disrupted if the user called .off() to remove all listeners @@ -131,6 +136,10 @@ def _attach_impl(self): "action": ProtocolMessageAction.ATTACH, "channel": self.name, } + + if self.__attach_resume: + attach_msg["flags"] = Flag.ATTACH_RESUME + self._send_message(attach_msg) # RTL5 @@ -307,7 +316,9 @@ def _on_message(self, msg): action = msg.get('action') if action == ProtocolMessageAction.ATTACHED: if self.state == ChannelState.ATTACHING: - self._notify_state(ChannelState.ATTACHED) + flags = msg.get('flags') + resumed = has_flag(flags, Flag.RESUMED) + self._notify_state(ChannelState.ATTACHED, resumed=resumed) else: log.warn("RealtimeChannel._on_message(): ATTACHED received while not attaching") elif action == ProtocolMessageAction.DETACHED: @@ -333,6 +344,12 @@ def _notify_state(self, state: ChannelState, reason=None, resumed=False): if state == self.state: return + # RTL4j1 + if state == ChannelState.ATTACHED: + self.__attach_resume = True + if state in (ChannelState.DETACHING, ChannelState.FAILED): + self.__attach_resume = False + state_change = ChannelStateChange(self.__state, state, resumed, reason=reason) self.__state = state From 0253ccf9c6d9cd1e6c58b298b1eba0ffeb02997a Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 31 Jan 2023 16:40:08 +0000 Subject: [PATCH 358/481] feat: send channelSerial in ATTACH messages --- ably/realtime/realtime_channel.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 396d57d9..885cf220 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -82,6 +82,7 @@ def __init__(self, realtime, name): self.__message_emitter = EventEmitter() self.__state_timer: Timer | None = None self.__attach_resume = False + self.__channel_serial: str | None = None # Used to listen to state changes internally, if we use the public event emitter interface then internals # will be disrupted if the user called .off() to remove all listeners @@ -139,6 +140,8 @@ def _attach_impl(self): if self.__attach_resume: attach_msg["flags"] = Flag.ATTACH_RESUME + if self.__channel_serial: + attach_msg["channelSerial"] = self.__channel_serial self._send_message(attach_msg) @@ -314,6 +317,12 @@ def unsubscribe(self, *args): def _on_message(self, msg): action = msg.get('action') + + # RTL4c1 + channel_serial = msg.get('channelSerial') + if channel_serial: + self.__channel_serial = channel_serial + if action == ProtocolMessageAction.ATTACHED: if self.state == ChannelState.ATTACHING: flags = msg.get('flags') @@ -350,6 +359,10 @@ def _notify_state(self, state: ChannelState, reason=None, resumed=False): if state in (ChannelState.DETACHING, ChannelState.FAILED): self.__attach_resume = False + # RTP5a1 + if state in (ChannelState.DETACHED, ChannelState.SUSPENDED, ChannelState.FAILED): + self.__channel_serial = None + state_change = ChannelStateChange(self.__state, state, resumed, reason=reason) self.__state = state From 5da304924bc45191f725607eb63e9dc1d1b6bed7 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 31 Jan 2023 16:40:21 +0000 Subject: [PATCH 359/481] test: add test for channel resume behaviour --- test/ably/realtimeresume_test.py | 61 ++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/test/ably/realtimeresume_test.py b/test/ably/realtimeresume_test.py index 7aba71f4..58570f46 100644 --- a/test/ably/realtimeresume_test.py +++ b/test/ably/realtimeresume_test.py @@ -1,9 +1,24 @@ +import asyncio from ably.realtime.connection import ConnectionState from ably.realtime.realtime_channel import ChannelState from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase, random_string +async def send_and_await(rest_channel, realtime_channel): + event = random_string(5) + message = random_string(5) + future = asyncio.Future() + + def on_message(_): + future.set_result(None) + + await realtime_channel.subscribe(event, on_message) + await rest_channel.publish(event, message) + + await future + + class TestRealtimeResume(BaseAsyncTestCase): async def asyncSetUp(self): self.test_vars = await RestSetup.get_test_vars() @@ -108,3 +123,49 @@ async def test_suspended_channel_reattaches_on_invalid_resume(self): await channel.once_async(ChannelState.ATTACHED) await ably.close() + + async def test_resume_receives_channel_messages_while_disconnected(self): + realtime = await RestSetup.get_ably_realtime() + rest = await RestSetup.get_ably_rest() + + channel_name = random_string(5) + + realtime_channel = realtime.channels.get(channel_name) + rest_channel = rest.channels.get(channel_name) + + await realtime.connection.once_async(ConnectionState.CONNECTED) + + asyncio.create_task(realtime_channel.attach()) + state_change = await realtime_channel.once_async(ChannelState.ATTACHED) + assert state_change.resumed is False + + await send_and_await(rest_channel, realtime_channel) + + assert realtime.connection.connection_manager.transport + await realtime.connection.connection_manager.transport.dispose() + realtime.connection.connection_manager.notify_state(ConnectionState.DISCONNECTED, retry_immediately=False) + + event_name = random_string(5) + message = random_string(5) + await rest_channel.publish(event_name, message) + + future = asyncio.Future() + + def on_message(message): + future.set_result(message) + + await realtime_channel.subscribe(event_name, on_message) + + realtime.connect() + await realtime.connection.once_async(ConnectionState.CONNECTED) + + state_change = await realtime_channel.once_async(ChannelState.ATTACHED) + + assert state_change.resumed is True + + received_message = await future + + assert received_message.data == message + + await realtime.close() + await rest.close() From b2b4ce176c4f6e6e153330b9ab98db67cbb218c7 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 31 Jan 2023 18:14:51 +0000 Subject: [PATCH 360/481] implement update event on already attached channel --- ably/realtime/realtime_channel.py | 18 +++++++++++++++-- test/ably/realtimeresume_test.py | 33 +++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 885cf220..9a162b2b 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -324,9 +324,23 @@ def _on_message(self, msg): self.__channel_serial = channel_serial if action == ProtocolMessageAction.ATTACHED: - if self.state == ChannelState.ATTACHING: - flags = msg.get('flags') + flags = msg.get('flags') + error = msg.get("error") + exception = None + resumed = None + + if error: + exception = AblyException(error.get('message'), error.get('statusCode'), error.get('code')) + + if flags: resumed = has_flag(flags, Flag.RESUMED) + + # RTL12 + if self.state == ChannelState.ATTACHED: + if not resumed: + state_change = ChannelStateChange(self.state, ChannelState.ATTACHED, resumed, exception) + self._emit("update", state_change) + elif self.state == ChannelState.ATTACHING: self._notify_state(ChannelState.ATTACHED, resumed=resumed) else: log.warn("RealtimeChannel._on_message(): ATTACHED received while not attaching") diff --git a/test/ably/realtimeresume_test.py b/test/ably/realtimeresume_test.py index 58570f46..da3e9e42 100644 --- a/test/ably/realtimeresume_test.py +++ b/test/ably/realtimeresume_test.py @@ -1,6 +1,7 @@ import asyncio from ably.realtime.connection import ConnectionState from ably.realtime.realtime_channel import ChannelState +from ably.realtime.websockettransport import ProtocolMessageAction from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase, random_string @@ -169,3 +170,35 @@ def on_message(message): await realtime.close() await rest.close() + + async def test_resume_update_channel_attached(self): + realtime = await RestSetup.get_ably_realtime() + + name = random_string(5) + channel = realtime.channels.get(name) + await channel.attach() + error_code = 123 + error_status_code = 456 + error_message = "some error" + message = { + "action": ProtocolMessageAction.ATTACHED, + "channel": name, + "error": { + "code": error_code, + "statusCode": error_status_code, + "message": error_message + } + } + future = asyncio.Future() + + def on_update(state_change): + future.set_result(state_change) + + channel.once("update", on_update) + await realtime.connection.connection_manager.transport.on_protocol_message(message) + + state_change = await future + assert state_change.reason.code == error_code + assert state_change.reason.status_code == error_status_code + assert state_change.reason.message == error_message + await realtime.close() From f31f9b1a8e0b95f6d4c0f9a78a44401e77d66134 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 1 Feb 2023 12:24:13 +0000 Subject: [PATCH 361/481] refactor: move ConnectionManager into its own module --- ably/realtime/connection.py | 440 +-------------------------- ably/realtime/connectionmanager.py | 410 +++++++++++++++++++++++++ ably/realtime/realtime_channel.py | 3 +- ably/types/connectionstate.py | 36 +++ test/ably/realtimechannel_test.py | 3 +- test/ably/realtimeconnection_test.py | 3 +- 6 files changed, 454 insertions(+), 441 deletions(-) create mode 100644 ably/realtime/connectionmanager.py create mode 100644 ably/types/connectionstate.py diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index c5f78d06..bf473597 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -1,53 +1,12 @@ import functools import logging -import asyncio -import httpx -from ably.realtime.websockettransport import WebSocketTransport, ProtocolMessageAction -from ably.transport.defaults import Defaults -from ably.util.exceptions import AblyException +from ably.realtime.connectionmanager import ConnectionManager +from ably.types.connectionstate import ConnectionEvent, ConnectionState from ably.util.eventemitter import EventEmitter -from enum import Enum -from datetime import datetime -from ably.util.helper import get_random_id, Timer -from dataclasses import dataclass -from typing import Optional -from ably.types.connectiondetails import ConnectionDetails -from queue import Queue log = logging.getLogger(__name__) -class ConnectionState(str, Enum): - INITIALIZED = 'initialized' - CONNECTING = 'connecting' - CONNECTED = 'connected' - DISCONNECTED = 'disconnected' - CLOSING = 'closing' - CLOSED = 'closed' - FAILED = 'failed' - SUSPENDED = 'suspended' - - -class ConnectionEvent(str, Enum): - INITIALIZED = 'initialized' - CONNECTING = 'connecting' - CONNECTED = 'connected' - DISCONNECTED = 'disconnected' - CLOSING = 'closing' - CLOSED = 'closed' - FAILED = 'failed' - SUSPENDED = 'suspended' - UPDATE = 'update' - - -@dataclass -class ConnectionStateChange: - previous: ConnectionState - current: ConnectionState - event: ConnectionEvent - reason: Optional[AblyException] = None # RTN4f - - class Connection(EventEmitter): # RTN4 """Ably Realtime Connection @@ -150,398 +109,3 @@ def connection_manager(self): @property def connection_details(self): return self.__connection_manager.connection_details - - -class ConnectionManager(EventEmitter): - def __init__(self, realtime, initial_state): - self.options = realtime.options - self.__ably = realtime - self.__state = initial_state - self.__ping_future = None - self.__timeout_in_secs = self.options.realtime_request_timeout / 1000 - self.transport: WebSocketTransport | None = None - self.__connection_details = None - self.connection_id = None - self.__fail_state = ConnectionState.DISCONNECTED - self.transition_timer: Timer | None = None - self.suspend_timer: Timer | None = None - self.retry_timer: Timer | None = None - self.connect_base_task: asyncio.Task | None = None - self.disconnect_transport_task: asyncio.Task | None = None - self.__fallback_hosts = self.options.get_fallback_realtime_hosts() - self.queued_messages = Queue() - super().__init__() - - def enact_state_change(self, state, reason=None): - current_state = self.__state - log.info(f'ConnectionManager.enact_state_change(): {current_state} -> {state}') - self.__state = state - self._emit('connectionstate', ConnectionStateChange(current_state, state, state, reason)) - - def check_connection(self): - try: - response = httpx.get(self.options.connectivity_check_url) - return 200 <= response.status_code < 300 and \ - (self.options.connectivity_check_url != Defaults.connectivity_check_url or "yes" in response.text) - except httpx.HTTPError: - return False - - def __get_transport_params(self): - protocol_version = Defaults.protocol_version - params = {"key": self.__ably.key, "v": protocol_version} - if self.connection_details: - params["resume"] = self.connection_details.connection_key - return params - - async def close_impl(self): - log.debug('ConnectionManager.close_impl()') - - self.cancel_suspend_timer() - self.start_transition_timer(ConnectionState.CLOSING, fail_state=ConnectionState.CLOSED) - if self.transport: - await self.transport.dispose() - if self.connect_base_task: - self.connect_base_task.cancel() - if self.disconnect_transport_task: - await self.disconnect_transport_task - self.cancel_retry_timer() - - self.notify_state(ConnectionState.CLOSED) - - async def send_protocol_message(self, protocol_message): - if self.state in ( - ConnectionState.DISCONNECTED, - ConnectionState.CONNECTING, - ): - self.queued_messages.put(protocol_message) - return - - if self.state == ConnectionState.CONNECTED: - if self.transport: - await self.transport.send(protocol_message) - else: - log.exception( - "ConnectionManager.send_protocol_message(): can not send message with no active transport" - ) - return - - raise AblyException(f"ConnectionManager.send_protocol_message(): called in {self.state}", 500, 50000) - - def send_queued_messages(self): - log.info(f'ConnectionManager.send_queued_messages(): sending {self.queued_messages.qsize()} message(s)') - while not self.queued_messages.empty(): - asyncio.create_task(self.send_protocol_message(self.queued_messages.get())) - - def fail_queued_messages(self, err): - log.info( - f"ConnectionManager.fail_queued_messages(): discarding {self.queued_messages.qsize()} messages;" + - f" reason = {err}" - ) - while not self.queued_messages.empty(): - msg = self.queued_messages.get() - log.exception(f"ConnectionManager.fail_queued_messages(): Failed to send protocol message: {msg}") - - async def ping(self): - if self.__ping_future: - try: - response = await self.__ping_future - except asyncio.CancelledError: - raise AblyException("Ping request cancelled due to request timeout", 504, 50003) - return response - - self.__ping_future = asyncio.Future() - if self.__state in [ConnectionState.CONNECTED, ConnectionState.CONNECTING]: - self.__ping_id = get_random_id() - ping_start_time = datetime.now().timestamp() - await self.send_protocol_message({"action": ProtocolMessageAction.HEARTBEAT, - "id": self.__ping_id}) - else: - raise AblyException("Cannot send ping request. Calling ping in invalid state", 40000, 400) - try: - await asyncio.wait_for(self.__ping_future, self.__timeout_in_secs) - except asyncio.TimeoutError: - raise AblyException("Timeout waiting for ping response", 504, 50003) - - ping_end_time = datetime.now().timestamp() - response_time_ms = (ping_end_time - ping_start_time) * 1000 - return round(response_time_ms, 2) - - def on_connected(self, connection_details: ConnectionDetails, connection_id: str, reason=None): - self.__fail_state = ConnectionState.DISCONNECTED - - self.__connection_details = connection_details - self.connection_id = connection_id - - if self.__state == ConnectionState.CONNECTED: - state_change = ConnectionStateChange(ConnectionState.CONNECTED, ConnectionState.CONNECTED, - ConnectionEvent.UPDATE) - self._emit(ConnectionEvent.UPDATE, state_change) - else: - self.notify_state(ConnectionState.CONNECTED, reason=reason) - - self.ably.channels._on_connected() - - def on_disconnected(self, msg: dict): - error = msg.get("error") - exception = AblyException(error.get('message'), error.get('statusCode'), error.get('code')) - self.notify_state(ConnectionState.DISCONNECTED, exception) - if error: - error_status_code = error.get("statusCode") - if error_status_code >= 500 or error_status_code <= 504: # RTN17f1 - if len(self.__fallback_hosts) > 0: - res = asyncio.create_task(self.connect_with_fallback_hosts(self.__fallback_hosts)) - if not res: - return - self.notify_state(self.__fail_state, reason=res) - else: - log.info("No fallback host to try for disconnected protocol message") - - async def on_error(self, msg: dict, exception: AblyException): - if msg.get('channel') is None: # RTN15i - self.enact_state_change(ConnectionState.FAILED, exception) - if self.transport: - await self.transport.dispose() - raise exception - - async def on_closed(self): - if self.transport: - await self.transport.dispose() - if self.connect_base_task: - self.connect_base_task.cancel() - - def on_channel_message(self, msg: dict): - self.__ably.channels._on_channel_message(msg) - - def on_heartbeat(self, id: Optional[str]): - if self.__ping_future: - # Resolve on heartbeat from ping request. - if self.__ping_id == id: - if not self.__ping_future.cancelled(): - self.__ping_future.set_result(None) - self.__ping_future = None - - def deactivate_transport(self, reason=None): - self.transport = None - self.enact_state_change(ConnectionState.DISCONNECTED, reason) - - def request_state(self, state: ConnectionState, force=False): - log.info(f'ConnectionManager.request_state(): state = {state}') - - if not force and state == self.state: - return - - if state == ConnectionState.CONNECTING and self.__state == ConnectionState.CONNECTED: - return - - if state == ConnectionState.CLOSING and self.__state == ConnectionState.CLOSED: - return - - if not force: - self.enact_state_change(state) - - if state == ConnectionState.CONNECTING: - self.start_connect() - - if state == ConnectionState.CLOSING: - asyncio.create_task(self.close_impl()) - - def start_connect(self): - self.start_suspend_timer() - self.start_transition_timer(ConnectionState.CONNECTING) - self.connect_base_task = asyncio.create_task(self.connect_base()) - - async def connect_with_fallback_hosts(self, fallback_hosts: list): - for host in fallback_hosts: - try: - if self.check_connection(): - await self.try_host(host) - return - else: - message = "Unable to connect, network unreachable" - log.exception(message) - exception = AblyException(message, status_code=404, code=80003) - self.notify_state(self.__fail_state, exception) - return - except Exception as exc: - exception = exc - log.exception(f'Connection to {host} failed, reason={exception}') - log.exception("No more fallback hosts to try") - return exception - - async def connect_base(self): - fallback_hosts = self.__fallback_hosts - primary_host = self.options.get_realtime_host() - try: - await self.try_host(primary_host) - return - except Exception as exception: - log.exception(f'Connection to {primary_host} failed, reason={exception}') - if len(fallback_hosts) > 0: - log.info("Attempting connection to fallback host(s)") - resp = await self.connect_with_fallback_hosts(fallback_hosts) - if not resp: - return - exception = resp - self.notify_state(self.__fail_state, reason=exception) - - async def try_host(self, host): - params = self.__get_transport_params() - self.transport = WebSocketTransport(self, host, params) - self._emit('transport.pending', self.transport) - self.transport.connect() - - future = asyncio.Future() - - def on_transport_connected(): - log.info('ConnectionManager.try_a_host(): transport connected') - if self.transport: - self.transport.off('failed', on_transport_failed) - future.set_result(None) - - async def on_transport_failed(exception): - log.info('ConnectionManager.try_a_host(): transport failed') - if self.transport: - self.transport.off('connected', on_transport_connected) - await self.transport.dispose() - future.set_exception(exception) - - self.transport.once('connected', on_transport_connected) - self.transport.once('failed', on_transport_failed) - # Fix asyncio CancelledError in python 3.7 - try: - await future - except asyncio.CancelledError: - return - - def notify_state(self, state: ConnectionState, reason=None, retry_immediately=None): - # RTN15a - retry_immediately = (retry_immediately is not False) and ( - state == ConnectionState.DISCONNECTED and self.__state == ConnectionState.CONNECTED) - - log.info( - f'ConnectionManager.notify_state(): new state: {state}' - + ('; will retry immediately' if retry_immediately else '') - ) - - if state == self.__state: - return - - self.cancel_transition_timer() - self.check_suspend_timer(state) - - if retry_immediately: - self.options.loop.call_soon(self.request_state, ConnectionState.CONNECTING) - elif state == ConnectionState.DISCONNECTED: - self.start_retry_timer(self.options.disconnected_retry_timeout) - elif state == ConnectionState.SUSPENDED: - self.start_retry_timer(self.options.suspended_retry_timeout) - - if (state == ConnectionState.DISCONNECTED and not retry_immediately) or state == ConnectionState.SUSPENDED: - self.disconnect_transport() - - self.enact_state_change(state, reason) - - if state == ConnectionState.CONNECTED: - self.send_queued_messages() - elif state in ( - ConnectionState.CLOSING, - ConnectionState.CLOSED, - ConnectionState.SUSPENDED, - ConnectionState.FAILED, - ): - self.fail_queued_messages(reason) - self.ably.channels._propagate_connection_interruption(state, reason) - - def start_transition_timer(self, state: ConnectionState, fail_state=None): - log.debug(f'ConnectionManager.start_transition_timer(): transition state = {state}') - - if self.transition_timer: - log.debug('ConnectionManager.start_transition_timer(): clearing already-running timer') - self.transition_timer.cancel() - - if fail_state is None: - fail_state = self.__fail_state if state != ConnectionState.CLOSING else ConnectionState.CLOSED - - timeout = self.options.realtime_request_timeout - - def on_transition_timer_expire(): - if self.transition_timer: - self.transition_timer = None - log.info(f'ConnectionManager {state} timer expired, notifying new state: {fail_state}') - self.notify_state( - fail_state, - AblyException("Connection cancelled due to request timeout", 504, 50003) - ) - - log.debug(f'ConnectionManager.start_transition_timer(): setting timer for {timeout}ms') - - self.transition_timer = Timer(timeout, on_transition_timer_expire) - - def cancel_transition_timer(self): - log.debug('ConnectionManager.cancel_transition_timer()') - if self.transition_timer: - self.transition_timer.cancel() - self.transition_timer = None - - def start_suspend_timer(self): - log.debug('ConnectionManager.start_suspend_timer()') - if self.suspend_timer: - return - - def on_suspend_timer_expire(): - if self.suspend_timer: - self.suspend_timer = None - log.info('ConnectionManager suspend timer expired, requesting new state: suspended') - self.notify_state( - ConnectionState.SUSPENDED, - AblyException("Connection to server unavailable", 400, 80002) - ) - self.__fail_state = ConnectionState.SUSPENDED - self.__connection_details = None - - self.suspend_timer = Timer(Defaults.connection_state_ttl, on_suspend_timer_expire) - - def check_suspend_timer(self, state: ConnectionState): - if state not in ( - ConnectionState.CONNECTING, - ConnectionState.DISCONNECTED, - ConnectionState.SUSPENDED, - ): - self.cancel_suspend_timer() - - def cancel_suspend_timer(self): - log.debug('ConnectionManager.cancel_suspend_timer()') - self.__fail_state = ConnectionState.DISCONNECTED - if self.suspend_timer: - self.suspend_timer.cancel() - self.suspend_timer = None - - def start_retry_timer(self, interval: int): - def on_retry_timeout(): - log.info('ConnectionManager retry timer expired, retrying') - self.retry_timer = None - self.request_state(ConnectionState.CONNECTING) - - self.retry_timer = Timer(interval, on_retry_timeout) - - def cancel_retry_timer(self): - if self.retry_timer: - self.retry_timer.cancel() - self.retry_timer = None - - def disconnect_transport(self): - log.info('ConnectionManager.disconnect_transport()') - if self.transport: - self.disconnect_transport_task = asyncio.create_task(self.transport.dispose()) - - @property - def ably(self): - return self.__ably - - @property - def state(self): - return self.__state - - @property - def connection_details(self): - return self.__connection_details diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py new file mode 100644 index 00000000..6672d6a0 --- /dev/null +++ b/ably/realtime/connectionmanager.py @@ -0,0 +1,410 @@ +import logging +import asyncio +import httpx +from ably.realtime.websockettransport import WebSocketTransport, ProtocolMessageAction +from ably.transport.defaults import Defaults +from ably.types.connectionstate import ConnectionEvent, ConnectionState, ConnectionStateChange +from ably.util.exceptions import AblyException +from ably.util.eventemitter import EventEmitter +from datetime import datetime +from ably.util.helper import get_random_id, Timer +from typing import Optional +from ably.types.connectiondetails import ConnectionDetails +from queue import Queue + +log = logging.getLogger(__name__) + + +class ConnectionManager(EventEmitter): + def __init__(self, realtime, initial_state): + self.options = realtime.options + self.__ably = realtime + self.__state = initial_state + self.__ping_future = None + self.__timeout_in_secs = self.options.realtime_request_timeout / 1000 + self.transport: WebSocketTransport | None = None + self.__connection_details = None + self.connection_id = None + self.__fail_state = ConnectionState.DISCONNECTED + self.transition_timer: Timer | None = None + self.suspend_timer: Timer | None = None + self.retry_timer: Timer | None = None + self.connect_base_task: asyncio.Task | None = None + self.disconnect_transport_task: asyncio.Task | None = None + self.__fallback_hosts = self.options.get_fallback_realtime_hosts() + self.queued_messages = Queue() + super().__init__() + + def enact_state_change(self, state, reason=None): + current_state = self.__state + log.info(f'ConnectionManager.enact_state_change(): {current_state} -> {state}') + self.__state = state + self._emit('connectionstate', ConnectionStateChange(current_state, state, state, reason)) + + def check_connection(self): + try: + response = httpx.get(self.options.connectivity_check_url) + return 200 <= response.status_code < 300 and \ + (self.options.connectivity_check_url != Defaults.connectivity_check_url or "yes" in response.text) + except httpx.HTTPError: + return False + + def __get_transport_params(self): + protocol_version = Defaults.protocol_version + params = {"key": self.__ably.key, "v": protocol_version} + if self.connection_details: + params["resume"] = self.connection_details.connection_key + return params + + async def close_impl(self): + log.debug('ConnectionManager.close_impl()') + + self.cancel_suspend_timer() + self.start_transition_timer(ConnectionState.CLOSING, fail_state=ConnectionState.CLOSED) + if self.transport: + await self.transport.dispose() + if self.connect_base_task: + self.connect_base_task.cancel() + if self.disconnect_transport_task: + await self.disconnect_transport_task + self.cancel_retry_timer() + + self.notify_state(ConnectionState.CLOSED) + + async def send_protocol_message(self, protocol_message): + if self.state in ( + ConnectionState.DISCONNECTED, + ConnectionState.CONNECTING, + ): + self.queued_messages.put(protocol_message) + return + + if self.state == ConnectionState.CONNECTED: + if self.transport: + await self.transport.send(protocol_message) + else: + log.exception( + "ConnectionManager.send_protocol_message(): can not send message with no active transport" + ) + return + + raise AblyException(f"ConnectionManager.send_protocol_message(): called in {self.state}", 500, 50000) + + def send_queued_messages(self): + log.info(f'ConnectionManager.send_queued_messages(): sending {self.queued_messages.qsize()} message(s)') + while not self.queued_messages.empty(): + asyncio.create_task(self.send_protocol_message(self.queued_messages.get())) + + def fail_queued_messages(self, err): + log.info( + f"ConnectionManager.fail_queued_messages(): discarding {self.queued_messages.qsize()} messages;" + + f" reason = {err}" + ) + while not self.queued_messages.empty(): + msg = self.queued_messages.get() + log.exception(f"ConnectionManager.fail_queued_messages(): Failed to send protocol message: {msg}") + + async def ping(self): + if self.__ping_future: + try: + response = await self.__ping_future + except asyncio.CancelledError: + raise AblyException("Ping request cancelled due to request timeout", 504, 50003) + return response + + self.__ping_future = asyncio.Future() + if self.__state in [ConnectionState.CONNECTED, ConnectionState.CONNECTING]: + self.__ping_id = get_random_id() + ping_start_time = datetime.now().timestamp() + await self.send_protocol_message({"action": ProtocolMessageAction.HEARTBEAT, + "id": self.__ping_id}) + else: + raise AblyException("Cannot send ping request. Calling ping in invalid state", 40000, 400) + try: + await asyncio.wait_for(self.__ping_future, self.__timeout_in_secs) + except asyncio.TimeoutError: + raise AblyException("Timeout waiting for ping response", 504, 50003) + + ping_end_time = datetime.now().timestamp() + response_time_ms = (ping_end_time - ping_start_time) * 1000 + return round(response_time_ms, 2) + + def on_connected(self, connection_details: ConnectionDetails, connection_id: str, reason=None): + self.__fail_state = ConnectionState.DISCONNECTED + + self.__connection_details = connection_details + self.connection_id = connection_id + + if self.__state == ConnectionState.CONNECTED: + state_change = ConnectionStateChange(ConnectionState.CONNECTED, ConnectionState.CONNECTED, + ConnectionEvent.UPDATE) + self._emit(ConnectionEvent.UPDATE, state_change) + else: + self.notify_state(ConnectionState.CONNECTED, reason=reason) + + self.ably.channels._on_connected() + + def on_disconnected(self, msg: dict): + error = msg.get("error") + exception = AblyException(error.get('message'), error.get('statusCode'), error.get('code')) + self.notify_state(ConnectionState.DISCONNECTED, exception) + if error: + error_status_code = error.get("statusCode") + if error_status_code >= 500 or error_status_code <= 504: # RTN17f1 + if len(self.__fallback_hosts) > 0: + res = asyncio.create_task(self.connect_with_fallback_hosts(self.__fallback_hosts)) + if not res: + return + self.notify_state(self.__fail_state, reason=res) + else: + log.info("No fallback host to try for disconnected protocol message") + + async def on_error(self, msg: dict, exception: AblyException): + if msg.get('channel') is None: # RTN15i + self.enact_state_change(ConnectionState.FAILED, exception) + if self.transport: + await self.transport.dispose() + raise exception + + async def on_closed(self): + if self.transport: + await self.transport.dispose() + if self.connect_base_task: + self.connect_base_task.cancel() + + def on_channel_message(self, msg: dict): + self.__ably.channels._on_channel_message(msg) + + def on_heartbeat(self, id: Optional[str]): + if self.__ping_future: + # Resolve on heartbeat from ping request. + if self.__ping_id == id: + if not self.__ping_future.cancelled(): + self.__ping_future.set_result(None) + self.__ping_future = None + + def deactivate_transport(self, reason=None): + self.transport = None + self.enact_state_change(ConnectionState.DISCONNECTED, reason) + + def request_state(self, state: ConnectionState, force=False): + log.info(f'ConnectionManager.request_state(): state = {state}') + + if not force and state == self.state: + return + + if state == ConnectionState.CONNECTING and self.__state == ConnectionState.CONNECTED: + return + + if state == ConnectionState.CLOSING and self.__state == ConnectionState.CLOSED: + return + + if not force: + self.enact_state_change(state) + + if state == ConnectionState.CONNECTING: + self.start_connect() + + if state == ConnectionState.CLOSING: + asyncio.create_task(self.close_impl()) + + def start_connect(self): + self.start_suspend_timer() + self.start_transition_timer(ConnectionState.CONNECTING) + self.connect_base_task = asyncio.create_task(self.connect_base()) + + async def connect_with_fallback_hosts(self, fallback_hosts: list): + for host in fallback_hosts: + try: + if self.check_connection(): + await self.try_host(host) + return + else: + message = "Unable to connect, network unreachable" + log.exception(message) + exception = AblyException(message, status_code=404, code=80003) + self.notify_state(self.__fail_state, exception) + return + except Exception as exc: + exception = exc + log.exception(f'Connection to {host} failed, reason={exception}') + log.exception("No more fallback hosts to try") + return exception + + async def connect_base(self): + fallback_hosts = self.__fallback_hosts + primary_host = self.options.get_realtime_host() + try: + await self.try_host(primary_host) + return + except Exception as exception: + log.exception(f'Connection to {primary_host} failed, reason={exception}') + if len(fallback_hosts) > 0: + log.info("Attempting connection to fallback host(s)") + resp = await self.connect_with_fallback_hosts(fallback_hosts) + if not resp: + return + exception = resp + self.notify_state(self.__fail_state, reason=exception) + + async def try_host(self, host): + params = self.__get_transport_params() + self.transport = WebSocketTransport(self, host, params) + self._emit('transport.pending', self.transport) + self.transport.connect() + + future = asyncio.Future() + + def on_transport_connected(): + log.info('ConnectionManager.try_a_host(): transport connected') + if self.transport: + self.transport.off('failed', on_transport_failed) + future.set_result(None) + + async def on_transport_failed(exception): + log.info('ConnectionManager.try_a_host(): transport failed') + if self.transport: + self.transport.off('connected', on_transport_connected) + await self.transport.dispose() + future.set_exception(exception) + + self.transport.once('connected', on_transport_connected) + self.transport.once('failed', on_transport_failed) + # Fix asyncio CancelledError in python 3.7 + try: + await future + except asyncio.CancelledError: + return + + def notify_state(self, state: ConnectionState, reason=None, retry_immediately=None): + # RTN15a + retry_immediately = (retry_immediately is not False) and ( + state == ConnectionState.DISCONNECTED and self.__state == ConnectionState.CONNECTED) + + log.info( + f'ConnectionManager.notify_state(): new state: {state}' + + ('; will retry immediately' if retry_immediately else '') + ) + + if state == self.__state: + return + + self.cancel_transition_timer() + self.check_suspend_timer(state) + + if retry_immediately: + self.options.loop.call_soon(self.request_state, ConnectionState.CONNECTING) + elif state == ConnectionState.DISCONNECTED: + self.start_retry_timer(self.options.disconnected_retry_timeout) + elif state == ConnectionState.SUSPENDED: + self.start_retry_timer(self.options.suspended_retry_timeout) + + if (state == ConnectionState.DISCONNECTED and not retry_immediately) or state == ConnectionState.SUSPENDED: + self.disconnect_transport() + + self.enact_state_change(state, reason) + + if state == ConnectionState.CONNECTED: + self.send_queued_messages() + elif state in ( + ConnectionState.CLOSING, + ConnectionState.CLOSED, + ConnectionState.SUSPENDED, + ConnectionState.FAILED, + ): + self.fail_queued_messages(reason) + self.ably.channels._propagate_connection_interruption(state, reason) + + def start_transition_timer(self, state: ConnectionState, fail_state=None): + log.debug(f'ConnectionManager.start_transition_timer(): transition state = {state}') + + if self.transition_timer: + log.debug('ConnectionManager.start_transition_timer(): clearing already-running timer') + self.transition_timer.cancel() + + if fail_state is None: + fail_state = self.__fail_state if state != ConnectionState.CLOSING else ConnectionState.CLOSED + + timeout = self.options.realtime_request_timeout + + def on_transition_timer_expire(): + if self.transition_timer: + self.transition_timer = None + log.info(f'ConnectionManager {state} timer expired, notifying new state: {fail_state}') + self.notify_state( + fail_state, + AblyException("Connection cancelled due to request timeout", 504, 50003) + ) + + log.debug(f'ConnectionManager.start_transition_timer(): setting timer for {timeout}ms') + + self.transition_timer = Timer(timeout, on_transition_timer_expire) + + def cancel_transition_timer(self): + log.debug('ConnectionManager.cancel_transition_timer()') + if self.transition_timer: + self.transition_timer.cancel() + self.transition_timer = None + + def start_suspend_timer(self): + log.debug('ConnectionManager.start_suspend_timer()') + if self.suspend_timer: + return + + def on_suspend_timer_expire(): + if self.suspend_timer: + self.suspend_timer = None + log.info('ConnectionManager suspend timer expired, requesting new state: suspended') + self.notify_state( + ConnectionState.SUSPENDED, + AblyException("Connection to server unavailable", 400, 80002) + ) + self.__fail_state = ConnectionState.SUSPENDED + self.__connection_details = None + + self.suspend_timer = Timer(Defaults.connection_state_ttl, on_suspend_timer_expire) + + def check_suspend_timer(self, state: ConnectionState): + if state not in ( + ConnectionState.CONNECTING, + ConnectionState.DISCONNECTED, + ConnectionState.SUSPENDED, + ): + self.cancel_suspend_timer() + + def cancel_suspend_timer(self): + log.debug('ConnectionManager.cancel_suspend_timer()') + self.__fail_state = ConnectionState.DISCONNECTED + if self.suspend_timer: + self.suspend_timer.cancel() + self.suspend_timer = None + + def start_retry_timer(self, interval: int): + def on_retry_timeout(): + log.info('ConnectionManager retry timer expired, retrying') + self.retry_timer = None + self.request_state(ConnectionState.CONNECTING) + + self.retry_timer = Timer(interval, on_retry_timeout) + + def cancel_retry_timer(self): + if self.retry_timer: + self.retry_timer.cancel() + self.retry_timer = None + + def disconnect_transport(self): + log.info('ConnectionManager.disconnect_transport()') + if self.transport: + self.disconnect_transport_task = asyncio.create_task(self.transport.dispose()) + + @property + def ably(self): + return self.__ably + + @property + def state(self): + return self.__state + + @property + def connection_details(self): + return self.__connection_details diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 9a162b2b..122ed956 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -3,7 +3,8 @@ import logging from typing import Optional -from ably.realtime.connection import ConnectionState, ProtocolMessageAction +from ably.realtime.connection import ConnectionState +from ably.realtime.websockettransport import ProtocolMessageAction from ably.rest.channel import Channel from ably.types.message import Message from ably.util.eventemitter import EventEmitter diff --git a/ably/types/connectionstate.py b/ably/types/connectionstate.py new file mode 100644 index 00000000..3a7fb111 --- /dev/null +++ b/ably/types/connectionstate.py @@ -0,0 +1,36 @@ +from enum import Enum +from dataclasses import dataclass +from typing import Optional + +from ably.util.exceptions import AblyException + + +class ConnectionState(str, Enum): + INITIALIZED = 'initialized' + CONNECTING = 'connecting' + CONNECTED = 'connected' + DISCONNECTED = 'disconnected' + CLOSING = 'closing' + CLOSED = 'closed' + FAILED = 'failed' + SUSPENDED = 'suspended' + + +class ConnectionEvent(str, Enum): + INITIALIZED = 'initialized' + CONNECTING = 'connecting' + CONNECTED = 'connected' + DISCONNECTED = 'disconnected' + CLOSING = 'closing' + CLOSED = 'closed' + FAILED = 'failed' + SUSPENDED = 'suspended' + UPDATE = 'update' + + +@dataclass +class ConnectionStateChange: + previous: ConnectionState + current: ConnectionState + event: ConnectionEvent + reason: Optional[AblyException] = None # RTN4f diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py index fa737d09..05ff005a 100644 --- a/test/ably/realtimechannel_test.py +++ b/test/ably/realtimechannel_test.py @@ -1,10 +1,11 @@ import asyncio import pytest from ably.realtime.realtime_channel import ChannelState +from ably.realtime.websockettransport import ProtocolMessageAction from ably.types.message import Message from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase, random_string -from ably.realtime.connection import ConnectionState, ProtocolMessageAction +from ably.realtime.connection import ConnectionState from ably.util.exceptions import AblyException diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index 8f13f319..c45b14f8 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -1,6 +1,7 @@ import asyncio -from ably.realtime.connection import ConnectionEvent, ConnectionState, ProtocolMessageAction +from ably.realtime.connection import ConnectionEvent, ConnectionState import pytest +from ably.realtime.websockettransport import ProtocolMessageAction from ably.util.exceptions import AblyException from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase From e53f1a0b91d8a225afde960a2397e1688589c307 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 1 Feb 2023 12:26:20 +0000 Subject: [PATCH 362/481] refactor: move WebSocketTransport into transport dir --- ably/realtime/connectionmanager.py | 2 +- ably/realtime/realtime_channel.py | 2 +- ably/{realtime => transport}/websockettransport.py | 0 test/ably/realtimechannel_test.py | 2 +- test/ably/realtimeconnection_test.py | 2 +- test/ably/realtimeresume_test.py | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) rename ably/{realtime => transport}/websockettransport.py (100%) diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index 6672d6a0..3a1a9e15 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -1,7 +1,7 @@ import logging import asyncio import httpx -from ably.realtime.websockettransport import WebSocketTransport, ProtocolMessageAction +from ably.transport.websockettransport import WebSocketTransport, ProtocolMessageAction from ably.transport.defaults import Defaults from ably.types.connectionstate import ConnectionEvent, ConnectionState, ConnectionStateChange from ably.util.exceptions import AblyException diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 122ed956..1036932d 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -4,7 +4,7 @@ from typing import Optional from ably.realtime.connection import ConnectionState -from ably.realtime.websockettransport import ProtocolMessageAction +from ably.transport.websockettransport import ProtocolMessageAction from ably.rest.channel import Channel from ably.types.message import Message from ably.util.eventemitter import EventEmitter diff --git a/ably/realtime/websockettransport.py b/ably/transport/websockettransport.py similarity index 100% rename from ably/realtime/websockettransport.py rename to ably/transport/websockettransport.py diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtimechannel_test.py index 05ff005a..4bc77044 100644 --- a/test/ably/realtimechannel_test.py +++ b/test/ably/realtimechannel_test.py @@ -1,7 +1,7 @@ import asyncio import pytest from ably.realtime.realtime_channel import ChannelState -from ably.realtime.websockettransport import ProtocolMessageAction +from ably.transport.websockettransport import ProtocolMessageAction from ably.types.message import Message from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase, random_string diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtimeconnection_test.py index c45b14f8..8034f84d 100644 --- a/test/ably/realtimeconnection_test.py +++ b/test/ably/realtimeconnection_test.py @@ -1,7 +1,7 @@ import asyncio from ably.realtime.connection import ConnectionEvent, ConnectionState import pytest -from ably.realtime.websockettransport import ProtocolMessageAction +from ably.transport.websockettransport import ProtocolMessageAction from ably.util.exceptions import AblyException from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase diff --git a/test/ably/realtimeresume_test.py b/test/ably/realtimeresume_test.py index da3e9e42..81eef739 100644 --- a/test/ably/realtimeresume_test.py +++ b/test/ably/realtimeresume_test.py @@ -1,7 +1,7 @@ import asyncio from ably.realtime.connection import ConnectionState from ably.realtime.realtime_channel import ChannelState -from ably.realtime.websockettransport import ProtocolMessageAction +from ably.transport.websockettransport import ProtocolMessageAction from test.ably.restsetup import RestSetup from test.ably.utils import BaseAsyncTestCase, random_string From 998a1d5585d38cb54a0a5b2bb3d985a1bc6fa7c9 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 1 Feb 2023 12:29:08 +0000 Subject: [PATCH 363/481] refactor: move ChannelState into its own module --- ably/realtime/realtime_channel.py | 23 ++--------------------- ably/types/channelstate.py | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 21 deletions(-) create mode 100644 ably/types/channelstate.py diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 1036932d..4a56191f 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -1,11 +1,10 @@ import asyncio -from dataclasses import dataclass import logging -from typing import Optional from ably.realtime.connection import ConnectionState from ably.transport.websockettransport import ProtocolMessageAction from ably.rest.channel import Channel +from ably.types.channelstate import ChannelState, ChannelStateChange from ably.types.message import Message from ably.util.eventemitter import EventEmitter from ably.util.exceptions import AblyException @@ -16,16 +15,6 @@ log = logging.getLogger(__name__) -class ChannelState(str, Enum): - INITIALIZED = 'initialized' - ATTACHING = 'attaching' - ATTACHED = 'attached' - DETACHING = 'detaching' - DETACHED = 'detached' - SUSPENDED = 'suspended' - FAILED = 'failed' - - class Flag(int, Enum): # Channel attach state flags HAS_PRESENCE = 1 << 0 @@ -44,14 +33,6 @@ def has_flag(message_flags: int, flag: Flag): return message_flags & flag > 0 -@dataclass -class ChannelStateChange: - previous: ChannelState - current: ChannelState - resumed: bool - reason: Optional[AblyException] = None - - class RealtimeChannel(EventEmitter, Channel): """ Ably Realtime Channel @@ -328,7 +309,7 @@ def _on_message(self, msg): flags = msg.get('flags') error = msg.get("error") exception = None - resumed = None + resumed = False if error: exception = AblyException(error.get('message'), error.get('statusCode'), error.get('code')) diff --git a/ably/types/channelstate.py b/ably/types/channelstate.py new file mode 100644 index 00000000..914b5956 --- /dev/null +++ b/ably/types/channelstate.py @@ -0,0 +1,22 @@ +from dataclasses import dataclass +from typing import Optional +from enum import Enum +from ably.util.exceptions import AblyException + + +class ChannelState(str, Enum): + INITIALIZED = 'initialized' + ATTACHING = 'attaching' + ATTACHED = 'attached' + DETACHING = 'detaching' + DETACHED = 'detached' + SUSPENDED = 'suspended' + FAILED = 'failed' + + +@dataclass +class ChannelStateChange: + previous: ChannelState + current: ChannelState + resumed: bool + reason: Optional[AblyException] = None From 4cef95626290f67ec75d4972a73a4c865316915f Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 1 Feb 2023 12:32:21 +0000 Subject: [PATCH 364/481] refactor: move Flag enum to its own module --- ably/realtime/realtime_channel.py | 20 +------------------- ably/types/flags.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 19 deletions(-) create mode 100644 ably/types/flags.py diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 4a56191f..1336bb35 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -5,34 +5,16 @@ from ably.transport.websockettransport import ProtocolMessageAction from ably.rest.channel import Channel from ably.types.channelstate import ChannelState, ChannelStateChange +from ably.types.flags import Flag, has_flag from ably.types.message import Message from ably.util.eventemitter import EventEmitter from ably.util.exceptions import AblyException -from enum import Enum from ably.util.helper import Timer, is_callable_or_coroutine log = logging.getLogger(__name__) -class Flag(int, Enum): - # Channel attach state flags - HAS_PRESENCE = 1 << 0 - HAS_BACKLOG = 1 << 1 - RESUMED = 1 << 2 - TRANSIENT = 1 << 4 - ATTACH_RESUME = 1 << 5 - # Channel mode flags - PRESENCE = 1 << 16 - PUBLISH = 1 << 17 - SUBSCRIBE = 1 << 18 - PRESENCE_SUBSCRIBE = 1 << 19 - - -def has_flag(message_flags: int, flag: Flag): - return message_flags & flag > 0 - - class RealtimeChannel(EventEmitter, Channel): """ Ably Realtime Channel diff --git a/ably/types/flags.py b/ably/types/flags.py new file mode 100644 index 00000000..1666434c --- /dev/null +++ b/ably/types/flags.py @@ -0,0 +1,19 @@ +from enum import Enum + + +class Flag(int, Enum): + # Channel attach state flags + HAS_PRESENCE = 1 << 0 + HAS_BACKLOG = 1 << 1 + RESUMED = 1 << 2 + TRANSIENT = 1 << 4 + ATTACH_RESUME = 1 << 5 + # Channel mode flags + PRESENCE = 1 << 16 + PUBLISH = 1 << 17 + SUBSCRIBE = 1 << 18 + PRESENCE_SUBSCRIBE = 1 << 19 + + +def has_flag(message_flags: int, flag: Flag): + return message_flags & flag > 0 From 4e3d9b296c5601e49bfa2f017f4c72ad26834650 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 1 Feb 2023 12:39:01 +0000 Subject: [PATCH 365/481] test: move rest/realtime tests into separate dirs --- test/ably/{ => realtime}/eventemitter_test.py | 0 test/ably/{ => realtime}/realtimechannel_test.py | 0 test/ably/{ => realtime}/realtimeconnection_test.py | 0 test/ably/{ => realtime}/realtimeinit_test.py | 0 test/ably/{ => realtime}/realtimeresume_test.py | 0 test/ably/{ => rest}/encoders_test.py | 0 test/ably/{ => rest}/restauth_test.py | 0 test/ably/{ => rest}/restcapability_test.py | 0 test/ably/{ => rest}/restchannelhistory_test.py | 0 test/ably/{ => rest}/restchannelpublish_test.py | 2 +- test/ably/{ => rest}/restchannels_test.py | 0 test/ably/{ => rest}/restchannelstatus_test.py | 0 test/ably/{ => rest}/restcrypto_test.py | 3 ++- test/ably/{ => rest}/resthttp_test.py | 0 test/ably/{ => rest}/restinit_test.py | 0 test/ably/{ => rest}/restpaginatedresult_test.py | 0 test/ably/{ => rest}/restpresence_test.py | 0 test/ably/{ => rest}/restpush_test.py | 0 test/ably/{ => rest}/restrequest_test.py | 0 test/ably/{ => rest}/reststats_test.py | 0 test/ably/{ => rest}/resttime_test.py | 0 test/ably/{ => rest}/resttoken_test.py | 0 22 files changed, 3 insertions(+), 2 deletions(-) rename test/ably/{ => realtime}/eventemitter_test.py (100%) rename test/ably/{ => realtime}/realtimechannel_test.py (100%) rename test/ably/{ => realtime}/realtimeconnection_test.py (100%) rename test/ably/{ => realtime}/realtimeinit_test.py (100%) rename test/ably/{ => realtime}/realtimeresume_test.py (100%) rename test/ably/{ => rest}/encoders_test.py (100%) rename test/ably/{ => rest}/restauth_test.py (100%) rename test/ably/{ => rest}/restcapability_test.py (100%) rename test/ably/{ => rest}/restchannelhistory_test.py (100%) rename test/ably/{ => rest}/restchannelpublish_test.py (99%) rename test/ably/{ => rest}/restchannels_test.py (100%) rename test/ably/{ => rest}/restchannelstatus_test.py (100%) rename test/ably/{ => rest}/restcrypto_test.py (98%) rename test/ably/{ => rest}/resthttp_test.py (100%) rename test/ably/{ => rest}/restinit_test.py (100%) rename test/ably/{ => rest}/restpaginatedresult_test.py (100%) rename test/ably/{ => rest}/restpresence_test.py (100%) rename test/ably/{ => rest}/restpush_test.py (100%) rename test/ably/{ => rest}/restrequest_test.py (100%) rename test/ably/{ => rest}/reststats_test.py (100%) rename test/ably/{ => rest}/resttime_test.py (100%) rename test/ably/{ => rest}/resttoken_test.py (100%) diff --git a/test/ably/eventemitter_test.py b/test/ably/realtime/eventemitter_test.py similarity index 100% rename from test/ably/eventemitter_test.py rename to test/ably/realtime/eventemitter_test.py diff --git a/test/ably/realtimechannel_test.py b/test/ably/realtime/realtimechannel_test.py similarity index 100% rename from test/ably/realtimechannel_test.py rename to test/ably/realtime/realtimechannel_test.py diff --git a/test/ably/realtimeconnection_test.py b/test/ably/realtime/realtimeconnection_test.py similarity index 100% rename from test/ably/realtimeconnection_test.py rename to test/ably/realtime/realtimeconnection_test.py diff --git a/test/ably/realtimeinit_test.py b/test/ably/realtime/realtimeinit_test.py similarity index 100% rename from test/ably/realtimeinit_test.py rename to test/ably/realtime/realtimeinit_test.py diff --git a/test/ably/realtimeresume_test.py b/test/ably/realtime/realtimeresume_test.py similarity index 100% rename from test/ably/realtimeresume_test.py rename to test/ably/realtime/realtimeresume_test.py diff --git a/test/ably/encoders_test.py b/test/ably/rest/encoders_test.py similarity index 100% rename from test/ably/encoders_test.py rename to test/ably/rest/encoders_test.py diff --git a/test/ably/restauth_test.py b/test/ably/rest/restauth_test.py similarity index 100% rename from test/ably/restauth_test.py rename to test/ably/rest/restauth_test.py diff --git a/test/ably/restcapability_test.py b/test/ably/rest/restcapability_test.py similarity index 100% rename from test/ably/restcapability_test.py rename to test/ably/rest/restcapability_test.py diff --git a/test/ably/restchannelhistory_test.py b/test/ably/rest/restchannelhistory_test.py similarity index 100% rename from test/ably/restchannelhistory_test.py rename to test/ably/rest/restchannelhistory_test.py diff --git a/test/ably/restchannelpublish_test.py b/test/ably/rest/restchannelpublish_test.py similarity index 99% rename from test/ably/restchannelpublish_test.py rename to test/ably/rest/restchannelpublish_test.py index a9a31649..ed571185 100644 --- a/test/ably/restchannelpublish_test.py +++ b/test/ably/rest/restchannelpublish_test.py @@ -383,7 +383,7 @@ async def test_interoperability(self): 'binary': bytearray, } - root_dir = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + root_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) path = os.path.join(root_dir, 'submodules', 'test-resources', 'messages-encoding.json') with open(path) as f: data = json.load(f) diff --git a/test/ably/restchannels_test.py b/test/ably/rest/restchannels_test.py similarity index 100% rename from test/ably/restchannels_test.py rename to test/ably/rest/restchannels_test.py diff --git a/test/ably/restchannelstatus_test.py b/test/ably/rest/restchannelstatus_test.py similarity index 100% rename from test/ably/restchannelstatus_test.py rename to test/ably/rest/restchannelstatus_test.py diff --git a/test/ably/restcrypto_test.py b/test/ably/rest/restcrypto_test.py similarity index 98% rename from test/ably/restcrypto_test.py rename to test/ably/rest/restcrypto_test.py index 518d19a9..3fa4918d 100644 --- a/test/ably/restcrypto_test.py +++ b/test/ably/rest/restcrypto_test.py @@ -204,7 +204,8 @@ class AbstractTestCryptoWithFixture: @classmethod def setUpClass(cls): - with open(os.path.dirname(__file__) + '/../../submodules/test-resources/%s' % cls.fixture_file, 'r') as f: + resources_path = os.path.dirname(__file__) + '/../../../submodules/test-resources/%s' % cls.fixture_file + with open(resources_path, 'r') as f: cls.fixture = json.loads(f.read()) cls.params = { 'secret_key': base64.b64decode(cls.fixture['key'].encode('ascii')), diff --git a/test/ably/resthttp_test.py b/test/ably/rest/resthttp_test.py similarity index 100% rename from test/ably/resthttp_test.py rename to test/ably/rest/resthttp_test.py diff --git a/test/ably/restinit_test.py b/test/ably/rest/restinit_test.py similarity index 100% rename from test/ably/restinit_test.py rename to test/ably/rest/restinit_test.py diff --git a/test/ably/restpaginatedresult_test.py b/test/ably/rest/restpaginatedresult_test.py similarity index 100% rename from test/ably/restpaginatedresult_test.py rename to test/ably/rest/restpaginatedresult_test.py diff --git a/test/ably/restpresence_test.py b/test/ably/rest/restpresence_test.py similarity index 100% rename from test/ably/restpresence_test.py rename to test/ably/rest/restpresence_test.py diff --git a/test/ably/restpush_test.py b/test/ably/rest/restpush_test.py similarity index 100% rename from test/ably/restpush_test.py rename to test/ably/rest/restpush_test.py diff --git a/test/ably/restrequest_test.py b/test/ably/rest/restrequest_test.py similarity index 100% rename from test/ably/restrequest_test.py rename to test/ably/rest/restrequest_test.py diff --git a/test/ably/reststats_test.py b/test/ably/rest/reststats_test.py similarity index 100% rename from test/ably/reststats_test.py rename to test/ably/rest/reststats_test.py diff --git a/test/ably/resttime_test.py b/test/ably/rest/resttime_test.py similarity index 100% rename from test/ably/resttime_test.py rename to test/ably/rest/resttime_test.py diff --git a/test/ably/resttoken_test.py b/test/ably/rest/resttoken_test.py similarity index 100% rename from test/ably/resttoken_test.py rename to test/ably/rest/resttoken_test.py From f5de88b8494053650e99b1c1b23f042f66081922 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 1 Feb 2023 12:45:07 +0000 Subject: [PATCH 366/481] test: rename RestSetup to TestApp this module is just used to create/use testapps from sandbox, it's used by rest and realtime tests so this is a better name for it --- test/ably/conftest.py | 6 +- test/ably/realtime/eventemitter_test.py | 8 +-- test/ably/realtime/realtimechannel_test.py | 44 +++++++------- test/ably/realtime/realtimeconnection_test.py | 60 +++++++++---------- test/ably/realtime/realtimeinit_test.py | 12 ++-- test/ably/realtime/realtimeresume_test.py | 20 +++---- test/ably/rest/encoders_test.py | 10 ++-- test/ably/rest/restauth_test.py | 60 +++++++++---------- test/ably/rest/restcapability_test.py | 6 +- test/ably/rest/restchannelhistory_test.py | 6 +- test/ably/rest/restchannelpublish_test.py | 28 ++++----- test/ably/rest/restchannels_test.py | 8 +-- test/ably/rest/restchannelstatus_test.py | 4 +- test/ably/rest/restcrypto_test.py | 8 +-- test/ably/rest/resthttp_test.py | 8 +-- test/ably/rest/restinit_test.py | 8 +-- test/ably/rest/restpaginatedresult_test.py | 4 +- test/ably/rest/restpresence_test.py | 8 +-- test/ably/rest/restpush_test.py | 4 +- test/ably/rest/restrequest_test.py | 6 +- test/ably/rest/reststats_test.py | 6 +- test/ably/rest/resttime_test.py | 6 +- test/ably/rest/resttoken_test.py | 26 ++++---- test/ably/{restsetup.py => testapp.py} | 16 ++--- 24 files changed, 186 insertions(+), 186 deletions(-) rename test/ably/{restsetup.py => testapp.py} (91%) diff --git a/test/ably/conftest.py b/test/ably/conftest.py index 3c1065ea..be61fec1 100644 --- a/test/ably/conftest.py +++ b/test/ably/conftest.py @@ -1,12 +1,12 @@ import asyncio import pytest -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp @pytest.fixture(scope='session', autouse=True) def event_loop(): loop = asyncio.get_event_loop_policy().new_event_loop() - loop.run_until_complete(RestSetup.get_test_vars()) + loop.run_until_complete(TestApp.get_test_vars()) yield loop - loop.run_until_complete(RestSetup.clear_test_vars()) + loop.run_until_complete(TestApp.clear_test_vars()) diff --git a/test/ably/realtime/eventemitter_test.py b/test/ably/realtime/eventemitter_test.py index 08a236fe..873c2f65 100644 --- a/test/ably/realtime/eventemitter_test.py +++ b/test/ably/realtime/eventemitter_test.py @@ -1,15 +1,15 @@ import asyncio from ably.realtime.connection import ConnectionState -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase class TestEventEmitter(BaseAsyncTestCase): async def asyncSetUp(self): - self.test_vars = await RestSetup.get_test_vars() + self.test_vars = await TestApp.get_test_vars() async def test_event_listener_error(self): - realtime = await RestSetup.get_ably_realtime() + realtime = await TestApp.get_ably_realtime() call_count = 0 def listener(_): @@ -28,7 +28,7 @@ def listener(_): await realtime.close() async def test_event_emitter_off(self): - realtime = await RestSetup.get_ably_realtime() + realtime = await TestApp.get_ably_realtime() call_count = 0 def listener(_): diff --git a/test/ably/realtime/realtimechannel_test.py b/test/ably/realtime/realtimechannel_test.py index 4bc77044..fe5cc7d3 100644 --- a/test/ably/realtime/realtimechannel_test.py +++ b/test/ably/realtime/realtimechannel_test.py @@ -3,7 +3,7 @@ from ably.realtime.realtime_channel import ChannelState from ably.transport.websockettransport import ProtocolMessageAction from ably.types.message import Message -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, random_string from ably.realtime.connection import ConnectionState from ably.util.exceptions import AblyException @@ -11,24 +11,24 @@ class TestRealtimeChannel(BaseAsyncTestCase): async def asyncSetUp(self): - self.test_vars = await RestSetup.get_test_vars() + self.test_vars = await TestApp.get_test_vars() self.valid_key_format = "api:key" async def test_channels_get(self): - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() channel = ably.channels.get('my_channel') assert channel == ably.channels.all['my_channel'] await ably.close() async def test_channels_release(self): - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() ably.channels.get('my_channel') ably.channels.release('my_channel') assert ably.channels.all.get('my_channel') is None await ably.close() async def test_channel_attach(self): - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() await ably.connection.once_async(ConnectionState.CONNECTED) channel = ably.channels.get('my_channel') assert channel.state == ChannelState.INITIALIZED @@ -37,7 +37,7 @@ async def test_channel_attach(self): await ably.close() async def test_channel_detach(self): - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() await ably.connection.once_async(ConnectionState.CONNECTED) channel = ably.channels.get('my_channel') await channel.attach() @@ -47,7 +47,7 @@ async def test_channel_detach(self): # RTL7b async def test_subscribe(self): - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() first_message_future = asyncio.Future() second_message_future = asyncio.Future() @@ -78,7 +78,7 @@ def listener(message): await ably.close() async def test_subscribe_coroutine(self): - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() await ably.connection.once_async(ConnectionState.CONNECTED) channel = ably.channels.get('my_channel') await channel.attach() @@ -92,7 +92,7 @@ async def listener(msg): await channel.subscribe('event', listener) # publish a message using rest client - rest = await RestSetup.get_ably_rest() + rest = await TestApp.get_ably_rest() rest_channel = rest.channels.get('my_channel') await rest_channel.publish('event', 'data') @@ -106,7 +106,7 @@ async def listener(msg): # RTL7a async def test_subscribe_all_events(self): - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() await ably.connection.once_async(ConnectionState.CONNECTED) channel = ably.channels.get('my_channel') await channel.attach() @@ -119,7 +119,7 @@ def listener(msg): await channel.subscribe(listener) # publish a message using rest client - rest = await RestSetup.get_ably_rest() + rest = await TestApp.get_ably_rest() rest_channel = rest.channels.get('my_channel') await rest_channel.publish('event', 'data') message = await message_future @@ -133,7 +133,7 @@ def listener(msg): # RTL7c async def test_subscribe_auto_attach(self): - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() await ably.connection.once_async(ConnectionState.CONNECTED) channel = ably.channels.get('my_channel') assert channel.state == ChannelState.INITIALIZED @@ -149,7 +149,7 @@ def listener(_): # RTL8b async def test_unsubscribe(self): - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() await ably.connection.once_async(ConnectionState.CONNECTED) channel = ably.channels.get('my_channel') await channel.attach() @@ -165,7 +165,7 @@ def listener(msg): await channel.subscribe('event', listener) # publish a message using rest client - rest = await RestSetup.get_ably_rest() + rest = await TestApp.get_ably_rest() rest_channel = rest.channels.get('my_channel') await rest_channel.publish('event', 'data') await message_future @@ -184,7 +184,7 @@ def listener(msg): # RTL8c async def test_unsubscribe_all(self): - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() await ably.connection.once_async(ConnectionState.CONNECTED) channel = ably.channels.get('my_channel') await channel.attach() @@ -200,7 +200,7 @@ def listener(msg): await channel.subscribe('event', listener) # publish a message using rest client - rest = await RestSetup.get_ably_rest() + rest = await TestApp.get_ably_rest() rest_channel = rest.channels.get('my_channel') await rest_channel.publish('event', 'data') await message_future @@ -218,7 +218,7 @@ def listener(msg): await rest.close() async def test_realtime_request_timeout_attach(self): - ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) + ably = await TestApp.get_ably_realtime(realtime_request_timeout=2000) await ably.connection.once_async(ConnectionState.CONNECTED) original_send_protocol_message = ably.connection.connection_manager.send_protocol_message @@ -236,7 +236,7 @@ async def new_send_protocol_message(msg): await ably.close() async def test_realtime_request_timeout_detach(self): - ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) + ably = await TestApp.get_ably_realtime(realtime_request_timeout=2000) await ably.connection.once_async(ConnectionState.CONNECTED) original_send_protocol_message = ably.connection.connection_manager.send_protocol_message @@ -255,7 +255,7 @@ async def new_send_protocol_message(msg): await ably.close() async def test_channel_detached_once_connection_closed(self): - ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) + ably = await TestApp.get_ably_realtime(realtime_request_timeout=2000) await ably.connection.once_async(ConnectionState.CONNECTED) channel = ably.channels.get(random_string(5)) await channel.attach() @@ -264,7 +264,7 @@ async def test_channel_detached_once_connection_closed(self): assert channel.state == ChannelState.DETACHED async def test_channel_failed_once_connection_failed(self): - ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) + ably = await TestApp.get_ably_realtime(realtime_request_timeout=2000) await ably.connection.once_async(ConnectionState.CONNECTED) channel = ably.channels.get(random_string(5)) await channel.attach() @@ -275,7 +275,7 @@ async def test_channel_failed_once_connection_failed(self): await ably.close() async def test_channel_suspended_once_connection_suspended(self): - ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) + ably = await TestApp.get_ably_realtime(realtime_request_timeout=2000) await ably.connection.once_async(ConnectionState.CONNECTED) channel = ably.channels.get(random_string(5)) await channel.attach() @@ -286,7 +286,7 @@ async def test_channel_suspended_once_connection_suspended(self): await ably.close() async def test_attach_while_connecting(self): - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() channel = ably.channels.get(random_string(5)) await channel.attach() assert channel.state == ChannelState.ATTACHED diff --git a/test/ably/realtime/realtimeconnection_test.py b/test/ably/realtime/realtimeconnection_test.py index 8034f84d..93ba9bd2 100644 --- a/test/ably/realtime/realtimeconnection_test.py +++ b/test/ably/realtime/realtimeconnection_test.py @@ -3,18 +3,18 @@ import pytest from ably.transport.websockettransport import ProtocolMessageAction from ably.util.exceptions import AblyException -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase from ably.transport.defaults import Defaults class TestRealtimeConnection(BaseAsyncTestCase): async def asyncSetUp(self): - self.test_vars = await RestSetup.get_test_vars() + self.test_vars = await TestApp.get_test_vars() self.valid_key_format = "api:key" async def test_connection_state(self): - ably = await RestSetup.get_ably_realtime(auto_connect=False) + ably = await TestApp.get_ably_realtime(auto_connect=False) assert ably.connection.state == ConnectionState.INITIALIZED ably.connect() await ably.connection.once_async() @@ -25,12 +25,12 @@ async def test_connection_state(self): assert ably.connection.state == ConnectionState.CLOSED async def test_connection_state_is_connecting_on_init(self): - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() assert ably.connection.state == ConnectionState.CONNECTING await ably.close() async def test_auth_invalid_key(self): - ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) + ably = await TestApp.get_ably_realtime(key=self.valid_key_format) state_change = await ably.connection.once_async() assert ably.connection.state == ConnectionState.FAILED assert state_change.reason @@ -42,7 +42,7 @@ async def test_auth_invalid_key(self): await ably.close() async def test_connection_ping_connected(self): - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() await ably.connection.once_async(ConnectionState.CONNECTED) response_time_ms = await ably.connection.ping() assert response_time_ms is not None @@ -50,7 +50,7 @@ async def test_connection_ping_connected(self): await ably.close() async def test_connection_ping_initialized(self): - ably = await RestSetup.get_ably_realtime(auto_connect=False) + ably = await TestApp.get_ably_realtime(auto_connect=False) assert ably.connection.state == ConnectionState.INITIALIZED with pytest.raises(AblyException) as exception: await ably.connection.ping() @@ -58,7 +58,7 @@ async def test_connection_ping_initialized(self): assert exception.value.status_code == 40000 async def test_connection_ping_failed(self): - ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) + ably = await TestApp.get_ably_realtime(key=self.valid_key_format) await ably.connection.once_async(ConnectionState.FAILED) assert ably.connection.state == ConnectionState.FAILED with pytest.raises(AblyException) as exception: @@ -68,7 +68,7 @@ async def test_connection_ping_failed(self): await ably.close() async def test_connection_ping_closed(self): - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() ably.connect() await ably.connection.once_async(ConnectionState.CONNECTED) await ably.close() @@ -78,7 +78,7 @@ async def test_connection_ping_closed(self): assert exception.value.status_code == 40000 async def test_auto_connect(self): - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() connect_future = asyncio.Future() ably.connection.on(ConnectionState.CONNECTED, lambda change: connect_future.set_result(change)) await connect_future @@ -86,7 +86,7 @@ async def test_auto_connect(self): await ably.close() async def test_connection_state_change(self): - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() connected_future = asyncio.Future() @@ -101,7 +101,7 @@ def on_state_change(change): await ably.close() async def test_connection_state_change_reason(self): - ably = await RestSetup.get_ably_realtime(key=self.valid_key_format) + ably = await TestApp.get_ably_realtime(key=self.valid_key_format) state_change = await ably.connection.once_async() @@ -112,7 +112,7 @@ async def test_connection_state_change_reason(self): await ably.close() async def test_realtime_request_timeout_connect(self): - ably = await RestSetup.get_ably_realtime(realtime_request_timeout=0.000001) + ably = await TestApp.get_ably_realtime(realtime_request_timeout=0.000001) state_change = await ably.connection.once_async() assert state_change.reason is not None assert state_change.reason.code == 50003 @@ -122,7 +122,7 @@ async def test_realtime_request_timeout_connect(self): await ably.close() async def test_realtime_request_timeout_ping(self): - ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) + ably = await TestApp.get_ably_realtime(realtime_request_timeout=2000) await ably.connection.once_async(ConnectionState.CONNECTED) original_send_protocol_message = ably.connection.connection_manager.send_protocol_message @@ -142,7 +142,7 @@ async def new_send_protocol_message(protocol_message): await ably.close() async def test_disconnected_retry_timeout(self): - ably = await RestSetup.get_ably_realtime(disconnected_retry_timeout=2000, auto_connect=False) + ably = await TestApp.get_ably_realtime(disconnected_retry_timeout=2000, auto_connect=False) original_connect = ably.connection.connection_manager.connect_base call_count = 0 @@ -167,24 +167,24 @@ async def new_connect(): await ably.close() async def test_connectivity_check_default(self): - ably = await RestSetup.get_ably_realtime(auto_connect=False) + ably = await TestApp.get_ably_realtime(auto_connect=False) # The default connectivity check should return True assert ably.connection.connection_manager.check_connection() is True async def test_connectivity_check_non_default(self): - ably = await RestSetup.get_ably_realtime( + ably = await TestApp.get_ably_realtime( connectivity_check_url="https://echo.ably.io/respondWith?status=200", auto_connect=False) # A non-default URL should return True with a HTTP OK despite not returning "Yes" in the body assert ably.connection.connection_manager.check_connection() is True async def test_connectivity_check_bad_status(self): - ably = await RestSetup.get_ably_realtime( + ably = await TestApp.get_ably_realtime( connectivity_check_url="https://echo.ably.io/respondWith?status=400", auto_connect=False) # Should return False when the URL returns a non-2xx response code assert ably.connection.connection_manager.check_connection() is False async def test_unroutable_host(self): - ably = await RestSetup.get_ably_realtime(realtime_host="10.255.255.1", realtime_request_timeout=3000) + ably = await TestApp.get_ably_realtime(realtime_host="10.255.255.1", realtime_request_timeout=3000) state_change = await ably.connection.once_async() assert state_change.reason assert state_change.reason.code == 50003 @@ -194,7 +194,7 @@ async def test_unroutable_host(self): await ably.close() async def test_invalid_host(self): - ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost") + ably = await TestApp.get_ably_realtime(realtime_host="iamnotahost") state_change = await ably.connection.once_async() assert state_change.reason assert state_change.reason.code == 40000 @@ -205,7 +205,7 @@ async def test_invalid_host(self): async def test_connection_state_ttl(self): Defaults.connection_state_ttl = 10 - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() state_change = await ably.connection.once_async() @@ -220,7 +220,7 @@ async def test_connection_state_ttl(self): Defaults.connection_state_ttl = 120000 async def test_handle_connected(self): - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() test_future = asyncio.Future() def on_update(connection_state): @@ -242,7 +242,7 @@ async def on_transport_pending(transport): await ably.close() async def test_max_idle_interval(self): - ably = await RestSetup.get_ably_realtime(realtime_request_timeout=2000) + ably = await TestApp.get_ably_realtime(realtime_request_timeout=2000) def on_transport_pending(transport): original_on_protocol_message = transport.on_protocol_message @@ -269,7 +269,7 @@ async def on_protocol_message(msg): # RTN15a async def test_retry_immediately_upon_unexpected_disconnection(self): # Set timeouts to 500s so that if the client uses retry delay the test will fail with a timeout - ably = await RestSetup.get_ably_realtime( + ably = await TestApp.get_ably_realtime( disconnected_retry_timeout=500_000, suspended_retry_timeout=500_000 ) @@ -289,8 +289,8 @@ async def test_retry_immediately_upon_unexpected_disconnection(self): async def test_fallback_host(self): fallback_host = 'sandbox-realtime.ably.io' - ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost", disconnected_retry_timeout=400000, - fallback_hosts=[fallback_host]) + ably = await TestApp.get_ably_realtime(realtime_host="iamnotahost", disconnected_retry_timeout=400000, + fallback_hosts=[fallback_host]) await ably.connection.once_async(ConnectionState.CONNECTED) assert ably.connection.connection_manager.transport.host == fallback_host assert ably.options.fallback_realtime_host == fallback_host @@ -298,8 +298,8 @@ async def test_fallback_host(self): async def test_fallback_host_no_connection(self): fallback_host = 'sandbox-realtime.ably.io' - ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost", disconnected_retry_timeout=400000, - fallback_hosts=[fallback_host]) + ably = await TestApp.get_ably_realtime(realtime_host="iamnotahost", disconnected_retry_timeout=400000, + fallback_hosts=[fallback_host]) def check_connection(): return False @@ -312,8 +312,8 @@ def check_connection(): async def test_fallback_host_disconnected_protocol_msg(self): fallback_host = 'sandbox-realtime.ably.io' - ably = await RestSetup.get_ably_realtime(realtime_host="iamnotahost", disconnected_retry_timeout=400000, - fallback_hosts=[fallback_host]) + ably = await TestApp.get_ably_realtime(realtime_host="iamnotahost", disconnected_retry_timeout=400000, + fallback_hosts=[fallback_host]) async def on_transport_pending(transport): await transport.on_protocol_message({'action': 6, "error": {"statusCode": 500, "code": 500}}) diff --git a/test/ably/realtime/realtimeinit_test.py b/test/ably/realtime/realtimeinit_test.py index a146ea25..96fa540c 100644 --- a/test/ably/realtime/realtimeinit_test.py +++ b/test/ably/realtime/realtimeinit_test.py @@ -2,34 +2,34 @@ import pytest from ably import Auth from ably.util.exceptions import AblyAuthException -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase class TestRealtimeInit(BaseAsyncTestCase): async def asyncSetUp(self): - self.test_vars = await RestSetup.get_test_vars() + self.test_vars = await TestApp.get_test_vars() self.valid_key_format = "api:key" async def test_init_with_valid_key(self): - ably = await RestSetup.get_ably_realtime(key=self.test_vars["keys"][0]["key_str"], auto_connect=False) + ably = await TestApp.get_ably_realtime(key=self.test_vars["keys"][0]["key_str"], auto_connect=False) assert Auth.Method.BASIC == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" assert ably.auth.auth_options.key_name == self.test_vars["keys"][0]['key_name'] assert ably.auth.auth_options.key_secret == self.test_vars["keys"][0]['key_secret'] async def test_init_with_incorrect_key(self): with pytest.raises(AblyAuthException): - await RestSetup.get_ably_realtime(key="some invalid key", auto_connect=False) + await TestApp.get_ably_realtime(key="some invalid key", auto_connect=False) async def test_init_with_valid_key_format(self): key = self.valid_key_format.split(":") - ably = await RestSetup.get_ably_realtime(key=self.valid_key_format, auto_connect=False) + ably = await TestApp.get_ably_realtime(key=self.valid_key_format, auto_connect=False) assert Auth.Method.BASIC == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" assert ably.auth.auth_options.key_name == key[0] assert ably.auth.auth_options.key_secret == key[1] async def test_init_without_autoconnect(self): - ably = await RestSetup.get_ably_realtime(auto_connect=False) + ably = await TestApp.get_ably_realtime(auto_connect=False) assert ably.connection.state == ConnectionState.INITIALIZED ably.connect() await ably.connection.once_async(ConnectionState.CONNECTED) diff --git a/test/ably/realtime/realtimeresume_test.py b/test/ably/realtime/realtimeresume_test.py index 81eef739..8ed8a3db 100644 --- a/test/ably/realtime/realtimeresume_test.py +++ b/test/ably/realtime/realtimeresume_test.py @@ -2,7 +2,7 @@ from ably.realtime.connection import ConnectionState from ably.realtime.realtime_channel import ChannelState from ably.transport.websockettransport import ProtocolMessageAction -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, random_string @@ -22,12 +22,12 @@ def on_message(_): class TestRealtimeResume(BaseAsyncTestCase): async def asyncSetUp(self): - self.test_vars = await RestSetup.get_test_vars() + self.test_vars = await TestApp.get_test_vars() self.valid_key_format = "api:key" # RTN15c6 - valid resume response async def test_connection_resume(self): - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() await ably.connection.once_async(ConnectionState.CONNECTED) prev_connection_id = ably.connection.connection_manager.connection_id @@ -44,7 +44,7 @@ async def test_connection_resume(self): # RTN15c4 - fatal resume error async def test_fatal_resume_error(self): - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() await ably.connection.once_async(ConnectionState.CONNECTED) key_name = ably.options.key_name @@ -59,7 +59,7 @@ async def test_fatal_resume_error(self): # RTN15c7 - invalid resume response async def test_invalid_resume_response(self): - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() await ably.connection.once_async(ConnectionState.CONNECTED) @@ -79,7 +79,7 @@ async def test_invalid_resume_response(self): await ably.close() async def test_attached_channel_reattaches_on_invalid_resume(self): - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() await ably.connection.once_async(ConnectionState.CONNECTED) @@ -103,7 +103,7 @@ async def test_attached_channel_reattaches_on_invalid_resume(self): await ably.close() async def test_suspended_channel_reattaches_on_invalid_resume(self): - ably = await RestSetup.get_ably_realtime() + ably = await TestApp.get_ably_realtime() await ably.connection.once_async(ConnectionState.CONNECTED) @@ -126,8 +126,8 @@ async def test_suspended_channel_reattaches_on_invalid_resume(self): await ably.close() async def test_resume_receives_channel_messages_while_disconnected(self): - realtime = await RestSetup.get_ably_realtime() - rest = await RestSetup.get_ably_rest() + realtime = await TestApp.get_ably_realtime() + rest = await TestApp.get_ably_rest() channel_name = random_string(5) @@ -172,7 +172,7 @@ def on_message(message): await rest.close() async def test_resume_update_channel_attached(self): - realtime = await RestSetup.get_ably_realtime() + realtime = await TestApp.get_ably_realtime() name = random_string(5) channel = realtime.channels.get(name) diff --git a/test/ably/rest/encoders_test.py b/test/ably/rest/encoders_test.py index d1328240..6bffba65 100644 --- a/test/ably/rest/encoders_test.py +++ b/test/ably/rest/encoders_test.py @@ -10,7 +10,7 @@ from ably.util.crypto import get_cipher from ably.types.message import Message -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase if sys.version_info >= (3, 8): @@ -23,7 +23,7 @@ class TestTextEncodersNoEncryption(BaseAsyncTestCase): async def asyncSetUp(self): - self.ably = await RestSetup.get_ably_rest(use_binary_protocol=False) + self.ably = await TestApp.get_ably_rest(use_binary_protocol=False) async def asyncTearDown(self): await self.ably.close() @@ -145,7 +145,7 @@ def test_decode_with_invalid_encoding(self): class TestTextEncodersEncryption(BaseAsyncTestCase): async def asyncSetUp(self): - self.ably = await RestSetup.get_ably_rest(use_binary_protocol=False) + self.ably = await TestApp.get_ably_rest(use_binary_protocol=False) self.cipher_params = CipherParams(secret_key='keyfordecrypt_16', algorithm='aes') @@ -259,7 +259,7 @@ async def test_with_json_list_data_decode(self): class TestBinaryEncodersNoEncryption(BaseAsyncTestCase): async def asyncSetUp(self): - self.ably = await RestSetup.get_ably_rest() + self.ably = await TestApp.get_ably_rest() async def asyncTearDown(self): await self.ably.close() @@ -350,7 +350,7 @@ async def test_with_json_list_data_decode(self): class TestBinaryEncodersEncryption(BaseAsyncTestCase): async def asyncSetUp(self): - self.ably = await RestSetup.get_ably_rest() + self.ably = await TestApp.get_ably_rest() self.cipher_params = CipherParams(secret_key='keyfordecrypt_16', algorithm='aes') async def asyncTearDown(self): diff --git a/test/ably/rest/restauth_test.py b/test/ably/rest/restauth_test.py index 63ce9b55..66695c70 100644 --- a/test/ably/rest/restauth_test.py +++ b/test/ably/rest/restauth_test.py @@ -17,7 +17,7 @@ from ably import AblyAuthException from ably.types.tokendetails import TokenDetails -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp from test.ably.utils import VaryByProtocolTestsMetaclass, dont_vary_protocol, BaseAsyncTestCase if sys.version_info >= (3, 8): @@ -31,7 +31,7 @@ # does not make any request, no need to vary by protocol class TestAuth(BaseAsyncTestCase): async def asyncSetUp(self): - self.test_vars = await RestSetup.get_test_vars() + self.test_vars = await TestApp.get_test_vars() def test_auth_init_key_only(self): ably = AblyRest(key=self.test_vars["keys"][0]["key_str"]) @@ -58,7 +58,7 @@ def token_callback(token_params): callback_called.append(True) return "this_is_not_really_a_token_request" - ably = await RestSetup.get_ably_rest( + ably = await TestApp.get_ably_rest( key=None, key_name=self.test_vars["keys"][0]["key_name"], auth_callback=token_callback) @@ -78,7 +78,7 @@ def test_auth_init_with_key_and_client_id(self): assert ably.auth.client_id == 'testClientId' async def test_auth_init_with_token(self): - ably = await RestSetup.get_ably_rest(key=None, token="this_is_not_really_a_token") + ably = await TestApp.get_ably_rest(key=None, token="this_is_not_really_a_token") assert Auth.Method.TOKEN == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" # RSA11 @@ -168,8 +168,8 @@ def test_with_default_token_params(self): class TestAuthAuthorize(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): async def asyncSetUp(self): - self.ably = await RestSetup.get_ably_rest() - self.test_vars = await RestSetup.get_test_vars() + self.ably = await TestApp.get_ably_rest() + self.test_vars = await TestApp.get_test_vars() async def asyncTearDown(self): await self.ably.close() @@ -224,22 +224,22 @@ async def test_authorize_adheres_to_request_token(self): async def test_with_token_str_https(self): token = await self.ably.auth.authorize() token = token.token - ably = await RestSetup.get_ably_rest(key=None, token=token, tls=True, - use_binary_protocol=self.use_binary_protocol) + ably = await TestApp.get_ably_rest(key=None, token=token, tls=True, + use_binary_protocol=self.use_binary_protocol) await ably.channels.test_auth_with_token_str.publish('event', 'foo_bar') await ably.close() async def test_with_token_str_http(self): token = await self.ably.auth.authorize() token = token.token - ably = await RestSetup.get_ably_rest(key=None, token=token, tls=False, - use_binary_protocol=self.use_binary_protocol) + ably = await TestApp.get_ably_rest(key=None, token=token, tls=False, + use_binary_protocol=self.use_binary_protocol) await ably.channels.test_auth_with_token_str.publish('event', 'foo_bar') await ably.close() async def test_if_default_client_id_is_used(self): - ably = await RestSetup.get_ably_rest(client_id='my_client_id', - use_binary_protocol=self.use_binary_protocol) + ably = await TestApp.get_ably_rest(client_id='my_client_id', + use_binary_protocol=self.use_binary_protocol) token = await ably.auth.authorize() assert token.client_id == 'my_client_id' await ably.close() @@ -304,7 +304,7 @@ async def test_timestamp_is_not_stored(self): async def test_client_id_precedence(self): client_id = uuid.uuid4().hex overridden_client_id = uuid.uuid4().hex - ably = await RestSetup.get_ably_rest( + ably = await TestApp.get_ably_rest( use_binary_protocol=self.use_binary_protocol, client_id=client_id, default_token_params={'client_id': overridden_client_id}) @@ -337,20 +337,20 @@ async def test_authorise(self): class TestRequestToken(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): async def asyncSetUp(self): - self.test_vars = await RestSetup.get_test_vars() + self.test_vars = await TestApp.get_test_vars() def per_protocol_setup(self, use_binary_protocol): self.use_binary_protocol = use_binary_protocol async def test_with_key(self): - ably = await RestSetup.get_ably_rest(use_binary_protocol=self.use_binary_protocol) + ably = await TestApp.get_ably_rest(use_binary_protocol=self.use_binary_protocol) token_details = await ably.auth.request_token() assert isinstance(token_details, TokenDetails) await ably.close() - ably = await RestSetup.get_ably_rest(key=None, token_details=token_details, - use_binary_protocol=self.use_binary_protocol) + ably = await TestApp.get_ably_rest(key=None, token_details=token_details, + use_binary_protocol=self.use_binary_protocol) channel = self.get_channel_name('test_request_token_with_key') await ably.channels[channel].publish('event', 'foo') @@ -364,7 +364,7 @@ async def test_with_key(self): async def test_with_auth_url_headers_and_params_POST(self): # noqa: N802 url = 'http://www.example.com' headers = {'foo': 'bar'} - ably = await RestSetup.get_ably_rest(key=None, auth_url=url) + ably = await TestApp.get_ably_rest(key=None, auth_url=url) auth_params = {'foo': 'auth', 'spam': 'eggs'} token_params = {'foo': 'token'} @@ -396,7 +396,7 @@ def call_back(request): async def test_with_auth_url_headers_and_params_GET(self): # noqa: N802 url = 'http://www.example.com' headers = {'foo': 'bar'} - ably = await RestSetup.get_ably_rest( + ably = await TestApp.get_ably_rest( key=None, auth_url=url, auth_headers={'this': 'will_not_be_used'}, auth_params={'this': 'will_not_be_used'}) @@ -429,7 +429,7 @@ async def callback(token_params): assert token_params == called_token_params return 'token_string' - ably = await RestSetup.get_ably_rest(key=None, auth_callback=callback) + ably = await TestApp.get_ably_rest(key=None, auth_callback=callback) token_details = await ably.auth.request_token( token_params=called_token_params, auth_callback=callback) @@ -450,7 +450,7 @@ async def callback(token_params): async def test_when_auth_url_has_query_string(self): url = 'http://www.example.com?with=query' headers = {'foo': 'bar'} - ably = await RestSetup.get_ably_rest(key=None, auth_url=url) + ably = await TestApp.get_ably_rest(key=None, auth_url=url) auth_route = respx.get('http://www.example.com', params={'with': 'query', 'spam': 'eggs'}).mock( return_value=Response(status_code=200, content='token_string')) await ably.auth.request_token(auth_url=url, @@ -461,7 +461,7 @@ async def test_when_auth_url_has_query_string(self): @dont_vary_protocol async def test_client_id_null_for_anonymous_auth(self): - ably = await RestSetup.get_ably_rest( + ably = await TestApp.get_ably_rest( key=None, key_name=self.test_vars["keys"][0]["key_name"], key_secret=self.test_vars["keys"][0]["key_secret"]) @@ -475,7 +475,7 @@ async def test_client_id_null_for_anonymous_auth(self): @dont_vary_protocol async def test_client_id_null_until_auth(self): client_id = uuid.uuid4().hex - token_ably = await RestSetup.get_ably_rest( + token_ably = await TestApp.get_ably_rest( default_token_params={'client_id': client_id}) # before auth, client_id is None assert token_ably.auth.client_id is None @@ -492,8 +492,8 @@ async def test_client_id_null_until_auth(self): class TestRenewToken(BaseAsyncTestCase): async def asyncSetUp(self): - self.test_vars = await RestSetup.get_test_vars() - self.ably = await RestSetup.get_ably_rest(use_binary_protocol=False) + self.test_vars = await TestApp.get_test_vars() + self.ably = await TestApp.get_ably_rest(use_binary_protocol=False) # with headers self.publish_attempts = 0 self.channel = uuid.uuid4().hex @@ -556,7 +556,7 @@ async def test_when_renewable(self): async def test_when_not_renewable(self): await self.ably.close() - self.ably = await RestSetup.get_ably_rest( + self.ably = await TestApp.get_ably_rest( key=None, token='token ID cannot be used to create a new token', use_binary_protocol=False) @@ -574,7 +574,7 @@ async def test_when_not_renewable(self): # RSA4a async def test_when_not_renewable_with_token_details(self): token_details = TokenDetails(token='a_dummy_token') - self.ably = await RestSetup.get_ably_rest( + self.ably = await TestApp.get_ably_rest( key=None, token_details=token_details, use_binary_protocol=False) @@ -593,7 +593,7 @@ async def test_when_not_renewable_with_token_details(self): class TestRenewExpiredToken(BaseAsyncTestCase): async def asyncSetUp(self): - self.test_vars = await RestSetup.get_test_vars() + self.test_vars = await TestApp.get_test_vars() self.publish_attempts = 0 self.channel = uuid.uuid4().hex @@ -645,7 +645,7 @@ async def asyncTearDown(self): # RSA4b1 async def test_query_time_false(self): - ably = await RestSetup.get_ably_rest() + ably = await TestApp.get_ably_rest() await ably.auth.authorize() self.publish_fail = True await ably.channels[self.channel].publish('evt', 'msg') @@ -654,7 +654,7 @@ async def test_query_time_false(self): # RSA4b1 async def test_query_time_true(self): - ably = await RestSetup.get_ably_rest(query_time=True) + ably = await TestApp.get_ably_rest(query_time=True) await ably.auth.authorize() self.publish_fail = False await ably.channels[self.channel].publish('evt', 'msg') diff --git a/test/ably/rest/restcapability_test.py b/test/ably/rest/restcapability_test.py index 826b0baf..0182dcb0 100644 --- a/test/ably/rest/restcapability_test.py +++ b/test/ably/rest/restcapability_test.py @@ -3,15 +3,15 @@ from ably.types.capability import Capability from ably.util.exceptions import AblyException -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp from test.ably.utils import VaryByProtocolTestsMetaclass, dont_vary_protocol, BaseAsyncTestCase class TestRestCapability(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): async def asyncSetUp(self): - self.test_vars = await RestSetup.get_test_vars() - self.ably = await RestSetup.get_ably_rest() + self.test_vars = await TestApp.get_test_vars() + self.ably = await TestApp.get_ably_rest() async def asyncTearDown(self): await self.ably.close() diff --git a/test/ably/rest/restchannelhistory_test.py b/test/ably/rest/restchannelhistory_test.py index 382bc251..30d94e91 100644 --- a/test/ably/rest/restchannelhistory_test.py +++ b/test/ably/rest/restchannelhistory_test.py @@ -5,7 +5,7 @@ from ably import AblyException from ably.http.paginatedresult import PaginatedResult -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp from test.ably.utils import VaryByProtocolTestsMetaclass, dont_vary_protocol, BaseAsyncTestCase log = logging.getLogger(__name__) @@ -14,8 +14,8 @@ class TestRestChannelHistory(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): async def asyncSetUp(self): - self.ably = await RestSetup.get_ably_rest() - self.test_vars = await RestSetup.get_test_vars() + self.ably = await TestApp.get_ably_rest() + self.test_vars = await TestApp.get_test_vars() async def asyncTearDown(self): await self.ably.close() diff --git a/test/ably/rest/restchannelpublish_test.py b/test/ably/rest/restchannelpublish_test.py index ed571185..ed415527 100644 --- a/test/ably/rest/restchannelpublish_test.py +++ b/test/ably/rest/restchannelpublish_test.py @@ -17,7 +17,7 @@ from ably.types.tokendetails import TokenDetails from ably.util import case -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp from test.ably.utils import VaryByProtocolTestsMetaclass, dont_vary_protocol, BaseAsyncTestCase log = logging.getLogger(__name__) @@ -28,10 +28,10 @@ class TestRestChannelPublish(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): async def asyncSetUp(self): - self.test_vars = await RestSetup.get_test_vars() - self.ably = await RestSetup.get_ably_rest() + self.test_vars = await TestApp.get_test_vars() + self.ably = await TestApp.get_ably_rest() self.client_id = uuid.uuid4().hex - self.ably_with_client_id = await RestSetup.get_ably_rest(client_id=self.client_id, use_token_auth=True) + self.ably_with_client_id = await TestApp.get_ably_rest(client_id=self.client_id, use_token_auth=True) async def asyncTearDown(self): await self.ably.close() @@ -119,7 +119,7 @@ async def test_message_list_generate_one_request(self): assert message['data'] == str(i) async def test_publish_error(self): - ably = await RestSetup.get_ably_rest(use_binary_protocol=self.use_binary_protocol) + ably = await TestApp.get_ably_rest(use_binary_protocol=self.use_binary_protocol) await ably.auth.authorize( token_params={'capability': {"only_subscribe": ["subscribe"]}}) @@ -297,9 +297,9 @@ async def test_publish_message_with_client_id_on_identified_client(self): async def test_publish_message_with_wrong_client_id_on_implicit_identified_client(self): new_token = await self.ably.auth.authorize(token_params={'client_id': uuid.uuid4().hex}) - new_ably = await RestSetup.get_ably_rest(key=None, - token=new_token.token, - use_binary_protocol=self.use_binary_protocol) + new_ably = await TestApp.get_ably_rest(key=None, + token=new_token.token, + use_binary_protocol=self.use_binary_protocol) channel = new_ably.channels[ self.get_channel_name('persisted:wrong_client_id_implicit_client')] @@ -314,7 +314,7 @@ async def test_publish_message_with_wrong_client_id_on_implicit_identified_clien # RSA15b async def test_wildcard_client_id_can_publish_as_others(self): wildcard_token_details = await self.ably.auth.request_token({'client_id': '*'}) - wildcard_ably = await RestSetup.get_ably_rest( + wildcard_ably = await TestApp.get_ably_rest( key=None, token_details=wildcard_token_details, use_binary_protocol=self.use_binary_protocol) @@ -442,8 +442,8 @@ async def test_publish_params(self): class TestRestChannelPublishIdempotent(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): async def asyncSetUp(self): - self.ably = await RestSetup.get_ably_rest() - self.ably_idempotent = await RestSetup.get_ably_rest(idempotent_rest_publishing=True) + self.ably = await TestApp.get_ably_rest() + self.ably_idempotent = await TestApp.get_ably_rest(idempotent_rest_publishing=True) async def asyncTearDown(self): await self.ably.close() @@ -463,11 +463,11 @@ async def test_idempotent_rest_publishing(self): assert self.ably.options.idempotent_rest_publishing is True # Test setting value explicitly - ably = await RestSetup.get_ably_rest(idempotent_rest_publishing=True) + ably = await TestApp.get_ably_rest(idempotent_rest_publishing=True) assert ably.options.idempotent_rest_publishing is True await ably.close() - ably = await RestSetup.get_ably_rest(idempotent_rest_publishing=False) + ably = await TestApp.get_ably_rest(idempotent_rest_publishing=False) assert ably.options.idempotent_rest_publishing is False await ably.close() @@ -523,7 +523,7 @@ def test_idempotent_mixed_ids(self): def get_ably_rest(self, *args, **kwargs): kwargs['use_binary_protocol'] = self.use_binary_protocol - return RestSetup.get_ably_rest(*args, **kwargs) + return TestApp.get_ably_rest(*args, **kwargs) # RSL1k4 async def test_idempotent_library_generated_retry(self): diff --git a/test/ably/rest/restchannels_test.py b/test/ably/rest/restchannels_test.py index 9ddcdbd7..c6a791fe 100644 --- a/test/ably/rest/restchannels_test.py +++ b/test/ably/rest/restchannels_test.py @@ -6,7 +6,7 @@ from ably.rest.channel import Channel, Channels, Presence from ably.util.crypto import generate_random_key -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase @@ -14,8 +14,8 @@ class TestChannels(BaseAsyncTestCase): async def asyncSetUp(self): - self.test_vars = await RestSetup.get_test_vars() - self.ably = await RestSetup.get_ably_rest() + self.test_vars = await TestApp.get_test_vars() + self.ably = await TestApp.get_ably_rest() async def asyncTearDown(self): await self.ably.close() @@ -91,7 +91,7 @@ def test_channel_has_presence(self): async def test_without_permissions(self): key = self.test_vars["keys"][2] - ably = await RestSetup.get_ably_rest(key=key["key_str"]) + ably = await TestApp.get_ably_rest(key=key["key_str"]) with pytest.raises(AblyException) as excinfo: await ably.channels['test_publish_without_permission'].publish('foo', 'woop') diff --git a/test/ably/rest/restchannelstatus_test.py b/test/ably/rest/restchannelstatus_test.py index 7673e410..c1c6e5e1 100644 --- a/test/ably/rest/restchannelstatus_test.py +++ b/test/ably/rest/restchannelstatus_test.py @@ -1,6 +1,6 @@ import logging -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp from test.ably.utils import VaryByProtocolTestsMetaclass, BaseAsyncTestCase log = logging.getLogger(__name__) @@ -9,7 +9,7 @@ class TestRestChannelStatus(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): async def asyncSetUp(self): - self.ably = await RestSetup.get_ably_rest() + self.ably = await TestApp.get_ably_rest() async def asyncTearDown(self): await self.ably.close() diff --git a/test/ably/rest/restcrypto_test.py b/test/ably/rest/restcrypto_test.py index 3fa4918d..18bf69ac 100644 --- a/test/ably/rest/restcrypto_test.py +++ b/test/ably/rest/restcrypto_test.py @@ -11,7 +11,7 @@ from Crypto import Random -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp from test.ably.utils import dont_vary_protocol, VaryByProtocolTestsMetaclass, BaseTestCase, BaseAsyncTestCase log = logging.getLogger(__name__) @@ -20,9 +20,9 @@ class TestRestCrypto(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): async def asyncSetUp(self): - self.test_vars = await RestSetup.get_test_vars() - self.ably = await RestSetup.get_ably_rest() - self.ably2 = await RestSetup.get_ably_rest() + self.test_vars = await TestApp.get_test_vars() + self.ably = await TestApp.get_ably_rest() + self.ably2 = await TestApp.get_ably_rest() async def asyncTearDown(self): await self.ably.close() diff --git a/test/ably/rest/resthttp_test.py b/test/ably/rest/resthttp_test.py index ed9db26c..ad1fe043 100644 --- a/test/ably/rest/resthttp_test.py +++ b/test/ably/rest/resthttp_test.py @@ -13,7 +13,7 @@ from ably.transport.defaults import Defaults from ably.types.options import Options from ably.util.exceptions import AblyException -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase @@ -105,7 +105,7 @@ async def test_no_host_fallback_nor_retries_if_custom_host(self): @pytest.mark.filterwarnings('ignore::DeprecationWarning') async def test_cached_fallback(self): timeout = 2000 - ably = await RestSetup.get_ably_rest(fallback_hosts_use_default=True, fallback_retry_timeout=timeout) + ably = await TestApp.get_ably_rest(fallback_hosts_use_default=True, fallback_retry_timeout=timeout) host = ably.options.get_rest_host() state = {'errors': 0} @@ -195,7 +195,7 @@ def test_custom_http_timeouts(self): # RSC7a, RSC7b async def test_request_headers(self): - ably = await RestSetup.get_ably_rest() + ably = await TestApp.get_ably_rest() r = await ably.http.make_request('HEAD', '/time', skip_auth=True) # API @@ -212,7 +212,7 @@ async def test_request_over_http2(self): url = 'https://www.example.com' respx.get(url).mock(return_value=Response(status_code=200)) - ably = await RestSetup.get_ably_rest(rest_host=url) + ably = await TestApp.get_ably_rest(rest_host=url) r = await ably.http.make_request('GET', url, skip_auth=True) assert r.http_version == 'HTTP/2' await ably.close() diff --git a/test/ably/rest/restinit_test.py b/test/ably/rest/restinit_test.py index 0b92691e..88a433da 100644 --- a/test/ably/rest/restinit_test.py +++ b/test/ably/rest/restinit_test.py @@ -8,14 +8,14 @@ from ably.transport.defaults import Defaults from ably.types.tokendetails import TokenDetails -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp from test.ably.utils import VaryByProtocolTestsMetaclass, dont_vary_protocol, BaseAsyncTestCase class TestRestInit(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): async def asyncSetUp(self): - self.test_vars = await RestSetup.get_test_vars() + self.test_vars = await TestApp.get_test_vars() @dont_vary_protocol def test_key_only(self): @@ -181,8 +181,8 @@ def test_with_no_auth_params(self): # RSA10k async def test_query_time_param(self): - ably = await RestSetup.get_ably_rest(query_time=True, - use_binary_protocol=self.use_binary_protocol) + ably = await TestApp.get_ably_rest(query_time=True, + use_binary_protocol=self.use_binary_protocol) timestamp = ably.auth._timestamp with patch('ably.rest.rest.AblyRest.time', wraps=ably.time) as server_time,\ diff --git a/test/ably/rest/restpaginatedresult_test.py b/test/ably/rest/restpaginatedresult_test.py index 5716d47b..1ad693bf 100644 --- a/test/ably/rest/restpaginatedresult_test.py +++ b/test/ably/rest/restpaginatedresult_test.py @@ -3,7 +3,7 @@ from ably.http.paginatedresult import PaginatedResult -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase @@ -28,7 +28,7 @@ def callback(request): return callback async def asyncSetUp(self): - self.ably = await RestSetup.get_ably_rest(use_binary_protocol=False) + self.ably = await TestApp.get_ably_rest(use_binary_protocol=False) # Mocked responses # without specific headers self.mocked_api = respx.mock(base_url='http://rest.ably.io') diff --git a/test/ably/rest/restpresence_test.py b/test/ably/rest/restpresence_test.py index f2ca42d8..2c525b02 100644 --- a/test/ably/rest/restpresence_test.py +++ b/test/ably/rest/restpresence_test.py @@ -7,14 +7,14 @@ from ably.types.presence import PresenceMessage from test.ably.utils import dont_vary_protocol, VaryByProtocolTestsMetaclass, BaseAsyncTestCase -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp class TestPresence(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): async def asyncSetUp(self): - self.test_vars = await RestSetup.get_test_vars() - self.ably = await RestSetup.get_ably_rest() + self.test_vars = await TestApp.get_test_vars() + self.ably = await TestApp.get_ably_rest() self.channel = self.ably.channels.get('persisted:presence_fixtures') self.ably.options.use_binary_protocol = True @@ -190,7 +190,7 @@ async def test_with_start_gt_end(self): class TestPresenceCrypt(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): async def asyncSetUp(self): - self.ably = await RestSetup.get_ably_rest() + self.ably = await TestApp.get_ably_rest() key = b'0123456789abcdef' self.channel = self.ably.channels.get('persisted:presence_fixtures', cipher={'key': key}) diff --git a/test/ably/rest/restpush_test.py b/test/ably/rest/restpush_test.py index b3862afe..acbe05a7 100644 --- a/test/ably/rest/restpush_test.py +++ b/test/ably/rest/restpush_test.py @@ -9,7 +9,7 @@ from ably import DeviceDetails, PushChannelSubscription from ably.http.paginatedresult import PaginatedResult -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp from test.ably.utils import VaryByProtocolTestsMetaclass, BaseAsyncTestCase from test.ably.utils import new_dict, random_string, get_random_key @@ -20,7 +20,7 @@ class TestPush(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): async def asyncSetUp(self): - self.ably = await RestSetup.get_ably_rest() + self.ably = await TestApp.get_ably_rest() # Register several devices for later use self.devices = {} diff --git a/test/ably/rest/restrequest_test.py b/test/ably/rest/restrequest_test.py index 5f843716..78702bc5 100644 --- a/test/ably/rest/restrequest_test.py +++ b/test/ably/rest/restrequest_test.py @@ -3,7 +3,7 @@ from ably import AblyRest from ably.http.paginatedresult import HttpPaginatedResponse -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase from test.ably.utils import VaryByProtocolTestsMetaclass, dont_vary_protocol @@ -12,8 +12,8 @@ class TestRestRequest(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): async def asyncSetUp(self): - self.ably = await RestSetup.get_ably_rest() - self.test_vars = await RestSetup.get_test_vars() + self.ably = await TestApp.get_ably_rest() + self.test_vars = await TestApp.get_test_vars() # Populate the channel (using the new api) self.channel = self.get_channel_name() diff --git a/test/ably/rest/reststats_test.py b/test/ably/rest/reststats_test.py index e5013f56..2b612ade 100644 --- a/test/ably/rest/reststats_test.py +++ b/test/ably/rest/reststats_test.py @@ -8,7 +8,7 @@ from ably.util.exceptions import AblyException from ably.http.paginatedresult import PaginatedResult -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp from test.ably.utils import VaryByProtocolTestsMetaclass, dont_vary_protocol, BaseAsyncTestCase log = logging.getLogger(__name__) @@ -26,8 +26,8 @@ def get_params(self): } async def asyncSetUp(self): - self.ably = await RestSetup.get_ably_rest() - self.ably_text = await RestSetup.get_ably_rest(use_binary_protocol=False) + self.ably = await TestApp.get_ably_rest() + self.ably_text = await TestApp.get_ably_rest(use_binary_protocol=False) self.last_year = datetime.now().year - 1 self.previous_year = datetime.now().year - 2 diff --git a/test/ably/rest/resttime_test.py b/test/ably/rest/resttime_test.py index 3fba06f2..6189ebd0 100644 --- a/test/ably/rest/resttime_test.py +++ b/test/ably/rest/resttime_test.py @@ -4,7 +4,7 @@ from ably import AblyException -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp from test.ably.utils import VaryByProtocolTestsMetaclass, dont_vary_protocol, BaseAsyncTestCase @@ -15,7 +15,7 @@ def per_protocol_setup(self, use_binary_protocol): self.use_binary_protocol = use_binary_protocol async def asyncSetUp(self): - self.ably = await RestSetup.get_ably_rest() + self.ably = await TestApp.get_ably_rest() async def asyncTearDown(self): await self.ably.close() @@ -36,7 +36,7 @@ async def test_time_without_key_or_token(self): @dont_vary_protocol async def test_time_fails_without_valid_host(self): - ably = await RestSetup.get_ably_rest(key=None, token='foo', rest_host="this.host.does.not.exist") + ably = await TestApp.get_ably_rest(key=None, token='foo', rest_host="this.host.does.not.exist") with pytest.raises(AblyException): await ably.time() diff --git a/test/ably/rest/resttoken_test.py b/test/ably/rest/resttoken_test.py index ea1e45cc..0d40d202 100644 --- a/test/ably/rest/resttoken_test.py +++ b/test/ably/rest/resttoken_test.py @@ -11,7 +11,7 @@ from ably.types.tokendetails import TokenDetails from ably.types.tokenrequest import TokenRequest -from test.ably.restsetup import RestSetup +from test.ably.testapp import TestApp from test.ably.utils import VaryByProtocolTestsMetaclass, dont_vary_protocol, BaseAsyncTestCase log = logging.getLogger(__name__) @@ -25,7 +25,7 @@ async def server_time(self): async def asyncSetUp(self): capability = {"*": ["*"]} self.permit_all = str(Capability(capability)) - self.ably = await RestSetup.get_ably_rest() + self.ably = await TestApp.get_ably_rest() async def asyncTearDown(self): await self.ably.close() @@ -93,7 +93,7 @@ async def test_request_token_with_capability_that_subsets_key_capability(self): assert capability == token_details.capability, "Unexpected capability" async def test_request_token_with_specified_key(self): - test_vars = await RestSetup.get_test_vars() + test_vars = await TestApp.get_test_vars() key = test_vars["keys"][1] token_details = await self.ably.auth.request_token( key_name=key["key_name"], key_secret=key["key_secret"]) @@ -161,7 +161,7 @@ async def test_request_token_float_and_timedelta(self): class TestCreateTokenRequest(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): async def asyncSetUp(self): - self.ably = await RestSetup.get_ably_rest() + self.ably = await TestApp.get_ably_rest() self.key_name = self.ably.options.key_name self.key_secret = self.ably.options.key_secret @@ -174,7 +174,7 @@ def per_protocol_setup(self, use_binary_protocol): @dont_vary_protocol async def test_key_name_and_secret_are_required(self): - ably = await RestSetup.get_ably_rest(key=None, token='not a real token') + ably = await TestApp.get_ably_rest(key=None, token='not a real token') with pytest.raises(AblyException, match="40101 401 No key specified"): await ably.auth.create_token_request() with pytest.raises(AblyException, match="40101 401 No key specified"): @@ -215,9 +215,9 @@ async def test_token_request_can_be_used_to_get_a_token(self): async def auth_callback(token_params): return token_request - ably = await RestSetup.get_ably_rest(key=None, - auth_callback=auth_callback, - use_binary_protocol=self.use_binary_protocol) + ably = await TestApp.get_ably_rest(key=None, + auth_callback=auth_callback, + use_binary_protocol=self.use_binary_protocol) token = await ably.auth.authorize() assert isinstance(token, TokenDetails) @@ -231,9 +231,9 @@ async def test_token_request_dict_can_be_used_to_get_a_token(self): async def auth_callback(token_params): return token_request.to_dict() - ably = await RestSetup.get_ably_rest(key=None, - auth_callback=auth_callback, - use_binary_protocol=self.use_binary_protocol) + ably = await TestApp.get_ably_rest(key=None, + auth_callback=auth_callback, + use_binary_protocol=self.use_binary_protocol) token = await ably.auth.authorize() assert isinstance(token, TokenDetails) @@ -307,8 +307,8 @@ async def test_capability(self): async def auth_callback(token_params): return token_request - ably = await RestSetup.get_ably_rest(key=None, auth_callback=auth_callback, - use_binary_protocol=self.use_binary_protocol) + ably = await TestApp.get_ably_rest(key=None, auth_callback=auth_callback, + use_binary_protocol=self.use_binary_protocol) token = await ably.auth.authorize() diff --git a/test/ably/restsetup.py b/test/ably/testapp.py similarity index 91% rename from test/ably/restsetup.py rename to test/ably/testapp.py index 32097567..f2c1c593 100644 --- a/test/ably/restsetup.py +++ b/test/ably/testapp.py @@ -33,12 +33,12 @@ use_binary_protocol=False) -class RestSetup: +class TestApp: __test_vars = None @staticmethod async def get_test_vars(sender=None): - if not RestSetup.__test_vars: + if not TestApp.__test_vars: r = await ably.http.post("/apps", body=app_spec_local, skip_auth=True) AblyException.raise_for_response(r) @@ -62,15 +62,15 @@ async def get_test_vars(sender=None): } for k in app_spec.get("keys", [])] } - RestSetup.__test_vars = test_vars + TestApp.__test_vars = test_vars log.debug([(app_id, k.get("id", ""), k.get("value", "")) for k in app_spec.get("keys", [])]) - return RestSetup.__test_vars + return TestApp.__test_vars @classmethod async def get_ably_rest(cls, **kw): - test_vars = await RestSetup.get_test_vars() + test_vars = await TestApp.get_test_vars() options = { 'key': test_vars["keys"][0]["key_str"], 'rest_host': test_vars["host"], @@ -84,7 +84,7 @@ async def get_ably_rest(cls, **kw): @classmethod async def get_ably_realtime(cls, **kw): - test_vars = await RestSetup.get_test_vars() + test_vars = await TestApp.get_test_vars() options = { 'key': test_vars["keys"][0]["key_str"], 'realtime_host': test_vars["realtime_host"], @@ -99,7 +99,7 @@ async def get_ably_realtime(cls, **kw): @classmethod async def clear_test_vars(cls): - test_vars = RestSetup.__test_vars + test_vars = TestApp.__test_vars options = Options(key=test_vars["keys"][0]["key_str"]) options.rest_host = test_vars["host"] options.port = test_vars["port"] @@ -107,5 +107,5 @@ async def clear_test_vars(cls): options.tls = test_vars["tls"] ably = await cls.get_ably_rest() await ably.http.delete('/apps/' + test_vars['app_id']) - RestSetup.__test_vars = None + TestApp.__test_vars = None await ably.close() From d7ba209d1cd0a3d3a2d5694fd143f154b3df8628 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 1 Feb 2023 12:47:28 +0000 Subject: [PATCH 367/481] test: fix some warnings in TestApp module --- test/ably/testapp.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/ably/testapp.py b/test/ably/testapp.py index f2c1c593..4fec4d56 100644 --- a/test/ably/testapp.py +++ b/test/ably/testapp.py @@ -22,7 +22,7 @@ tls_port = 443 if host and not host.endswith("rest.ably.io"): - tls = tls and not host.equals("localhost") + tls = tls and host != "localhost" port = 8080 tls_port = 8081 @@ -37,7 +37,7 @@ class TestApp: __test_vars = None @staticmethod - async def get_test_vars(sender=None): + async def get_test_vars(): if not TestApp.__test_vars: r = await ably.http.post("/apps", body=app_spec_local, skip_auth=True) AblyException.raise_for_response(r) @@ -68,8 +68,8 @@ async def get_test_vars(sender=None): return TestApp.__test_vars - @classmethod - async def get_ably_rest(cls, **kw): + @staticmethod + async def get_ably_rest(**kw): test_vars = await TestApp.get_test_vars() options = { 'key': test_vars["keys"][0]["key_str"], @@ -82,8 +82,8 @@ async def get_ably_rest(cls, **kw): options.update(kw) return AblyRest(**options) - @classmethod - async def get_ably_realtime(cls, **kw): + @staticmethod + async def get_ably_realtime(**kw): test_vars = await TestApp.get_test_vars() options = { 'key': test_vars["keys"][0]["key_str"], @@ -97,15 +97,15 @@ async def get_ably_realtime(cls, **kw): options.update(kw) return AblyRealtime(**options) - @classmethod - async def clear_test_vars(cls): + @staticmethod + async def clear_test_vars(): test_vars = TestApp.__test_vars options = Options(key=test_vars["keys"][0]["key_str"]) options.rest_host = test_vars["host"] options.port = test_vars["port"] options.tls_port = test_vars["tls_port"] options.tls = test_vars["tls"] - ably = await cls.get_ably_rest() + ably = await TestApp.get_ably_rest() await ably.http.delete('/apps/' + test_vars['app_id']) TestApp.__test_vars = None await ably.close() From ed8646d36b35fefde5e6cc77e073ef9a042b669b Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 1 Feb 2023 16:38:54 +0000 Subject: [PATCH 368/481] refactor: fix typing error in Channels.__getattr__ Behaves the same like this, even in python 3.7. I guess calling super().__getattr__ was the way to do it in some old version of python but it's no longer necessary and emits a type error so this is better --- ably/rest/channel.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ably/rest/channel.py b/ably/rest/channel.py index be2671de..f33ad34b 100644 --- a/ably/rest/channel.py +++ b/ably/rest/channel.py @@ -203,10 +203,7 @@ def __getitem__(self, key): return self.get(key) def __getattr__(self, name): - try: - return super().__getattr__(name) - except AttributeError: - return self.get(name) + return self.get(name) def __contains__(self, item): if isinstance(item, Channel): From 4917790cbd63580dc141044e30b286d1b5f95596 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 1 Feb 2023 16:40:44 +0000 Subject: [PATCH 369/481] refactor: rename Channels.__attached to Channels.__all This name is misleading when we're dealing with realtime clients because being 'attached' has a different meaning. --- ably/rest/channel.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ably/rest/channel.py b/ably/rest/channel.py index f33ad34b..aae177a4 100644 --- a/ably/rest/channel.py +++ b/ably/rest/channel.py @@ -184,16 +184,16 @@ def options(self, options): class Channels: def __init__(self, rest): self.__ably = rest - self.__attached = OrderedDict() + self.__all = OrderedDict() def get(self, name, **kwargs): if isinstance(name, bytes): name = name.decode('ascii') - if name not in self.__attached: - result = self.__attached[name] = Channel(self.__ably, name, kwargs) + if name not in self.__all: + result = self.__all[name] = Channel(self.__ably, name, kwargs) else: - result = self.__attached[name] + result = self.__all[name] if len(kwargs) != 0: result.options = kwargs @@ -213,13 +213,13 @@ def __contains__(self, item): else: name = item - return name in self.__attached + return name in self.__all def __iter__(self): - return iter(self.__attached.values()) + return iter(self.__all.values()) def release(self, key): - del self.__attached[key] + del self.__all[key] def __delitem__(self, key): return self.release(key) From 4dde8c26cc1551fabc35b5bdb935bf0f89de9f5c Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 1 Feb 2023 16:55:58 +0000 Subject: [PATCH 370/481] refactor: Realtime.Channels extends Rest.Channels Rest.Channels has some useful methods for attribute reading, iteration, etc, so this allows us to inherit those for Realtime.Channels --- ably/realtime/realtime.py | 30 ++++++++++------------ test/ably/realtime/realtimechannel_test.py | 10 +++++--- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index a3987ef4..1f8875a4 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -4,6 +4,7 @@ from ably.rest.auth import Auth from ably.rest.rest import AblyRest from ably.types.options import Options +from ably.rest.channel import Channels as RestChannels from ably.realtime.realtime_channel import ChannelState, RealtimeChannel @@ -154,7 +155,7 @@ def channels(self): return self.__channels -class Channels: +class Channels(RestChannels): """Creates and destroys RealtimeChannel objects. Methods @@ -165,10 +166,6 @@ class Channels: Releases a channel """ - def __init__(self, realtime): - self.all = {} - self.__realtime = realtime - # RTS3 def get(self, name): """Creates a new RealtimeChannel object, or returns the existing channel object. @@ -179,9 +176,11 @@ def get(self, name): name: str Channel name """ - if not self.all.get(name): - self.all[name] = RealtimeChannel(self.__realtime, name) - return self.all[name] + if name not in self.__all: + channel = self.__all[name] = RealtimeChannel(self.__ably, name) + else: + channel = self.__all[name] + return channel # RTS4 def release(self, name): @@ -196,9 +195,9 @@ def release(self, name): name: str Channel name """ - if not self.all.get(name): + if name not in self.__all: return - del self.all[name] + del self.__all[name] def _on_channel_message(self, msg): channel_name = msg.get('channel') @@ -209,7 +208,7 @@ def _on_channel_message(self, msg): ) return - channel = self.all.get(msg.get('channel')) + channel = self.__all[channel_name] if not channel: log.warning( 'Channels.on_channel_message()', @@ -234,15 +233,14 @@ def _propagate_connection_interruption(self, state: ConnectionState, reason): ConnectionState.SUSPENDED: ChannelState.SUSPENDED, } - for name in self.all.keys(): - channel = self.all[name] + for channel_name in self.__all: + channel = self.__all[channel_name] if channel.state in from_channel_states: channel._notify_state(connection_to_channel_state[state], reason) def _on_connected(self): - for channel_name in self.all.keys(): - channel = self.all[channel_name] - + for channel_name in self.__all: + channel = self.__all[channel_name] if channel.state == ChannelState.ATTACHING or channel.state == ChannelState.DETACHING: channel._check_pending_state() elif channel.state == ChannelState.SUSPENDED: diff --git a/test/ably/realtime/realtimechannel_test.py b/test/ably/realtime/realtimechannel_test.py index fe5cc7d3..c48ea8a9 100644 --- a/test/ably/realtime/realtimechannel_test.py +++ b/test/ably/realtime/realtimechannel_test.py @@ -1,6 +1,6 @@ import asyncio import pytest -from ably.realtime.realtime_channel import ChannelState +from ably.realtime.realtime_channel import ChannelState, RealtimeChannel from ably.transport.websockettransport import ProtocolMessageAction from ably.types.message import Message from test.ably.testapp import TestApp @@ -17,14 +17,18 @@ async def asyncSetUp(self): async def test_channels_get(self): ably = await TestApp.get_ably_realtime() channel = ably.channels.get('my_channel') - assert channel == ably.channels.all['my_channel'] + assert channel == ably.channels.get('my_channel') + assert isinstance(channel, RealtimeChannel) await ably.close() async def test_channels_release(self): ably = await TestApp.get_ably_realtime() ably.channels.get('my_channel') ably.channels.release('my_channel') - assert ably.channels.all.get('my_channel') is None + + for _ in ably.channels: + raise AssertionError("Expected no channels to exist") + await ably.close() async def test_channel_attach(self): From d732d389dc30e5a971c27653aeb98c3e689a3807 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 1 Feb 2023 17:00:09 +0000 Subject: [PATCH 371/481] refactor: improve Realtime.Channels typings Makes it so that realtime.channels.get(name) returns a RealtimeChannel and iteators over realtime.channels gives strings --- ably/realtime/realtime.py | 2 +- ably/rest/channel.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 1f8875a4..d3112f1f 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -167,7 +167,7 @@ class Channels(RestChannels): """ # RTS3 - def get(self, name): + def get(self, name) -> RealtimeChannel: """Creates a new RealtimeChannel object, or returns the existing channel object. Parameters diff --git a/ably/rest/channel.py b/ably/rest/channel.py index aae177a4..5ea8efd3 100644 --- a/ably/rest/channel.py +++ b/ably/rest/channel.py @@ -3,6 +3,7 @@ import logging import json import os +from typing import Iterator from urllib import parse import warnings @@ -215,7 +216,7 @@ def __contains__(self, item): return name in self.__all - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(self.__all.values()) def release(self, key): From 033c94132ecc8f82eb4122deed94096da95ca2ca Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 1 Feb 2023 20:12:47 +0000 Subject: [PATCH 372/481] refactor: add channel_retry_timeout option + default --- ably/transport/defaults.py | 1 + ably/types/options.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ably/transport/defaults.py b/ably/transport/defaults.py index d4960f65..7a732d9a 100644 --- a/ably/transport/defaults.py +++ b/ably/transport/defaults.py @@ -21,6 +21,7 @@ class Defaults: comet_recv_timeout = 90000 comet_send_timeout = 10000 realtime_request_timeout = 10000 + channel_retry_timeout = 15000 disconnected_retry_timeout = 15000 connection_state_ttl = 120000 suspended_retry_timeout = 30000 diff --git a/ably/types/options.py b/ably/types/options.py index 750b91ac..4db971e8 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -15,7 +15,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti http_max_retry_count=None, http_max_retry_duration=None, fallback_hosts=None, fallback_hosts_use_default=None, fallback_retry_timeout=None, disconnected_retry_timeout=None, idempotent_rest_publishing=None, loop=None, auto_connect=True, suspended_retry_timeout=None, - connectivity_check_url=None, **kwargs): + connectivity_check_url=None, channel_retry_timeout=Defaults.channel_retry_timeout, **kwargs): super().__init__(**kwargs) # TODO check these defaults @@ -69,6 +69,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti self.__fallback_hosts_use_default = fallback_hosts_use_default self.__fallback_retry_timeout = fallback_retry_timeout self.__disconnected_retry_timeout = disconnected_retry_timeout + self.__channel_retry_timeout = channel_retry_timeout self.__idempotent_rest_publishing = idempotent_rest_publishing self.__loop = loop self.__auto_connect = auto_connect @@ -217,6 +218,10 @@ def fallback_retry_timeout(self): def disconnected_retry_timeout(self): return self.__disconnected_retry_timeout + @property + def channel_retry_timeout(self): + return self.__channel_retry_timeout + @property def idempotent_rest_publishing(self): return self.__idempotent_rest_publishing From 931a9f2dd722acff792878f2d269fc5538a317c9 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 1 Feb 2023 20:13:04 +0000 Subject: [PATCH 373/481] docs: add docstring for channel_retry_timeout --- ably/realtime/realtime.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index a3987ef4..d1b02635 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -61,6 +61,10 @@ def __init__(self, key=None, loop=None, **kwargs): disconnected_retry_timeout: float If the connection is still in the DISCONNECTED state after this delay, the client library will attempt to reconnect automatically. The default is 15 seconds. + channel_retry_timeout: float + When a channel becomes SUSPENDED following a server initiated DETACHED, after this delay, if the + channel is still SUSPENDED and the connection is in CONNECTED, the client library will attempt to + re-attach the channel automatically. The default is 15 seconds. fallback_hosts: list[str] An array of fallback hosts to be used in the case of an error necessitating the use of an alternative host. If you have been provided a set of custom fallback hosts by Ably, please specify From 4a354e3644170c3242bb28ab41105e0f8ddcfb3f Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 1 Feb 2023 20:28:49 +0000 Subject: [PATCH 374/481] feat: retry immediately upon unexpected DETACHED --- ably/realtime/realtime_channel.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 1336bb35..d337af45 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -311,8 +311,10 @@ def _on_message(self, msg): elif action == ProtocolMessageAction.DETACHED: if self.state == ChannelState.DETACHING: self._notify_state(ChannelState.DETACHED) + elif self.state == ChannelState.ATTACHING: + self._notify_state(ChannelState.SUSPENDED) else: - log.warn("RealtimeChannel._on_message(): DETACHED recieved while not detaching") + self._request_state(ChannelState.ATTACHING) elif action == ProtocolMessageAction.MESSAGE: messages = Message.from_encoded_array(msg.get('messages')) for message in messages: From ea9aff6c030a902395979095e800cdb1defe603c Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 2 Feb 2023 13:33:56 +0000 Subject: [PATCH 375/481] refactor: add static AblyException.from_dict method --- ably/realtime/connectionmanager.py | 2 +- ably/realtime/realtime_channel.py | 2 +- ably/transport/websockettransport.py | 4 ++-- ably/util/exceptions.py | 4 ++++ 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index 3a1a9e15..e8a99156 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -146,7 +146,7 @@ def on_connected(self, connection_details: ConnectionDetails, connection_id: str def on_disconnected(self, msg: dict): error = msg.get("error") - exception = AblyException(error.get('message'), error.get('statusCode'), error.get('code')) + exception = AblyException.from_dict(error) self.notify_state(ConnectionState.DISCONNECTED, exception) if error: error_status_code = error.get("statusCode") diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 1336bb35..0c619d10 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -294,7 +294,7 @@ def _on_message(self, msg): resumed = False if error: - exception = AblyException(error.get('message'), error.get('statusCode'), error.get('code')) + exception = AblyException.from_dict(error) if flags: resumed = has_flag(flags, Flag.RESUMED) diff --git a/ably/transport/websockettransport.py b/ably/transport/websockettransport.py index 949e06b3..fccf94d4 100644 --- a/ably/transport/websockettransport.py +++ b/ably/transport/websockettransport.py @@ -101,7 +101,7 @@ async def on_protocol_message(self, msg): error = msg.get('error') exception = None if error: - exception = AblyException(error.get('message'), error.get('statusCode'), error.get('code')) + exception = AblyException.from_dict(error) max_idle_interval = connection_details.max_idle_interval if max_idle_interval: @@ -119,7 +119,7 @@ async def on_protocol_message(self, msg): await self.connection_manager.on_closed() elif action == ProtocolMessageAction.ERROR: error = msg.get('error') - exception = AblyException(error.get('message'), error.get('statusCode'), error.get('code')) + exception = AblyException.from_dict(error) await self.connection_manager.on_error(msg, exception) elif action == ProtocolMessageAction.HEARTBEAT: id = msg.get('id') diff --git a/ably/util/exceptions.py b/ably/util/exceptions.py index c2636801..c59ab5f5 100644 --- a/ably/util/exceptions.py +++ b/ably/util/exceptions.py @@ -63,6 +63,10 @@ def from_exception(e): return e return AblyException("Unexpected exception: %s" % e, 500, 50000) + @staticmethod + def from_dict(value: dict): + return AblyException(value.get('message'), value.get('statusCode'), value.get('code')) + def catch_all(func): @functools.wraps(func) From 14a6b4f070db236687a0e92d020f282fb2f1b7b6 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 1 Feb 2023 20:29:28 +0000 Subject: [PATCH 376/481] feat: implement channel retry behaviour --- ably/realtime/realtime_channel.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index d337af45..b7138428 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -47,6 +47,7 @@ def __init__(self, realtime, name): self.__state_timer: Timer | None = None self.__attach_resume = False self.__channel_serial: str | None = None + self.__retry_timer: Timer | None = None # Used to listen to state changes internally, if we use the public event emitter interface then internals # will be disrupted if the user called .off() to remove all listeners @@ -333,6 +334,11 @@ def _notify_state(self, state: ChannelState, reason=None, resumed=False): if state == self.state: return + if state == ChannelState.SUSPENDED and self.ably.connection.state == ConnectionState.CONNECTED: + self.__start_retry_timer() + else: + self.__cancel_retry_timer() + # RTL4j1 if state == ChannelState.ATTACHED: self.__attach_resume = True @@ -389,6 +395,23 @@ def __timeout_pending_state(self): else: self._check_pending_state() + def __start_retry_timer(self): + if self.__retry_timer: + return + + self.__retry_timer = Timer(self.ably.options.channel_retry_timeout, self.__on_retry_timer_expire) + + def __cancel_retry_timer(self): + if self.__retry_timer: + self.__retry_timer.cancel() + self.__retry_timer = None + + def __on_retry_timer_expire(self): + if self.state == ChannelState.SUSPENDED and self.ably.connection.state == ConnectionState.CONNECTED: + self.__retry_timer = None + log.info("RealtimeChannel retry timer expired, attempting a new attach") + self._request_state(ChannelState.ATTACHING) + # RTL23 @property def name(self): From 1c3fb49a29ed74aef36a884ec289c0225ae86135 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 1 Feb 2023 20:31:00 +0000 Subject: [PATCH 377/481] test: add test for channel retrying immediately on unexpected DETACHED --- test/ably/realtime/realtimechannel_test.py | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/ably/realtime/realtimechannel_test.py b/test/ably/realtime/realtimechannel_test.py index fe5cc7d3..4b271494 100644 --- a/test/ably/realtime/realtimechannel_test.py +++ b/test/ably/realtime/realtimechannel_test.py @@ -291,3 +291,26 @@ async def test_attach_while_connecting(self): await channel.attach() assert channel.state == ChannelState.ATTACHED await ably.close() + + # RTL13a + async def test_channel_attach_retry_immediately_on_unexpected_detached(self): + ably = await TestApp.get_ably_realtime(channel_retry_timeout=500) + channel_name = random_string(5) + channel = ably.channels.get(channel_name) + await channel.attach() + + # Simulate an unexpected DETACHED message from ably + message = { + "action": ProtocolMessageAction.DETACHED, + "channel": channel_name, + } + assert ably.connection.connection_manager.transport + await ably.connection.connection_manager.transport.on_protocol_message(message) + + # The channel should retry attachment immediately + assert channel.state == ChannelState.ATTACHING + + # Make sure the channel sucessfully re-attaches + await channel.once_async(ChannelState.ATTACHED) + + await ably.close() From e78aafdff0400a6a0b81898013dcafb14462bbf2 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 1 Feb 2023 20:36:42 +0000 Subject: [PATCH 378/481] test: add test for channel SUSPENDED on failed attach --- test/ably/realtime/realtimechannel_test.py | 29 ++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/ably/realtime/realtimechannel_test.py b/test/ably/realtime/realtimechannel_test.py index 4b271494..8933a85e 100644 --- a/test/ably/realtime/realtimechannel_test.py +++ b/test/ably/realtime/realtimechannel_test.py @@ -314,3 +314,32 @@ async def test_channel_attach_retry_immediately_on_unexpected_detached(self): await channel.once_async(ChannelState.ATTACHED) await ably.close() + + # RTL13b + async def test_channel_attach_retry_after_unsuccessful_attach(self): + ably = await TestApp.get_ably_realtime(channel_retry_timeout=500, realtime_request_timeout=1000) + channel_name = random_string(5) + channel = ably.channels.get(channel_name) + call_count = 0 + + original_send_protocol_message = ably.connection.connection_manager.send_protocol_message + + # Discard the first ATTACHED message recieved + async def new_send_protocol_message(msg): + nonlocal call_count + if call_count == 0 and msg.get('action') == ProtocolMessageAction.ATTACH: + call_count += 1 + return + await original_send_protocol_message(msg) + ably.connection.connection_manager.send_protocol_message = new_send_protocol_message + + with pytest.raises(AblyException): + await channel.attach() + + # The channel should become SUSPENDED but will still retry again after channel_retry_timeout + assert channel.state == ChannelState.SUSPENDED + + # Make sure the channel sucessfully re-attaches + await channel.once_async(ChannelState.ATTACHED) + + await ably.close() From 2f7bb35188116d252b20fc6a9409470853195524 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 6 Feb 2023 13:49:23 +0000 Subject: [PATCH 379/481] Merge `realtime-examples-update` into `release/2.0.0-beta.3` --- README.md | 53 +++++++++++++++++++++++---------- test/ably/rest/resthttp_test.py | 43 ++++++++++---------------- 2 files changed, 54 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 919b3331..585b71ee 100644 --- a/README.md +++ b/README.md @@ -197,16 +197,51 @@ await client.time() await client.close() ``` -### Using the Realtime API -The python realtime client currently only supports basic authentication. +## Realtime client (beta) + +We currently have a preview version of our first ever Python realtime client available for beta testing. +Currently the realtime client only supports authentication using basic auth and message subscription. +Realtime publishing, token authentication, and realtime presence are upcoming but not yet supported. +Check out the [roadmap](./roadmap.md) to see our plan for the realtime client. + +### Installing the realtime client + +The beta realtime client is available as a [PyPI](https://pypi.org/project/ably/2.0.0b3/) package. + +``` +pip install ably==2.0.0b3 +``` + +### Using the realtime client + #### Creating a client ```python from ably import AblyRealtime async def main(): + # Create a client using an Ably API key client = AblyRealtime('api:key') ``` +#### Subscribe to connection state changes + +```python +# subscribe to 'failed' connection state +client.connection.on('failed', listener) + +# subscribe to 'connected' connection state +client.connection.on('connected', listener) + +# subscribe to all connection state changes +client.connection.on(listener) + +# wait for the next state change +await client.connection.once_async() + +# wait for the connection to become connected +await client.connection.once_async('connected') +``` + #### Get a realtime channel instance ```python channel = client.channels.get('channel_name') @@ -234,18 +269,6 @@ channel.unsubscribe('event', listener) channel.unsubscribe() ``` -#### Subscribe to connection state change -```python -# subscribe to 'failed' connection state -client.connection.on('failed', listener) - -# subscribe to 'connected' connection state -client.connection.on('connected', listener) - -# subscribe to all connection state changes -client.connection.on(listener) -``` - #### Attach to a channel ```python await channel.attach() @@ -259,7 +282,7 @@ await channel.detach() ```python # Establish a realtime connection. # Explicitly calling connect() is unnecessary unless the autoConnect attribute of the ClientOptions object is false -await client.connect() +client.connect() # Close a connection await client.close() diff --git a/test/ably/rest/resthttp_test.py b/test/ably/rest/resthttp_test.py index ad1fe043..bab45344 100644 --- a/test/ably/rest/resthttp_test.py +++ b/test/ably/rest/resthttp_test.py @@ -78,26 +78,19 @@ def make_url(host): expected_hosts_set.remove(prep_request_tuple[0].headers.get('host')) await ably.close() - @pytest.mark.skip(reason="skipped due to httpx changes") + @respx.mock async def test_no_host_fallback_nor_retries_if_custom_host(self): custom_host = 'example.org' ably = AblyRest(token="foo", rest_host=custom_host) - custom_url = "%s://%s:%d/" % ( - ably.http.preferred_scheme, - custom_host, - ably.http.preferred_port) + mock_route = respx.get("https://example.org").mock(side_effect=httpx.RequestError('')) - with mock.patch('httpx.Request', wraps=httpx.Request) as request_mock: - with mock.patch('httpx.AsyncClient.send', side_effect=httpx.RequestError('')) as send_mock: - with pytest.raises(httpx.RequestError): - await ably.http.make_request('GET', '/', skip_auth=True) + with pytest.raises(httpx.RequestError): + await ably.http.make_request('GET', '/', skip_auth=True) + + assert mock_route.call_count == 1 + assert respx.calls.call_count == 1 - assert send_mock.call_count == 1 - assert request_mock.call_args == mock.call(mock.ANY, - custom_url, - content=mock.ANY, - headers=mock.ANY) await ably.close() # RSC15f @@ -137,7 +130,7 @@ async def side_effect(*args, **kwargs): await client.aclose() await ably.close() - @pytest.mark.skip(reason="skipped due to httpx changes") + @respx.mock async def test_no_retry_if_not_500_to_599_http_code(self): default_host = Options().get_rest_host() ably = AblyRest(token="foo") @@ -147,20 +140,16 @@ async def test_no_retry_if_not_500_to_599_http_code(self): default_host, ably.http.preferred_port) - def raise_ably_exception(*args, **kwargs): - raise AblyException(message="", status_code=600, code=50500) + mock_response = httpx.Response(600, json={'message': "", 'status_code': 600, 'code': 50500}) - with mock.patch('httpx.Request', wraps=httpx.Request) as request_mock: - with mock.patch('ably.util.exceptions.AblyException.raise_for_response', - side_effect=raise_ably_exception) as send_mock: - with pytest.raises(AblyException): - await ably.http.make_request('GET', '/', skip_auth=True) + mock_route = respx.get(default_url).mock(return_value=mock_response) + + with pytest.raises(AblyException): + await ably.http.make_request('GET', '/', skip_auth=True) + + assert mock_route.call_count == 1 + assert respx.calls.call_count == 1 - assert send_mock.call_count == 1 - assert request_mock.call_args == mock.call(mock.ANY, - default_url, - content=mock.ANY, - headers=mock.ANY) await ably.close() async def test_500_errors(self): From cf4c7ca8405b56692cd09f0023f1ec676aa6cf90 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 6 Feb 2023 14:22:40 +0000 Subject: [PATCH 380/481] chore: update changelog for 2.0.0-beta.3 release --- CHANGELOG.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6913dac0..3614d251 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,37 @@ # Change Log +## [v2.0.0-beta.3](https://github.com/ably/ably-python/tree/v2.0.0-beta.3) + +This new beta release of the ably-python realtime client implements a number of new features to improve the stability of realtime connections, allowing the client to reconnect during a temporary disconnection, use fallback hosts when necessary, and catch up on messages missed while the client was disconnected. + +[Full Changelog](https://github.com/ably/ably-python/compare/v2.0.0-beta.2...v2.0.0-beta.3) + +- Resend protocol messages for pending channels upon resume [\#347](https://github.com/ably/ably-python/issues/347) +- Attempt to resume connection when disconnected unexpectedly [\#346](https://github.com/ably/ably-python/issues/346) +- Handle `CONNECTED` messages once connected [\#345](https://github.com/ably/ably-python/issues/345) +- Implement `maxIdleInterval` [\#344](https://github.com/ably/ably-python/issues/344) +- Implement realtime connectivity check [\#343](https://github.com/ably/ably-python/issues/343) +- Use fallback realtime hosts when encountering an appropriate error [\#342](https://github.com/ably/ably-python/issues/342) +- Add `fallbackHosts` client option for realtime clients [\#341](https://github.com/ably/ably-python/issues/341) +- Implement `connectionStateTtl` [\#340](https://github.com/ably/ably-python/issues/340) +- Implement `disconnectedRetryTimeout` [\#339](https://github.com/ably/ably-python/issues/339) +- Handle recoverable connection opening errors [\#338](https://github.com/ably/ably-python/issues/338) +- Implement `channelRetryTimeout` [\#442](https://github.com/ably/ably-python/issues/436) +- Queue protocol messages when connection state is `CONNECTING` or `DISCONNECTED` [\#418](https://github.com/ably/ably-python/issues/418) +- Propagate connection interruptions to realtime channels [\#417](https://github.com/ably/ably-python/issues/417) +- Spec compliance: `Realtime.connect` should be sync [\#413](https://github.com/ably/ably-python/issues/413) +- Emit `update` event on additional `ATTACHED` message [\#386](https://github.com/ably/ably-python/issues/386) +- Set the `ATTACH_RESUME` flag on unclean attach [\#385](https://github.com/ably/ably-python/issues/385) +- Handle fatal resume error [\#384](https://github.com/ably/ably-python/issues/384) +- Handle invalid resume response [\#383](https://github.com/ably/ably-python/issues/383) +- Handle clean resume response [\#382](https://github.com/ably/ably-python/issues/382) +- Send resume query param when reconnecting within `connectionStateTtl` [\#381](https://github.com/ably/ably-python/issues/381) +- Immediately reattempt connection when unexpectedly disconnected [\#380](https://github.com/ably/ably-python/issues/380) +- Clear connection state when `connectionStateTtl` elapsed [\#379](https://github.com/ably/ably-python/issues/379) +- Refactor websocket async tasks into WebSocketTransport class [\#373](https://github.com/ably/ably-python/issues/373) +- Send version transport param [\#368](https://github.com/ably/ably-python/issues/368) +- Clear `Connection.error_reason` when `Connection.connect` is called [\#367](https://github.com/ably/ably-python/issues/367) + ## [v2.0.0-beta.2](https://github.com/ably/ably-python/tree/v2.0.0-beta.2) [Full Changelog](https://github.com/ably/ably-python/compare/v2.0.0-beta.1...v2.0.0-beta.2) From 93346ead630be253579f0f5179be245b1b3ab733 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 6 Feb 2023 14:23:31 +0000 Subject: [PATCH 381/481] chore: bump version for 2.0.0-beta.3 release --- ably/__init__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ably/__init__.py b/ably/__init__.py index 1d0d927c..88c0f542 100644 --- a/ably/__init__.py +++ b/ably/__init__.py @@ -15,4 +15,4 @@ logger.addHandler(logging.NullHandler()) api_version = '1.2' -lib_version = '2.0.0-beta.2' +lib_version = '2.0.0-beta.3' diff --git a/pyproject.toml b/pyproject.toml index b0044934..5d16edbe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "ably" -version = "2.0.0-beta.2" +version = "2.0.0-beta.3" description = "Python REST and Realtime client library SDK for Ably realtime messaging service" license = "Apache-2.0" authors = ["Ably <support@ably.com>"] From f909a887a884dcd34d0090c6ade2741c668cd382 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Thu, 2 Feb 2023 17:08:59 +0000 Subject: [PATCH 382/481] implement get auth params in rest --- ably/rest/auth.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ably/rest/auth.py b/ably/rest/auth.py index 7903ee13..4350f292 100644 --- a/ably/rest/auth.py +++ b/ably/rest/auth.py @@ -1,3 +1,4 @@ +import asyncio import base64 from datetime import timedelta import logging @@ -75,6 +76,17 @@ def __init__(self, ably, options): raise ValueError("Can't authenticate via token, must provide " "auth_callback, auth_url, key, token or a TokenDetail") + def get_auth_transport_param(self): + if self.__auth_mechanism == Auth.Method.BASIC: + key_name = self.__auth_options.key_name + key_secret = self.__auth_options.key_secret + return {"key": f"{key_name}:{key_secret}"} + elif self.__auth_mechanism == Auth.Method.TOKEN: + token_details = asyncio.create_task(self.__authorize_when_necessary()) + return {"accessToken": token_details} + else: + log.info("Auth mechanism not known or invalid") + async def __authorize_when_necessary(self, token_params=None, auth_options=None, force=False): self.__auth_mechanism = Auth.Method.TOKEN From 29079ce1a63dffbd6b8d4f611abfcfb79694e0ec Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Thu, 2 Feb 2023 17:42:04 +0000 Subject: [PATCH 383/481] update get_transport_param method --- ably/realtime/connectionmanager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index e8a99156..63be5561 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -51,7 +51,10 @@ def check_connection(self): def __get_transport_params(self): protocol_version = Defaults.protocol_version - params = {"key": self.__ably.key, "v": protocol_version} + params = {"v": protocol_version} + auth_params = self.ably.auth.get_auth_transport_param() + if 'key' in auth_params: + params["key"] = self.__ably.key if self.connection_details: params["resume"] = self.connection_details.connection_key return params From 5d9103758bc9ee29110f6e2cfb04f93b89207d7a Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 6 Feb 2023 11:41:40 +0000 Subject: [PATCH 384/481] refactor realtime constructor --- ably/realtime/connectionmanager.py | 12 +++++++++--- ably/realtime/realtime.py | 26 ++++--------------------- ably/rest/auth.py | 7 ++++--- test/ably/realtime/realtimeinit_test.py | 18 +++++++++++++++++ 4 files changed, 35 insertions(+), 28 deletions(-) diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index 63be5561..87c74111 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -49,12 +49,18 @@ def check_connection(self): except httpx.HTTPError: return False - def __get_transport_params(self): + async def __get_transport_params(self): protocol_version = Defaults.protocol_version params = {"v": protocol_version} - auth_params = self.ably.auth.get_auth_transport_param() + auth_params = await self.ably.auth.get_auth_transport_param() + + print(auth_params, "==") + if 'key' in auth_params: params["key"] = self.__ably.key + if 'accessToken' in auth_params: + token = auth_params["accessToken"] + params["accessToken"] = token if self.connection_details: params["resume"] = self.connection_details.connection_key return params @@ -251,7 +257,7 @@ async def connect_base(self): self.notify_state(self.__fail_state, reason=exception) async def try_host(self, host): - params = self.__get_transport_params() + params = await self.__get_transport_params() self.transport = WebSocketTransport(self, host, params) self._emit('transport.pending', self.transport) self.transport.connect() diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 59af2a5c..1ca1fa1b 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -6,6 +6,7 @@ from ably.types.options import Options from ably.rest.channel import Channels as RestChannels from ably.realtime.realtime_channel import ChannelState, RealtimeChannel +from ably.types.tokendetails import TokenDetails log = logging.getLogger(__name__) @@ -86,8 +87,6 @@ def __init__(self, key=None, loop=None, **kwargs): ValueError If no authentication key is not provided """ - # RTC1 - super().__init__(key, **kwargs) if loop is None: try: @@ -95,21 +94,15 @@ def __init__(self, key=None, loop=None, **kwargs): except RuntimeError: log.warning('Realtime client created outside event loop') - if key is not None: - options = Options(key=key, loop=loop, **kwargs) - else: - raise ValueError("Key is missing. Provide an API key.") - - log.info(f'Realtime client initialised with options: {vars(options)}') + # RTC1 + super().__init__(key, loop=loop, **kwargs) - self.__auth = Auth(self, options) - self.__options = options self.key = key self.__connection = Connection(self) self.__channels = Channels(self) # RTN3 - if options.auto_connect: + if self.options.auto_connect: self.connection.connection_manager.request_state(ConnectionState.CONNECTING, force=True) # RTC15 @@ -135,17 +128,6 @@ async def close(self): await self.connection.close() await super().close() - # RTC4 - @property - def auth(self): - """Returns the auth object""" - return self.__auth - - @property - def options(self): - """Returns the auth options object""" - return self.__options - # RTC2 @property def connection(self): diff --git a/ably/rest/auth.py b/ably/rest/auth.py index 4350f292..239e6c75 100644 --- a/ably/rest/auth.py +++ b/ably/rest/auth.py @@ -76,14 +76,15 @@ def __init__(self, ably, options): raise ValueError("Can't authenticate via token, must provide " "auth_callback, auth_url, key, token or a TokenDetail") - def get_auth_transport_param(self): + async def get_auth_transport_param(self): + print("called", self.__auth_mechanism) if self.__auth_mechanism == Auth.Method.BASIC: key_name = self.__auth_options.key_name key_secret = self.__auth_options.key_secret return {"key": f"{key_name}:{key_secret}"} elif self.__auth_mechanism == Auth.Method.TOKEN: - token_details = asyncio.create_task(self.__authorize_when_necessary()) - return {"accessToken": token_details} + token_details = await self.__authorize_when_necessary() + return {"accessToken": token_details.token} else: log.info("Auth mechanism not known or invalid") diff --git a/test/ably/realtime/realtimeinit_test.py b/test/ably/realtime/realtimeinit_test.py index 96fa540c..608bae55 100644 --- a/test/ably/realtime/realtimeinit_test.py +++ b/test/ably/realtime/realtimeinit_test.py @@ -32,7 +32,25 @@ async def test_init_without_autoconnect(self): ably = await TestApp.get_ably_realtime(auto_connect=False) assert ably.connection.state == ConnectionState.INITIALIZED ably.connect() + await ably.connection.once_async(ConnectionState.CONNECTED) assert ably.connection.state == ConnectionState.CONNECTED + print(await ably.connection.ping()) await ably.close() assert ably.connection.state == ConnectionState.CLOSED + + +class TestRealtimeAuth(BaseAsyncTestCase): + async def asyncSetUp(self): + self.test_vars = await TestApp.get_test_vars() + self.valid_key_format = "api:key" + + async def test_auth_with_token_str(self): + # ably = await TestApp.get_ably_realtime() + self.rest = await TestApp.get_ably_rest() + token = await self.rest.auth.request_token() + # print(token, "===") + print(token, "+++++") + ably = await TestApp.get_ably_realtime(token=token) + await ably.connect() + # print(await ably.connection.ping()) \ No newline at end of file From 137d3a8f78ee79815382c149d9eebb997fec9022 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 6 Feb 2023 12:31:37 +0000 Subject: [PATCH 385/481] update auth transport param --- ably/realtime/connectionmanager.py | 12 ++---------- ably/rest/auth.py | 3 --- ably/transport/websockettransport.py | 2 +- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index 87c74111..90fab5e1 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -51,16 +51,8 @@ def check_connection(self): async def __get_transport_params(self): protocol_version = Defaults.protocol_version - params = {"v": protocol_version} - auth_params = await self.ably.auth.get_auth_transport_param() - - print(auth_params, "==") - - if 'key' in auth_params: - params["key"] = self.__ably.key - if 'accessToken' in auth_params: - token = auth_params["accessToken"] - params["accessToken"] = token + params = await self.ably.auth.get_auth_transport_param() + params["v"] = protocol_version if self.connection_details: params["resume"] = self.connection_details.connection_key return params diff --git a/ably/rest/auth.py b/ably/rest/auth.py index 239e6c75..a5eac4d2 100644 --- a/ably/rest/auth.py +++ b/ably/rest/auth.py @@ -77,7 +77,6 @@ def __init__(self, ably, options): "auth_callback, auth_url, key, token or a TokenDetail") async def get_auth_transport_param(self): - print("called", self.__auth_mechanism) if self.__auth_mechanism == Auth.Method.BASIC: key_name = self.__auth_options.key_name key_secret = self.__auth_options.key_secret @@ -85,8 +84,6 @@ async def get_auth_transport_param(self): elif self.__auth_mechanism == Auth.Method.TOKEN: token_details = await self.__authorize_when_necessary() return {"accessToken": token_details.token} - else: - log.info("Auth mechanism not known or invalid") async def __authorize_when_necessary(self, token_params=None, auth_options=None, force=False): self.__auth_mechanism = Auth.Method.TOKEN diff --git a/ably/transport/websockettransport.py b/ably/transport/websockettransport.py index fccf94d4..8ab70b73 100644 --- a/ably/transport/websockettransport.py +++ b/ably/transport/websockettransport.py @@ -92,7 +92,7 @@ async def ws_connect(self, ws_url, headers): async def on_protocol_message(self, msg): self.on_activity() - log.info(f'WebSocketTransport.on_protocol_message(): receieved protocol message: {msg}') + log.info(f'WebSocketTransport.on_protocol_message(): received protocol message: {msg}') action = msg.get('action') if action == ProtocolMessageAction.CONNECTED: connection_id = msg.get('connectionId') From 7196233073e8bf90dea6c9d88311a6d49f8987e9 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 6 Feb 2023 12:34:47 +0000 Subject: [PATCH 386/481] refactor testapp to use token in realtime test --- test/ably/testapp.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/ably/testapp.py b/test/ably/testapp.py index 4fec4d56..9c88d942 100644 --- a/test/ably/testapp.py +++ b/test/ably/testapp.py @@ -85,8 +85,12 @@ async def get_ably_rest(**kw): @staticmethod async def get_ably_realtime(**kw): test_vars = await TestApp.get_test_vars() + options = TestApp.get_options(test_vars, **kw) + return AblyRealtime(**options) + + @staticmethod + def get_options(test_vars, **kwargs): options = { - 'key': test_vars["keys"][0]["key_str"], 'realtime_host': test_vars["realtime_host"], 'rest_host': test_vars["host"], 'port': test_vars["port"], @@ -94,8 +98,13 @@ async def get_ably_realtime(**kw): 'tls': test_vars["tls"], 'environment': test_vars["environment"], } - options.update(kw) - return AblyRealtime(**options) + auth_methods = ["auth_url", "auth_callback", "token", "token_details", "key"] + if not any(x in kwargs for x in auth_methods): + options["key"] = test_vars["keys"][0]["key_str"] + + options.update(kwargs) + return options + @staticmethod async def clear_test_vars(): From e96827590883e42be3dbbc264a196ec80f078129 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 6 Feb 2023 15:31:02 +0000 Subject: [PATCH 387/481] add test for token and token_details auth --- ably/realtime/realtime.py | 4 -- ably/rest/auth.py | 3 +- test/ably/realtime/realtimeauth_test.py | 63 +++++++++++++++++++++++ test/ably/realtime/realtimeinit_test.py | 18 ------- test/ably/realtime/realtimeresume_test.py | 7 ++- test/ably/testapp.py | 3 +- 6 files changed, 68 insertions(+), 30 deletions(-) create mode 100644 test/ably/realtime/realtimeauth_test.py diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 1ca1fa1b..413128cb 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -1,12 +1,8 @@ import logging import asyncio from ably.realtime.connection import Connection, ConnectionState -from ably.rest.auth import Auth from ably.rest.rest import AblyRest -from ably.types.options import Options -from ably.rest.channel import Channels as RestChannels from ably.realtime.realtime_channel import ChannelState, RealtimeChannel -from ably.types.tokendetails import TokenDetails log = logging.getLogger(__name__) diff --git a/ably/rest/auth.py b/ably/rest/auth.py index a5eac4d2..e34d221f 100644 --- a/ably/rest/auth.py +++ b/ably/rest/auth.py @@ -1,4 +1,3 @@ -import asyncio import base64 from datetime import timedelta import logging @@ -82,7 +81,7 @@ async def get_auth_transport_param(self): key_secret = self.__auth_options.key_secret return {"key": f"{key_name}:{key_secret}"} elif self.__auth_mechanism == Auth.Method.TOKEN: - token_details = await self.__authorize_when_necessary() + token_details = await self.__authorize_when_necessary() return {"accessToken": token_details.token} async def __authorize_when_necessary(self, token_params=None, auth_options=None, force=False): diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py new file mode 100644 index 00000000..f4590467 --- /dev/null +++ b/test/ably/realtime/realtimeauth_test.py @@ -0,0 +1,63 @@ +from ably.realtime.connection import ConnectionState +from ably.types.tokendetails import TokenDetails +from test.ably.testapp import TestApp +from test.ably.utils import BaseAsyncTestCase + + +class TestRealtimeAuth(BaseAsyncTestCase): + async def asyncSetUp(self): + self.test_vars = await TestApp.get_test_vars() + self.valid_key_format = "api:key" + + async def test_auth_valid_api_key(self): + ably = await TestApp.get_ably_realtime() + await ably.connection.once_async(ConnectionState.CONNECTED) + assert ably.connection.error_reason is None + response_time_ms = await ably.connection.ping() + assert response_time_ms is not None + await ably.close() + + async def test_auth_wrong_api_key(self): + api_key = "js9de7r:08sdnuvfasd" + ably = await TestApp.get_ably_realtime(key=api_key) + state_change = await ably.connection.once_async(ConnectionState.FAILED) + assert ably.connection.error_reason == state_change.reason + assert state_change.reason.code == 40005 + assert state_change.reason.status_code == 400 + await ably.close() + + async def test_auth_with_token_str(self): + self.rest = await TestApp.get_ably_rest() + token_details = await self.rest.auth.request_token() + ably = await TestApp.get_ably_realtime(token=token_details.token) + await ably.connection.once_async(ConnectionState.CONNECTED) + response_time_ms = await ably.connection.ping() + assert response_time_ms is not None + assert ably.connection.error_reason is None + await ably.close() + + async def test_auth_with_invalid_token_str(self): + invalid_token = "Sdnurv_some_invalid_token_nkds9r7" + ably = await TestApp.get_ably_realtime(token=invalid_token) + state_change = await ably.connection.once_async(ConnectionState.FAILED) + assert state_change.reason.code == 40005 + assert state_change.reason.status_code == 400 + await ably.close() + + async def test_auth_with_token_details(self): + self.rest = await TestApp.get_ably_rest() + token_details = await self.rest.auth.request_token() + ably = await TestApp.get_ably_realtime(token_details=token_details) + await ably.connection.once_async(ConnectionState.CONNECTED) + response_time_ms = await ably.connection.ping() + assert response_time_ms is not None + assert ably.connection.error_reason is None + await ably.close() + + async def test_auth_with_invalid_token_details(self): + invalid_token_details = TokenDetails(token="invalid-token") + ably = await TestApp.get_ably_realtime(token_details=invalid_token_details) + state_change = await ably.connection.once_async(ConnectionState.FAILED) + assert state_change.reason.code == 40005 + assert state_change.reason.status_code == 400 + await ably.close() diff --git a/test/ably/realtime/realtimeinit_test.py b/test/ably/realtime/realtimeinit_test.py index 608bae55..96fa540c 100644 --- a/test/ably/realtime/realtimeinit_test.py +++ b/test/ably/realtime/realtimeinit_test.py @@ -32,25 +32,7 @@ async def test_init_without_autoconnect(self): ably = await TestApp.get_ably_realtime(auto_connect=False) assert ably.connection.state == ConnectionState.INITIALIZED ably.connect() - await ably.connection.once_async(ConnectionState.CONNECTED) assert ably.connection.state == ConnectionState.CONNECTED - print(await ably.connection.ping()) await ably.close() assert ably.connection.state == ConnectionState.CLOSED - - -class TestRealtimeAuth(BaseAsyncTestCase): - async def asyncSetUp(self): - self.test_vars = await TestApp.get_test_vars() - self.valid_key_format = "api:key" - - async def test_auth_with_token_str(self): - # ably = await TestApp.get_ably_realtime() - self.rest = await TestApp.get_ably_rest() - token = await self.rest.auth.request_token() - # print(token, "===") - print(token, "+++++") - ably = await TestApp.get_ably_realtime(token=token) - await ably.connect() - # print(await ably.connection.ping()) \ No newline at end of file diff --git a/test/ably/realtime/realtimeresume_test.py b/test/ably/realtime/realtimeresume_test.py index 8ed8a3db..7d1a72e8 100644 --- a/test/ably/realtime/realtimeresume_test.py +++ b/test/ably/realtime/realtimeresume_test.py @@ -47,14 +47,13 @@ async def test_fatal_resume_error(self): ably = await TestApp.get_ably_realtime() await ably.connection.once_async(ConnectionState.CONNECTED) - key_name = ably.options.key_name - ably.key = f"{key_name}:wrong-secret" + ably.auth.auth_options.key_name = "wrong-key" await ably.connection.connection_manager.transport.dispose() ably.connection.connection_manager.notify_state(ConnectionState.DISCONNECTED) state_change = await ably.connection.once_async(ConnectionState.FAILED) - assert state_change.reason.code == 40101 - assert state_change.reason.status_code == 401 + assert state_change.reason.code == 40005 + assert state_change.reason.status_code == 400 await ably.close() # RTN15c7 - invalid resume response diff --git a/test/ably/testapp.py b/test/ably/testapp.py index 9c88d942..80bfe925 100644 --- a/test/ably/testapp.py +++ b/test/ably/testapp.py @@ -101,11 +101,10 @@ def get_options(test_vars, **kwargs): auth_methods = ["auth_url", "auth_callback", "token", "token_details", "key"] if not any(x in kwargs for x in auth_methods): options["key"] = test_vars["keys"][0]["key_str"] - + options.update(kwargs) return options - @staticmethod async def clear_test_vars(): test_vars = TestApp.__test_vars From dad956f9fe559728d6b52af21e6a4b9f0218f3d8 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 7 Feb 2023 13:20:33 +0000 Subject: [PATCH 388/481] add test for auth using auth_callback --- test/ably/realtime/realtimeauth_test.py | 35 +++++++++++++++++++------ 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index f4590467..e7f48cc6 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -5,10 +5,6 @@ class TestRealtimeAuth(BaseAsyncTestCase): - async def asyncSetUp(self): - self.test_vars = await TestApp.get_test_vars() - self.valid_key_format = "api:key" - async def test_auth_valid_api_key(self): ably = await TestApp.get_ably_realtime() await ably.connection.once_async(ConnectionState.CONNECTED) @@ -27,8 +23,8 @@ async def test_auth_wrong_api_key(self): await ably.close() async def test_auth_with_token_str(self): - self.rest = await TestApp.get_ably_rest() - token_details = await self.rest.auth.request_token() + rest = await TestApp.get_ably_rest() + token_details = await rest.auth.request_token() ably = await TestApp.get_ably_realtime(token=token_details.token) await ably.connection.once_async(ConnectionState.CONNECTED) response_time_ms = await ably.connection.ping() @@ -45,8 +41,8 @@ async def test_auth_with_invalid_token_str(self): await ably.close() async def test_auth_with_token_details(self): - self.rest = await TestApp.get_ably_rest() - token_details = await self.rest.auth.request_token() + rest = await TestApp.get_ably_rest() + token_details = await rest.auth.request_token() ably = await TestApp.get_ably_realtime(token_details=token_details) await ably.connection.once_async(ConnectionState.CONNECTED) response_time_ms = await ably.connection.ping() @@ -61,3 +57,26 @@ async def test_auth_with_invalid_token_details(self): assert state_change.reason.code == 40005 assert state_change.reason.status_code == 400 await ably.close() + + async def test_auth_with_auth_callback(self): + rest = await TestApp.get_ably_rest() + async def callback(params): + token = await rest.auth.create_token_request(token_params=params) + return token + + ably = await TestApp.get_ably_realtime(auth_callback=callback) + await ably.connection.once_async(ConnectionState.CONNECTED) + response_time_ms = await ably.connection.ping() + assert response_time_ms is not None + assert ably.connection.error_reason is None + await ably.close() + + async def test_auth_with_auth_callback_invalid_token(self): + async def callback(params): + return "invalid token" + + ably = await TestApp.get_ably_realtime(auth_callback=callback) + state_change = await ably.connection.once_async(ConnectionState.FAILED) + assert state_change.reason.code == 40005 + assert state_change.reason.status_code == 400 + await ably.close() From cf50dead240093481f72d985b2e144fd0a271dfc Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 7 Feb 2023 15:28:29 +0000 Subject: [PATCH 389/481] add test for auth with auth_url --- test/ably/realtime/realtimeauth_test.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index e7f48cc6..5c215c0c 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -1,3 +1,4 @@ +import json from ably.realtime.connection import ConnectionState from ably.types.tokendetails import TokenDetails from test.ably.testapp import TestApp @@ -60,6 +61,7 @@ async def test_auth_with_invalid_token_details(self): async def test_auth_with_auth_callback(self): rest = await TestApp.get_ably_rest() + async def callback(params): token = await rest.auth.create_token_request(token_params=params) return token @@ -80,3 +82,17 @@ async def callback(params): assert state_change.reason.code == 40005 assert state_change.reason.status_code == 400 await ably.close() + + async def test_auth_with_auth_url(self): + echo_url = 'https://echo.ably.io/' + rest = await TestApp.get_ably_rest() + token_details = await rest.auth.request_token() + token_details_json = json.dumps(token_details.to_dict()) + url_path = f"{echo_url}?type=json&body={token_details_json}" + + ably = await TestApp.get_ably_realtime(auth_url=url_path) + await ably.connection.once_async(ConnectionState.CONNECTED) + response_time_ms = await ably.connection.ping() + assert response_time_ms is not None + assert ably.connection.error_reason is None + await ably.close() From 4efedc9f638227e7b7e050ae167bb0dd9fc2c5fb Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 7 Feb 2023 15:46:54 +0000 Subject: [PATCH 390/481] update auth_callback test --- test/ably/realtime/realtimeauth_test.py | 38 +++++++++++++++++++++---- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index 5c215c0c..ebf6ba20 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -23,7 +23,7 @@ async def test_auth_wrong_api_key(self): assert state_change.reason.status_code == 400 await ably.close() - async def test_auth_with_token_str(self): + async def test_auth_with_token_string(self): rest = await TestApp.get_ably_rest() token_details = await rest.auth.request_token() ably = await TestApp.get_ably_realtime(token=token_details.token) @@ -33,7 +33,7 @@ async def test_auth_with_token_str(self): assert ably.connection.error_reason is None await ably.close() - async def test_auth_with_invalid_token_str(self): + async def test_auth_with_invalid_token_string(self): invalid_token = "Sdnurv_some_invalid_token_nkds9r7" ably = await TestApp.get_ably_realtime(token=invalid_token) state_change = await ably.connection.once_async(ConnectionState.FAILED) @@ -59,12 +59,40 @@ async def test_auth_with_invalid_token_details(self): assert state_change.reason.status_code == 400 await ably.close() - async def test_auth_with_auth_callback(self): + async def test_auth_with_auth_callback_with_token_request(self): rest = await TestApp.get_ably_rest() async def callback(params): - token = await rest.auth.create_token_request(token_params=params) - return token + token_details = await rest.auth.create_token_request(token_params=params) + return token_details + + ably = await TestApp.get_ably_realtime(auth_callback=callback) + await ably.connection.once_async(ConnectionState.CONNECTED) + response_time_ms = await ably.connection.ping() + assert response_time_ms is not None + assert ably.connection.error_reason is None + await ably.close() + + async def test_auth_with_auth_callback_token_with_details(self): + rest = await TestApp.get_ably_rest() + + async def callback(params): + token_details = await rest.auth.request_token(token_params=params) + return token_details + + ably = await TestApp.get_ably_realtime(auth_callback=callback) + await ably.connection.once_async(ConnectionState.CONNECTED) + response_time_ms = await ably.connection.ping() + assert response_time_ms is not None + assert ably.connection.error_reason is None + await ably.close() + + async def test_auth_with_auth_callback_with_token_string(self): + rest = await TestApp.get_ably_rest() + + async def callback(params): + token_details = await rest.auth.request_token(token_params=params) + return token_details.token ably = await TestApp.get_ably_realtime(auth_callback=callback) await ably.connection.once_async(ConnectionState.CONNECTED) From 0cc1f6e1d4bfe1875455fd9dfca3c9d31f90ea3f Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 7 Feb 2023 16:58:44 +0000 Subject: [PATCH 391/481] update auth_url test --- ably/realtime/realtime.py | 1 + test/ably/realtime/realtimeauth_test.py | 33 ++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 413128cb..6e093a6c 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -2,6 +2,7 @@ import asyncio from ably.realtime.connection import Connection, ConnectionState from ably.rest.rest import AblyRest +from ably.rest.channel import Channels as RestChannels from ably.realtime.realtime_channel import ChannelState, RealtimeChannel diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index ebf6ba20..c124155d 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -111,12 +111,12 @@ async def callback(params): assert state_change.reason.status_code == 400 await ably.close() - async def test_auth_with_auth_url(self): - echo_url = 'https://echo.ably.io/' + async def test_auth_with_auth_url_json(self): + echo_url = 'https://echo.ably.io' rest = await TestApp.get_ably_rest() token_details = await rest.auth.request_token() token_details_json = json.dumps(token_details.to_dict()) - url_path = f"{echo_url}?type=json&body={token_details_json}" + url_path = f"{echo_url}/?type=json&body={token_details_json}" ably = await TestApp.get_ably_realtime(auth_url=url_path) await ably.connection.once_async(ConnectionState.CONNECTED) @@ -124,3 +124,30 @@ async def test_auth_with_auth_url(self): assert response_time_ms is not None assert ably.connection.error_reason is None await ably.close() + + async def test_auth_with_auth_url_text_plain(self): + echo_url = 'https://echo.ably.io' + rest = await TestApp.get_ably_rest() + token_details = await rest.auth.request_token() + url_path = f"{echo_url}/?type=text&body={token_details.token}" + + ably = await TestApp.get_ably_realtime(auth_url=url_path) + await ably.connection.once_async(ConnectionState.CONNECTED) + response_time_ms = await ably.connection.ping() + assert response_time_ms is not None + assert ably.connection.error_reason is None + await ably.close() + + async def test_auth_with_auth_url_post(self): + echo_url = 'https://echo.ably.io' + rest = await TestApp.get_ably_rest() + token_details = await rest.auth.request_token() + url_path = f"{echo_url}/?type=json&" + + ably = await TestApp.get_ably_realtime(auth_url=url_path, auth_method='POST', + auth_params=token_details) + await ably.connection.once_async(ConnectionState.CONNECTED) + response_time_ms = await ably.connection.ping() + assert response_time_ms is not None + assert ably.connection.error_reason is None + await ably.close() From ed5abc2dcf12fff9e922ea6ac837b63a5cc6a4eb Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 7 Feb 2023 18:36:44 +0000 Subject: [PATCH 392/481] fix hanging test --- ably/rest/auth.py | 2 ++ ably/types/tokendetails.py | 5 ++++- test/ably/realtime/realtimeauth_test.py | 8 ++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ably/rest/auth.py b/ably/rest/auth.py index e34d221f..c3b3a5cd 100644 --- a/ably/rest/auth.py +++ b/ably/rest/auth.py @@ -352,6 +352,8 @@ async def token_request_from_auth_url(self, method, url, token_params, headers, body = {} params = dict(auth_params, **token_params) elif method == 'POST': + if isinstance(auth_params, TokenDetails): + auth_params = auth_params.to_dict() params = {} body = dict(auth_params, **token_params) diff --git a/ably/types/tokendetails.py b/ably/types/tokendetails.py index 63a1e8dc..f3b79e47 100644 --- a/ably/types/tokendetails.py +++ b/ably/types/tokendetails.py @@ -21,7 +21,10 @@ def __init__(self, token=None, expires=None, issued=0, self.__token = token self.__issued = issued if capability and isinstance(capability, str): - self.__capability = Capability(json.loads(capability)) + try: + self.__capability = Capability(json.loads(capability)) + except json.JSONDecodeError: + self.__capability = Capability(json.loads(capability.replace("'", '"'))) else: self.__capability = Capability(capability or {}) self.__client_id = client_id diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index c124155d..60a1031c 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -3,6 +3,9 @@ from ably.types.tokendetails import TokenDetails from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase +import urllib.parse + +echo_url = 'https://echo.ably.io' class TestRealtimeAuth(BaseAsyncTestCase): @@ -112,11 +115,10 @@ async def callback(params): await ably.close() async def test_auth_with_auth_url_json(self): - echo_url = 'https://echo.ably.io' rest = await TestApp.get_ably_rest() token_details = await rest.auth.request_token() token_details_json = json.dumps(token_details.to_dict()) - url_path = f"{echo_url}/?type=json&body={token_details_json}" + url_path = f"{echo_url}/?type=json&body={urllib.parse.quote_plus(token_details_json)}" ably = await TestApp.get_ably_realtime(auth_url=url_path) await ably.connection.once_async(ConnectionState.CONNECTED) @@ -126,7 +128,6 @@ async def test_auth_with_auth_url_json(self): await ably.close() async def test_auth_with_auth_url_text_plain(self): - echo_url = 'https://echo.ably.io' rest = await TestApp.get_ably_rest() token_details = await rest.auth.request_token() url_path = f"{echo_url}/?type=text&body={token_details.token}" @@ -139,7 +140,6 @@ async def test_auth_with_auth_url_text_plain(self): await ably.close() async def test_auth_with_auth_url_post(self): - echo_url = 'https://echo.ably.io' rest = await TestApp.get_ably_rest() token_details = await rest.auth.request_token() url_path = f"{echo_url}/?type=json&" From fa105455660813841889097511c96cf974e03408 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 13 Feb 2023 15:43:10 +0000 Subject: [PATCH 393/481] initialize channel from terminal state --- ably/realtime/connectionmanager.py | 3 +++ ably/realtime/realtime.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index 90fab5e1..30b9d787 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -200,6 +200,9 @@ def request_state(self, state: ConnectionState, force=False): if state == ConnectionState.CLOSING and self.__state == ConnectionState.CLOSED: return + if state == ConnectionState.CONNECTING and self.__state in (ConnectionState.CLOSED, ConnectionState.FAILED): + self.ably.channels._initialize_channels() + if not force: self.enact_state_change(state) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 6e093a6c..02add5a8 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -95,6 +95,8 @@ def __init__(self, key=None, loop=None, **kwargs): super().__init__(key, loop=loop, **kwargs) self.key = key + # print(self.auth) + # self.__auth = self.auth self.__connection = Connection(self) self.__channels = Channels(self) @@ -230,3 +232,8 @@ def _on_connected(self): asyncio.create_task(channel.attach()) elif channel.state == ChannelState.ATTACHED: channel._request_state(ChannelState.ATTACHING) + + def _initialize_channels(self): + for channel_name in self.__all: + channel = self.__all[channel_name] + channel._request_state(ChannelState.INITIALIZED) From 351f9e3a9cdeae595bbd96123daedef591e049af Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 13 Feb 2023 15:43:45 +0000 Subject: [PATCH 394/481] add test for channel initialize from terminal state --- ably/realtime/connectionmanager.py | 3 ++- test/ably/realtime/realtimechannel_test.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index 30b9d787..22e3a1e5 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -200,7 +200,8 @@ def request_state(self, state: ConnectionState, force=False): if state == ConnectionState.CLOSING and self.__state == ConnectionState.CLOSED: return - if state == ConnectionState.CONNECTING and self.__state in (ConnectionState.CLOSED, ConnectionState.FAILED): + if state == ConnectionState.CONNECTING and self.__state in (ConnectionState.CLOSED, + ConnectionState.FAILED): self.ably.channels._initialize_channels() if not force: diff --git a/test/ably/realtime/realtimechannel_test.py b/test/ably/realtime/realtimechannel_test.py index bebef88e..5f52927a 100644 --- a/test/ably/realtime/realtimechannel_test.py +++ b/test/ably/realtime/realtimechannel_test.py @@ -347,3 +347,13 @@ async def new_send_protocol_message(msg): await channel.once_async(ChannelState.ATTACHED) await ably.close() + + async def test_channel_initialized_on_connection_from_terminal_state(self): + ably = await TestApp.get_ably_realtime() + channel_name = random_string(5) + channel = ably.channels.get(channel_name) + await channel.attach() + await ably.close() + ably.connect() + assert channel.state == ChannelState.INITIALIZED + await ably.close() From 61993afa6f79786b877428055b0d6814aeae1a6c Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 13 Feb 2023 15:48:42 +0000 Subject: [PATCH 395/481] take out comment --- ably/realtime/realtime.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 02add5a8..8521dc8c 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -95,8 +95,6 @@ def __init__(self, key=None, loop=None, **kwargs): super().__init__(key, loop=loop, **kwargs) self.key = key - # print(self.auth) - # self.__auth = self.auth self.__connection = Connection(self) self.__channels = Channels(self) From b3d5f876f757d7a11aeff1973512143692bc0763 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 13 Feb 2023 16:02:50 +0000 Subject: [PATCH 396/481] feat: transition channels to failed upon error --- ably/realtime/connectionmanager.py | 2 ++ ably/realtime/realtime_channel.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index 22e3a1e5..17378595 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -166,6 +166,8 @@ async def on_error(self, msg: dict, exception: AblyException): if self.transport: await self.transport.dispose() raise exception + else: + self.on_channel_message(msg) async def on_closed(self): if self.transport: diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 78ab6b01..5e074824 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -320,6 +320,9 @@ def _on_message(self, msg): messages = Message.from_encoded_array(msg.get('messages')) for message in messages: self.__message_emitter._emit(message.name, message) + elif action == ProtocolMessageAction.ERROR: + error = AblyException.from_dict(msg.get('error')) + self._notify_state(ChannelState.FAILED, reason=error) def _request_state(self, state: ChannelState): log.info(f'RealtimeChannel._request_state(): state = {state}') From 2e5fe98d24b90a899a8e747c9edb3e4c1077bd59 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 13 Feb 2023 16:03:05 +0000 Subject: [PATCH 397/481] test: add test for channel error protocol message --- test/ably/realtime/realtimechannel_test.py | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/ably/realtime/realtimechannel_test.py b/test/ably/realtime/realtimechannel_test.py index 5f52927a..3dcd0dd2 100644 --- a/test/ably/realtime/realtimechannel_test.py +++ b/test/ably/realtime/realtimechannel_test.py @@ -357,3 +357,26 @@ async def test_channel_initialized_on_connection_from_terminal_state(self): ably.connect() assert channel.state == ChannelState.INITIALIZED await ably.close() + + async def test_channel_error(self): + ably = await TestApp.get_ably_realtime() + channel_name = random_string(5) + channel = ably.channels.get(channel_name) + await channel.attach() + code = 12345 + status_code = 123 + + msg = { + "action": ProtocolMessageAction.ERROR, + "channel": channel_name, + "error": { + "message": "test error", + "code": code, + "statusCode": status_code, + }, + } + + assert ably.connection.connection_manager.transport + await ably.connection.connection_manager.transport.on_protocol_message(msg) + + assert channel.state == ChannelState.FAILED From 083618149389ae84015671276cda7364e84a2e3f Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 13 Feb 2023 16:05:26 +0000 Subject: [PATCH 398/481] refactor: add `RealtimeChannel.error_reason` field --- ably/realtime/realtime_channel.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 5e074824..d2a61fe4 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -25,6 +25,8 @@ class RealtimeChannel(EventEmitter, Channel): Channel name state: str Channel state + error_reason: AblyException + An AblyException instance describing the last error which occurred on the channel, if any. Methods ------- @@ -48,6 +50,7 @@ def __init__(self, realtime, name): self.__attach_resume = False self.__channel_serial: str | None = None self.__retry_timer: Timer | None = None + self.__error_reason: AblyException | None = None # Used to listen to state changes internally, if we use the public event emitter interface then internals # will be disrupted if the user called .off() to remove all listeners @@ -430,3 +433,9 @@ def state(self): @state.setter def state(self, state: ChannelState): self.__state = state + + # RTL24 + @property + def error_reason(self): + """An AblyException instance describing the last error which occurred on the channel, if any.""" + return self.__error_reason From 4fb47faec1ad908df1ed9396378bf985c0f46107 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 13 Feb 2023 16:13:21 +0000 Subject: [PATCH 399/481] feat: implement `RealtimeChannel.error_reason` --- ably/realtime/realtime_channel.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index d2a61fe4..8a27a771 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -77,6 +77,8 @@ async def attach(self): if self.state == ChannelState.ATTACHED: return + self.__error_reason = None + # RTL4b if self.__realtime.connection.state not in [ ConnectionState.CONNECTING, @@ -340,6 +342,12 @@ def _notify_state(self, state: ChannelState, reason=None, resumed=False): if state == self.state: return + if reason is not None: + self.__error_reason = reason + + if state == ChannelState.INITIALIZED: + self.__error_reason = None + if state == ChannelState.SUSPENDED and self.ably.connection.state == ConnectionState.CONNECTED: self.__start_retry_timer() else: From 6ae7d97ab11f4110d9cf008773c15fba2f8273a5 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 13 Feb 2023 16:20:09 +0000 Subject: [PATCH 400/481] test: add tests for `RealtimeChannel.error_reason` behaviour --- test/ably/realtime/realtimechannel_test.py | 61 ++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/test/ably/realtime/realtimechannel_test.py b/test/ably/realtime/realtimechannel_test.py index 3dcd0dd2..aaf17e1f 100644 --- a/test/ably/realtime/realtimechannel_test.py +++ b/test/ably/realtime/realtimechannel_test.py @@ -380,3 +380,64 @@ async def test_channel_error(self): await ably.connection.connection_manager.transport.on_protocol_message(msg) assert channel.state == ChannelState.FAILED + assert channel.error_reason + assert channel.error_reason.code == code + assert channel.error_reason.status_code == status_code + + await ably.close() + + async def test_channel_error_cleared_upon_attach(self): + ably = await TestApp.get_ably_realtime() + channel_name = random_string(5) + channel = ably.channels.get(channel_name) + await channel.attach() + code = 12345 + status_code = 123 + + msg = { + "action": ProtocolMessageAction.ERROR, + "channel": channel_name, + "error": { + "message": "test error", + "code": code, + "statusCode": status_code, + }, + } + + assert ably.connection.connection_manager.transport + await ably.connection.connection_manager.transport.on_protocol_message(msg) + + assert channel.error_reason is not None + await channel.attach() + assert channel.error_reason is None + + await ably.close() + + async def test_channel_error_cleared_upon_connect_from_terminal_state(self): + ably = await TestApp.get_ably_realtime() + channel_name = random_string(5) + channel = ably.channels.get(channel_name) + await channel.attach() + code = 12345 + status_code = 123 + + msg = { + "action": ProtocolMessageAction.ERROR, + "channel": channel_name, + "error": { + "message": "test error", + "code": code, + "statusCode": status_code, + }, + } + + assert ably.connection.connection_manager.transport + await ably.connection.connection_manager.transport.on_protocol_message(msg) + + await ably.close() + + assert channel.error_reason is not None + ably.connect() + assert channel.error_reason is None + + await ably.close() From 6889e4b05e17cb8af8bc85319cff90d9f2fd8ce1 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 9 Feb 2023 15:31:26 +0000 Subject: [PATCH 401/481] refactor: add Rest._is_realtime property --- ably/realtime/realtime.py | 1 + ably/rest/rest.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 8521dc8c..55fc0c63 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -93,6 +93,7 @@ def __init__(self, key=None, loop=None, **kwargs): # RTC1 super().__init__(key, loop=loop, **kwargs) + self._is_realtime = True self.key = key self.__connection = Connection(self) diff --git a/ably/rest/rest.py b/ably/rest/rest.py index 235ff36a..79c7a960 100644 --- a/ably/rest/rest.py +++ b/ably/rest/rest.py @@ -60,6 +60,8 @@ def __init__(self, key=None, token=None, token_details=None, **kwargs): else: options = Options(**kwargs) + self._is_realtime = False + self.__http = Http(self, options) self.__auth = Auth(self, options) self.__http.auth = self.__auth From a3f1d34d0fa4ae9ae4aa6fb1c0481b1684eb6ed4 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 9 Feb 2023 17:24:37 +0000 Subject: [PATCH 402/481] refactor: add `ConnectionManager.get_state_error` --- ably/realtime/connectionmanager.py | 4 ++++ ably/types/connectionerrors.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 ably/types/connectionerrors.py diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index 17378595..2c6f710b 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -3,6 +3,7 @@ import httpx from ably.transport.websockettransport import WebSocketTransport, ProtocolMessageAction from ably.transport.defaults import Defaults +from ably.types.connectionerrors import ConnectionErrors from ably.types.connectionstate import ConnectionEvent, ConnectionState, ConnectionStateChange from ably.util.exceptions import AblyException from ably.util.eventemitter import EventEmitter @@ -49,6 +50,9 @@ def check_connection(self): except httpx.HTTPError: return False + def get_state_error(self): + return ConnectionErrors[self.state] + async def __get_transport_params(self): protocol_version = Defaults.protocol_version params = await self.ably.auth.get_auth_transport_param() diff --git a/ably/types/connectionerrors.py b/ably/types/connectionerrors.py new file mode 100644 index 00000000..bb2fa1f4 --- /dev/null +++ b/ably/types/connectionerrors.py @@ -0,0 +1,30 @@ +from ably.types.connectionstate import ConnectionState +from ably.util.exceptions import AblyException + +ConnectionErrors = { + ConnectionState.DISCONNECTED: AblyException( + 'Connection to server temporarily unavailable', + 400, + 80003, + ), + ConnectionState.SUSPENDED: AblyException( + 'Connection to server unavailable', + 400, + 80002, + ), + ConnectionState.FAILED: AblyException( + 'Connection failed or disconnected by server', + 400, + 80000, + ), + ConnectionState.CLOSING: AblyException( + 'Connection closing', + 400, + 80017, + ), + ConnectionState.CLOSED: AblyException( + 'Connection closed', + 400, + 80017, + ), +} From 2bf81fa1a63e3453c999e6d21f5876d2eb296726 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 9 Feb 2023 17:32:57 +0000 Subject: [PATCH 403/481] feat: reauth when authorize called from realtime client --- ably/realtime/connectionmanager.py | 50 +++++++++++++++++++++++++++- ably/rest/auth.py | 11 +++++- ably/transport/websockettransport.py | 1 + 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index 2c6f710b..7c46da0e 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -5,6 +5,7 @@ from ably.transport.defaults import Defaults from ably.types.connectionerrors import ConnectionErrors from ably.types.connectionstate import ConnectionEvent, ConnectionState, ConnectionStateChange +from ably.types.tokendetails import TokenDetails from ably.util.exceptions import AblyException from ably.util.eventemitter import EventEmitter from datetime import datetime @@ -270,7 +271,8 @@ def on_transport_connected(): log.info('ConnectionManager.try_a_host(): transport connected') if self.transport: self.transport.off('failed', on_transport_failed) - future.set_result(None) + if not future.done(): + future.set_result(None) async def on_transport_failed(exception): log.info('ConnectionManager.try_a_host(): transport failed') @@ -408,6 +410,52 @@ def disconnect_transport(self): if self.transport: self.disconnect_transport_task = asyncio.create_task(self.transport.dispose()) + async def on_auth_updated(self, token_details: TokenDetails): + log.info(f"ConnectionManager.on_auth_updated(): state = {self.state}") + if self.state == ConnectionState.CONNECTED: + auth_message = { + "action": ProtocolMessageAction.AUTH, + "auth": { + "accessToken": token_details.token + } + } + await self.send_protocol_message(auth_message) + + state_change = await self.once_async() + + if state_change.current == ConnectionState.CONNECTED: + return + elif state_change.current == ConnectionState.FAILED: + raise state_change.reason + elif self.state == ConnectionState.CONNECTING: + if self.connect_base_task and not self.connect_base_task.done(): + self.connect_base_task.cancel() + if self.transport: + await self.transport.dispose() + if self.state != ConnectionState.CONNECTED: + future = asyncio.Future() + + def on_state_change(state_change): + if state_change.current == ConnectionState.CONNECTED: + self.off('connectionstate', on_state_change) + future.set_result(token_details) + if state_change.current in ( + ConnectionState.CLOSED, + ConnectionState.FAILED, + ConnectionState.SUSPENDED + ): + self.off('connectionstate', on_state_change) + future.set_exception(state_change.reason or self.get_state_error()) + + self.on('connectionstate', on_state_change) + + if self.state == ConnectionState.CONNECTING: + self.start_connect() + else: + self.request_state(ConnectionState.CONNECTING) + + return await future + @property def ably(self): return self.__ably diff --git a/ably/rest/auth.py b/ably/rest/auth.py index c3b3a5cd..71cfa5a7 100644 --- a/ably/rest/auth.py +++ b/ably/rest/auth.py @@ -81,10 +81,18 @@ async def get_auth_transport_param(self): key_secret = self.__auth_options.key_secret return {"key": f"{key_name}:{key_secret}"} elif self.__auth_mechanism == Auth.Method.TOKEN: - token_details = await self.__authorize_when_necessary() + token_details = await self.__ensure_valid_auth_credentials() return {"accessToken": token_details.token} async def __authorize_when_necessary(self, token_params=None, auth_options=None, force=False): + token_details = await self.__ensure_valid_auth_credentials(token_params, auth_options, force) + + if self.ably._is_realtime: + await self.ably.connection.connection_manager.on_auth_updated(token_details) + + return token_details + + async def __ensure_valid_auth_credentials(self, token_params=None, auth_options=None, force=False): self.__auth_mechanism = Auth.Method.TOKEN if token_params is None: @@ -107,6 +115,7 @@ async def __authorize_when_necessary(self, token_params=None, auth_options=None, self.__token_details = await self.request_token(token_params, **auth_options) self._configure_client_id(self.__token_details.client_id) + return self.__token_details def token_details_has_expired(self): diff --git a/ably/transport/websockettransport.py b/ably/transport/websockettransport.py index 8ab70b73..47f7f51a 100644 --- a/ably/transport/websockettransport.py +++ b/ably/transport/websockettransport.py @@ -32,6 +32,7 @@ class ProtocolMessageAction(IntEnum): DETACH = 12 DETACHED = 13 MESSAGE = 15 + AUTH = 17 class WebSocketTransport(EventEmitter): From 0004a74af851c863c8d93419c1bad2293e7ca3d4 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 13 Feb 2023 12:50:28 +0000 Subject: [PATCH 404/481] test: add tests for reauth success before/after connection --- test/ably/realtime/realtimeauth_test.py | 71 +++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index 60a1031c..a22f8d02 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -1,5 +1,8 @@ +import asyncio import json from ably.realtime.connection import ConnectionState +from ably.transport.websockettransport import ProtocolMessageAction +from ably.types.connectionstate import ConnectionEvent from ably.types.tokendetails import TokenDetails from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase @@ -151,3 +154,71 @@ async def test_auth_with_auth_url_post(self): assert response_time_ms is not None assert ably.connection.error_reason is None await ably.close() + + async def test_reauth_while_connected(self): + rest = await TestApp.get_ably_rest() + + async def callback(params): + token_details = await rest.auth.request_token(token_params=params) + return token_details.token + + ably = await TestApp.get_ably_realtime(auth_callback=callback) + await ably.connection.once_async(ConnectionState.CONNECTED) + + assert ably.connection.connection_manager.transport + original_access_token = ably.connection.connection_manager.transport.params.get('accessToken') + assert original_access_token is not None + + original_send_protocol_message = ably.connection.connection_manager.send_protocol_message + fut1 = asyncio.Future() + + async def send_protocol_message(protocol_message): + if protocol_message.get('action') == ProtocolMessageAction.AUTH: + fut1.set_result(protocol_message) + await original_send_protocol_message(protocol_message) + ably.connection.connection_manager.send_protocol_message = send_protocol_message + + fut2 = asyncio.Future() + + def on_update(state_change): + fut2.set_result(state_change) + + ably.connection.on(ConnectionEvent.UPDATE, on_update) + + await ably.auth.authorize() + message = await fut1 + new_access_token = message.get('auth').get('accessToken') + assert new_access_token is not None + assert new_access_token is not original_access_token + + state_change = await fut2 + assert state_change.current == ConnectionState.CONNECTED + await ably.close() + + async def test_reauth_while_connecting(self): + rest = await TestApp.get_ably_rest() + + async def callback(params): + token_details = await rest.auth.request_token(token_params=params) + return token_details.token + + ably = await TestApp.get_ably_realtime(auth_callback=callback) + original_transport = await ably.connection.connection_manager.once_async('transport.pending') + await ably.auth.authorize() + assert ably.connection.state == ConnectionState.CONNECTED + assert ably.connection.connection_manager.transport is not original_transport + + await ably.close() + + async def test_reauth_immediately(self): + rest = await TestApp.get_ably_rest() + + async def callback(params): + token_details = await rest.auth.request_token(token_params=params) + return token_details.token + + ably = await TestApp.get_ably_realtime(auth_callback=callback) + await ably.auth.authorize() + assert ably.connection.state == ConnectionState.CONNECTED + + await ably.close() From 207ff3d898e994bb2e10625768d65ddd1e1dc822 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 13 Feb 2023 17:10:10 +0000 Subject: [PATCH 405/481] test: add tests for capability changes while connected --- test/ably/realtime/realtimeauth_test.py | 55 ++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index a22f8d02..c862e016 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -2,10 +2,11 @@ import json from ably.realtime.connection import ConnectionState from ably.transport.websockettransport import ProtocolMessageAction +from ably.types.channelstate import ChannelState from ably.types.connectionstate import ConnectionEvent from ably.types.tokendetails import TokenDetails from test.ably.testapp import TestApp -from test.ably.utils import BaseAsyncTestCase +from test.ably.utils import BaseAsyncTestCase, random_string import urllib.parse echo_url = 'https://echo.ably.io' @@ -222,3 +223,55 @@ async def callback(params): assert ably.connection.state == ConnectionState.CONNECTED await ably.close() + + async def test_capability_change_without_loss_of_continuity(self): + rest = await TestApp.get_ably_rest() + channel_name = random_string(5) + + async def callback(params): + token_details = await rest.auth.request_token(token_params=params) + return token_details.token + + ably = await TestApp.get_ably_realtime(auth_callback=callback) + + await ably.auth.authorize({"capability": {channel_name: "*"}}) + + channel = ably.channels.get(channel_name) + await channel.attach() + + await ably.auth.authorize({"capability": {channel_name: "*", random_string(5): "*"}}) + await channel.once_async(ChannelState.ATTACHED) + + await ably.close() + + async def test_capability_downgrade(self): + rest = await TestApp.get_ably_rest() + channel_name = random_string(5) + + async def callback(params): + token_details = await rest.auth.request_token(token_params=params) + return token_details.token + + ably = await TestApp.get_ably_realtime(auth_callback=callback) + + await ably.auth.authorize({"capability": {channel_name: "*"}}) + + channel = ably.channels.get(channel_name) + await channel.attach() + + future = asyncio.Future() + + def on_channel_state_change(state_change): + future.set_result(state_change) + + channel.on(ChannelState.FAILED, on_channel_state_change) + + await ably.auth.authorize({"capability": {random_string(5): "*"}}) + + state_change = await future + + assert state_change.reason is not None + assert state_change.reason.code == 40160 + assert state_change.reason.status_code == 401 + + await ably.close() From 5fc185e328b6f0f24b9e55d50d74a130ae596452 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 14 Feb 2023 12:33:22 +0000 Subject: [PATCH 406/481] implement reauth on inbound auth protocol msg --- ably/rest/auth.py | 1 - ably/transport/websockettransport.py | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ably/rest/auth.py b/ably/rest/auth.py index 71cfa5a7..39df90be 100644 --- a/ably/rest/auth.py +++ b/ably/rest/auth.py @@ -94,7 +94,6 @@ async def __authorize_when_necessary(self, token_params=None, auth_options=None, async def __ensure_valid_auth_credentials(self, token_params=None, auth_options=None, force=False): self.__auth_mechanism = Auth.Method.TOKEN - if token_params is None: token_params = dict(self.auth_options.default_token_params) else: diff --git a/ably/transport/websockettransport.py b/ably/transport/websockettransport.py index 47f7f51a..200db8b1 100644 --- a/ably/transport/websockettransport.py +++ b/ably/transport/websockettransport.py @@ -114,6 +114,12 @@ async def on_protocol_message(self, msg): self.connection_manager.on_connected(connection_details, connection_id, reason=exception) elif action == ProtocolMessageAction.DISCONNECTED: self.connection_manager.on_disconnected(msg) + elif action == ProtocolMessageAction.AUTH: + try: + await self.connection_manager.ably.auth.authorize() + except Exception as exc: + log.exception(f"WebSocketTransport.on_protocol_message(): An exception \ + occurred during reauth: {exc}") elif action == ProtocolMessageAction.CLOSED: if self.ws_connect_task: self.ws_connect_task.cancel() From fc25447e00c8de4f0538fbbbfb0011dcc905958f Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 14 Feb 2023 12:33:43 +0000 Subject: [PATCH 407/481] add test for inbound auth msg --- test/ably/realtime/realtimeauth_test.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index c862e016..78110f17 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -275,3 +275,26 @@ def on_channel_state_change(state_change): assert state_change.reason.status_code == 401 await ably.close() + + async def test_reauth_inbound_auth_protocol_msg(self): + rest = await TestApp.get_ably_rest() + + async def callback(params): + token_details = await rest.auth.request_token(token_params=params) + return token_details.token + + ably = await TestApp.get_ably_realtime(auth_callback=callback) + msg = { + "action": ProtocolMessageAction.AUTH, + } + + await ably.connection.once_async(ConnectionState.CONNECTED) + auth_future = asyncio.Future() + + def on_update(state_change): + auth_future.set_result(state_change) + + ably.connection.on("update", on_update) + await ably.connection.connection_manager.transport.on_protocol_message(msg) + await auth_future + await ably.close() From 32cd13ac512664e297e562c2182ee0e71368c13e Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 14 Feb 2023 13:14:05 +0000 Subject: [PATCH 408/481] fix: don't block ws_read_loop on handling inbound messages --- ably/transport/websockettransport.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ably/transport/websockettransport.py b/ably/transport/websockettransport.py index 200db8b1..4dbcbe60 100644 --- a/ably/transport/websockettransport.py +++ b/ably/transport/websockettransport.py @@ -146,10 +146,19 @@ async def ws_read_loop(self): except ConnectionClosedOK: break msg = json.loads(raw) - await self.on_protocol_message(msg) + task = asyncio.create_task(self.on_protocol_message(msg)) + task.add_done_callback(self.on_protcol_message_handled) else: raise Exception('ws_read_loop running with no websocket') + def on_protcol_message_handled(self, task): + try: + exception = task.exception() + except Exception as e: + exception = e + if exception is not None: + log.exception(f"WebSocketTransport.on_protocol_message_handled(): uncaught exception: {exception}") + def on_read_loop_done(self, task: asyncio.Task): try: exception = task.exception() From 64140eff1e9fb8f35779e3989d71a6a8fdea678c Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 14 Feb 2023 13:15:37 +0000 Subject: [PATCH 409/481] test: add test for jwt reauth --- test/ably/realtime/realtimeauth_test.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index 78110f17..cc0efdeb 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -1,5 +1,7 @@ import asyncio import json + +import httpx from ably.realtime.connection import ConnectionState from ably.transport.websockettransport import ProtocolMessageAction from ably.types.channelstate import ChannelState @@ -298,3 +300,26 @@ def on_update(state_change): await ably.connection.connection_manager.transport.on_protocol_message(msg) await auth_future await ably.close() + + # RSC8a4 + async def test_jwt_reauth(self): + test_vars = await TestApp.get_test_vars() + key = test_vars["keys"][0] + key_name = key["key_name"] + key_secret = key["key_secret"] + + async def auth_callback(_): + response = httpx.get( + echo_url + '/createJWT', + params={"keyName": key_name, "keySecret": key_secret, "expiresIn": 35} + ) + return response.text + + ably = await TestApp.get_ably_realtime(auth_callback=auth_callback) + + await ably.connection.once_async(ConnectionState.CONNECTED) + original_token_details = ably.auth.token_details + await ably.connection.once_async(ConnectionEvent.UPDATE) + assert ably.auth.token_details is not original_token_details + + await ably.close() From 7f30ef50680e708814f4e91d280220d521548095 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Tue, 14 Feb 2023 16:45:03 +0000 Subject: [PATCH 410/481] refactor(WebSocketTransport): simplify `ws_read_loop` This appears to be the recommended way to do it from the websockets documentation (and it's a bit more readable IMO) --- ably/transport/websockettransport.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/ably/transport/websockettransport.py b/ably/transport/websockettransport.py index 4dbcbe60..6ef27c7f 100644 --- a/ably/transport/websockettransport.py +++ b/ably/transport/websockettransport.py @@ -139,17 +139,15 @@ async def on_protocol_message(self, msg): self.connection_manager.on_channel_message(msg) async def ws_read_loop(self): - while True: - if self.websocket is not None: - try: - raw = await self.websocket.recv() - except ConnectionClosedOK: - break + if not self.websocket: + raise AblyException('ws_read_loop started with no websocket', 500, 50000) + try: + async for raw in self.websocket: msg = json.loads(raw) task = asyncio.create_task(self.on_protocol_message(msg)) task.add_done_callback(self.on_protcol_message_handled) - else: - raise Exception('ws_read_loop running with no websocket') + except ConnectionClosedOK: + return def on_protcol_message_handled(self, task): try: From b769d2fd95b80bda2b83be5cc63bce68024185e5 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 15 Feb 2023 20:29:14 +0000 Subject: [PATCH 411/481] handle connection failure on token error --- ably/realtime/connectionmanager.py | 35 ++++++++++++++++++++++++------ ably/util/helper.py | 4 ++++ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index 7c46da0e..901c8035 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -9,7 +9,7 @@ from ably.util.exceptions import AblyException from ably.util.eventemitter import EventEmitter from datetime import datetime -from ably.util.helper import get_random_id, Timer +from ably.util.helper import get_random_id, Timer, is_token_error from typing import Optional from ably.types.connectiondetails import ConnectionDetails from queue import Queue @@ -35,12 +35,14 @@ def __init__(self, realtime, initial_state): self.disconnect_transport_task: asyncio.Task | None = None self.__fallback_hosts = self.options.get_fallback_realtime_hosts() self.queued_messages = Queue() + self.__error_reason = None super().__init__() def enact_state_change(self, state, reason=None): current_state = self.__state log.info(f'ConnectionManager.enact_state_change(): {current_state} -> {state}') self.__state = state + self.__error_reason = reason self._emit('connectionstate', ConnectionStateChange(current_state, state, state, reason)) def check_connection(self): @@ -166,13 +168,32 @@ def on_disconnected(self, msg: dict): log.info("No fallback host to try for disconnected protocol message") async def on_error(self, msg: dict, exception: AblyException): - if msg.get('channel') is None: # RTN15i - self.enact_state_change(ConnectionState.FAILED, exception) - if self.transport: - await self.transport.dispose() - raise exception + error = msg.get('error') + code = error.get('code') + if is_token_error(code) and msg.get('channel') is None: + if isinstance(self.__error_reason, AblyException): + previous_error_code = self.__error_reason.code + if not is_token_error(previous_error_code): + try: + await self.ably.auth.authorize() + except Exception as e: + log.exception(f"Attempt to renew token fails: {e}") + self.notify_state(ConnectionState.DISCONNECTED, e) + return + self.notify_state(ConnectionState.DISCONNECTED, AblyException.from_dict(error)) + return + await self.ably.auth.authorize() + elif code == 40171: + log.info(f"No means to renew authentication token: {error}") + self.notify_state(ConnectionState.FAILED, AblyException.from_dict(error)) else: - self.on_channel_message(msg) + if msg.get('channel') is None: # RTN15i + self.enact_state_change(ConnectionState.FAILED, exception) + if self.transport: + await self.transport.dispose() + raise exception + else: + self.on_channel_message(msg) async def on_closed(self): if self.transport: diff --git a/ably/util/helper.py b/ably/util/helper.py index e221d1b8..d45e39b5 100644 --- a/ably/util/helper.py +++ b/ably/util/helper.py @@ -21,6 +21,10 @@ def unix_time_ms(): return round(time.time_ns() / 1_000_000) +def is_token_error(code): + return 40140 <= code < 40150 + + class Timer: def __init__(self, timeout: float, callback: Callable): self._timeout = timeout From e8b27c6929df52a30d5d52a1b1e8d5e1e8f027a0 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 15 Feb 2023 20:43:18 +0000 Subject: [PATCH 412/481] add test for single attempt token reauth --- test/ably/realtime/realtimeauth_test.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index cc0efdeb..837a0786 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -323,3 +323,25 @@ async def auth_callback(_): assert ably.auth.token_details is not original_token_details await ably.close() + + async def test_renew_token_single_attempt(self): + rest = await TestApp.get_ably_rest() + + async def callback(params): + token_details = await rest.auth.request_token(token_params=params) + return token_details.token + + ably = await TestApp.get_ably_realtime(auth_callback=callback) + msg = { + "action": ProtocolMessageAction.ERROR, + "error": { + "code": 40142, + "statusCode": 401 + } + } + + await ably.connection.once_async(ConnectionState.CONNECTED) + original_token_details = ably.auth.token_details + await ably.connection.connection_manager.transport.on_protocol_message(msg) + assert ably.auth.token_details is not original_token_details + await ably.close() From 13586ab795dedc864678f4cd9d3297b8996c28d9 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 15 Feb 2023 20:48:33 +0000 Subject: [PATCH 413/481] test new token request fail --- ably/realtime/connectionmanager.py | 2 ++ test/ably/realtime/realtimeauth_test.py | 28 +++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index 901c8035..6ed8fca9 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -170,6 +170,7 @@ def on_disconnected(self, msg: dict): async def on_error(self, msg: dict, exception: AblyException): error = msg.get('error') code = error.get('code') + #RTN14b if is_token_error(code) and msg.get('channel') is None: if isinstance(self.__error_reason, AblyException): previous_error_code = self.__error_reason.code @@ -183,6 +184,7 @@ async def on_error(self, msg: dict, exception: AblyException): self.notify_state(ConnectionState.DISCONNECTED, AblyException.from_dict(error)) return await self.ably.auth.authorize() + #RSA4a elif code == 40171: log.info(f"No means to renew authentication token: {error}") self.notify_state(ConnectionState.FAILED, AblyException.from_dict(error)) diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index 837a0786..6f863372 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -7,6 +7,7 @@ from ably.types.channelstate import ChannelState from ably.types.connectionstate import ConnectionEvent from ably.types.tokendetails import TokenDetails +from ably.util.exceptions import AblyException from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, random_string import urllib.parse @@ -324,6 +325,7 @@ async def auth_callback(_): await ably.close() + #RTN14b async def test_renew_token_single_attempt(self): rest = await TestApp.get_ably_rest() @@ -345,3 +347,29 @@ async def callback(params): await ably.connection.connection_manager.transport.on_protocol_message(msg) assert ably.auth.token_details is not original_token_details await ably.close() + + #RTN14b + async def test_renew_token_connection_attempt_fails(self): + rest = await TestApp.get_ably_rest() + + async def callback(params): + token_details = await rest.auth.request_token(token_params=params) + return token_details.token + + ably = await TestApp.get_ably_realtime(auth_callback=callback) + msg = { + "action": ProtocolMessageAction.ERROR, + "error": { + "code": 40142, + "statusCode": 401 + } + } + + await ably.connection.once_async(ConnectionState.CONNECTED) + ably.connection.connection_manager.enact_state_change(ConnectionState.DISCONNECTED, AblyException("oks", 401, 40143)) + await ably.connection.connection_manager.transport.on_protocol_message(msg) + state_change = await ably.connection.once_async(ConnectionState.DISCONNECTED) + assert ably.connection.error_reason == state_change.reason + assert state_change.reason.code == 40143 + assert state_change.reason.status_code == 401 + await ably.close() From 71aa6d882fa0c720f89798a3af4ae2eec98b00eb Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 15 Feb 2023 20:54:59 +0000 Subject: [PATCH 414/481] test renew token with no means to renew --- ably/realtime/connectionmanager.py | 4 +-- test/ably/realtime/realtimeauth_test.py | 33 ++++++++++++++++++++++--- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index 6ed8fca9..d1a0062c 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -170,7 +170,7 @@ def on_disconnected(self, msg: dict): async def on_error(self, msg: dict, exception: AblyException): error = msg.get('error') code = error.get('code') - #RTN14b + # RTN14b if is_token_error(code) and msg.get('channel') is None: if isinstance(self.__error_reason, AblyException): previous_error_code = self.__error_reason.code @@ -184,7 +184,7 @@ async def on_error(self, msg: dict, exception: AblyException): self.notify_state(ConnectionState.DISCONNECTED, AblyException.from_dict(error)) return await self.ably.auth.authorize() - #RSA4a + # RSA4a elif code == 40171: log.info(f"No means to renew authentication token: {error}") self.notify_state(ConnectionState.FAILED, AblyException.from_dict(error)) diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index 6f863372..88357f42 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -325,7 +325,7 @@ async def auth_callback(_): await ably.close() - #RTN14b + # RTN14b async def test_renew_token_single_attempt(self): rest = await TestApp.get_ably_rest() @@ -348,7 +348,7 @@ async def callback(params): assert ably.auth.token_details is not original_token_details await ably.close() - #RTN14b + # RTN14b async def test_renew_token_connection_attempt_fails(self): rest = await TestApp.get_ably_rest() @@ -366,10 +366,37 @@ async def callback(params): } await ably.connection.once_async(ConnectionState.CONNECTED) - ably.connection.connection_manager.enact_state_change(ConnectionState.DISCONNECTED, AblyException("oks", 401, 40143)) + ably.connection.connection_manager.enact_state_change(ConnectionState.DISCONNECTED, + AblyException("token error", 401, 40143)) await ably.connection.connection_manager.transport.on_protocol_message(msg) state_change = await ably.connection.once_async(ConnectionState.DISCONNECTED) assert ably.connection.error_reason == state_change.reason assert state_change.reason.code == 40143 assert state_change.reason.status_code == 401 await ably.close() + + # RSA4a + async def test_renew_token_no_renew_means_provided(self): + rest = await TestApp.get_ably_rest() + + async def callback(params): + token_details = await rest.auth.request_token(token_params=params) + return token_details.token + + ably = await TestApp.get_ably_realtime(auth_callback=callback) + msg = { + "action": ProtocolMessageAction.ERROR, + "error": { + "code": 40171, + "statusCode": 401 + } + } + + await ably.connection.once_async(ConnectionState.CONNECTED) + await ably.connection.connection_manager.transport.on_protocol_message(msg) + state_change = await ably.connection.once_async(ConnectionState.FAILED) + assert state_change.current == ConnectionState.FAILED + assert ably.connection.error_reason == state_change.reason + assert state_change.reason.code == 40171 + assert state_change.reason.status_code == 401 + await ably.close() From 18a11d52d9a02a47cb2fb12aac835bc634e1cdd0 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 20 Feb 2023 16:55:35 +0000 Subject: [PATCH 415/481] refactor renew token --- ably/http/http.py | 5 ++- ably/realtime/connection.py | 3 +- ably/realtime/connectionmanager.py | 68 ++++++++++++++++-------------- ably/rest/auth.py | 11 +++-- ably/util/helper.py | 4 +- 5 files changed, 50 insertions(+), 41 deletions(-) diff --git a/ably/http/http.py b/ably/http/http.py index d53b540f..3d45b068 100644 --- a/ably/http/http.py +++ b/ably/http/http.py @@ -11,6 +11,7 @@ from ably.http.httputils import HttpUtils from ably.transport.defaults import Defaults from ably.util.exceptions import AblyException, AblyAuthException +from ably.util.helper import is_token_error log = logging.getLogger(__name__) @@ -33,7 +34,7 @@ async def wrapper(rest, *args, **kwargs): try: return await func(rest, *args, **kwargs) except AblyException as e: - if 40140 <= e.code < 40150 and not retried: + if is_token_error(e) and not retried: await rest.reauth() return await func(rest, *args, **kwargs) @@ -138,7 +139,7 @@ async def reauth(self): try: await self.auth.authorize() except AblyAuthException as e: - if e.code == 40101: + if e.code == 40171: e.message = ("The provided token is not renewable and there is" " no means to generate a new token") raise e diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index bf473597..93f59462 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -80,7 +80,8 @@ async def ping(self): def _on_state_update(self, state_change): log.info(f'Connection state changing from {self.state} to {state_change.current}') self.__state = state_change.current - self.__error_reason = state_change.reason + if state_change.reason is not None: + self.__error_reason = state_change.reason self.__realtime.options.loop.call_soon(functools.partial(self._emit, state_change.current, state_change)) def _on_connection_update(self, state_change): diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index d1a0062c..827613a1 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -24,25 +24,26 @@ def __init__(self, realtime, initial_state): self.__state = initial_state self.__ping_future = None self.__timeout_in_secs = self.options.realtime_request_timeout / 1000 - self.transport: WebSocketTransport | None = None + self.transport: WebSocketTransport or None = None self.__connection_details = None self.connection_id = None self.__fail_state = ConnectionState.DISCONNECTED - self.transition_timer: Timer | None = None - self.suspend_timer: Timer | None = None - self.retry_timer: Timer | None = None - self.connect_base_task: asyncio.Task | None = None - self.disconnect_transport_task: asyncio.Task | None = None + self.transition_timer: Timer or None = None + self.suspend_timer: Timer or None = None + self.retry_timer: Timer or None = None + self.connect_base_task: asyncio.Task or None = None + self.disconnect_transport_task: asyncio.Task or None = None self.__fallback_hosts = self.options.get_fallback_realtime_hosts() self.queued_messages = Queue() - self.__error_reason = None + self.__error_reason: AblyException or None = None super().__init__() def enact_state_change(self, state, reason=None): current_state = self.__state log.info(f'ConnectionManager.enact_state_change(): {current_state} -> {state}') self.__state = state - self.__error_reason = reason + if reason: + self.__error_reason = reason self._emit('connectionstate', ConnectionStateChange(current_state, state, state, reason)) def check_connection(self): @@ -168,34 +169,37 @@ def on_disconnected(self, msg: dict): log.info("No fallback host to try for disconnected protocol message") async def on_error(self, msg: dict, exception: AblyException): - error = msg.get('error') - code = error.get('code') - # RTN14b - if is_token_error(code) and msg.get('channel') is None: - if isinstance(self.__error_reason, AblyException): - previous_error_code = self.__error_reason.code - if not is_token_error(previous_error_code): - try: - await self.ably.auth.authorize() - except Exception as e: - log.exception(f"Attempt to renew token fails: {e}") - self.notify_state(ConnectionState.DISCONNECTED, e) + if msg.get("channel") is not None: # RTN15i + self.on_channel_message(msg) + return + if self.transport: + await self.transport.dispose() + if is_token_error(exception): # RTN14b + if self.__error_reason is None or not is_token_error(self.__error_reason): + self.__error_reason = exception + try: + await self.ably.auth._ensure_valid_auth_credentials(force=True) + except Exception as e: + self.on_error_from_authorize(e) return - self.notify_state(ConnectionState.DISCONNECTED, AblyException.from_dict(error)) + self.notify_state(self.__fail_state, exception, retry_immediately=True) return - await self.ably.auth.authorize() + self.notify_state(self.__fail_state, exception) + else: + self.enact_state_change(ConnectionState.FAILED, exception) + + def on_error_from_authorize(self, exception: AblyException): # RSA4a - elif code == 40171: - log.info(f"No means to renew authentication token: {error}") - self.notify_state(ConnectionState.FAILED, AblyException.from_dict(error)) + if exception.code == 40171: + self.notify_state(ConnectionState.FAILED, exception) + elif exception.status_code == 403: + msg = 'Client configured authentication provider returned 403; failing the connection' + log.error(f'ConnectionManager.on_error_from_authorize(): {msg}') + self.notify_state(ConnectionState.FAILED, AblyException(msg, 80019, 403)) else: - if msg.get('channel') is None: # RTN15i - self.enact_state_change(ConnectionState.FAILED, exception) - if self.transport: - await self.transport.dispose() - raise exception - else: - self.on_channel_message(msg) + msg = 'Client configured authentication provider request failed' + log.warning = (f'ConnectionManager.on_error_from_authorize: {msg}') + self.notify_state(self.__fail_state, AblyException(msg, 80019, 401)) async def on_closed(self): if self.transport: diff --git a/ably/rest/auth.py b/ably/rest/auth.py index 39df90be..f40047d8 100644 --- a/ably/rest/auth.py +++ b/ably/rest/auth.py @@ -81,18 +81,18 @@ async def get_auth_transport_param(self): key_secret = self.__auth_options.key_secret return {"key": f"{key_name}:{key_secret}"} elif self.__auth_mechanism == Auth.Method.TOKEN: - token_details = await self.__ensure_valid_auth_credentials() + token_details = await self._ensure_valid_auth_credentials() return {"accessToken": token_details.token} async def __authorize_when_necessary(self, token_params=None, auth_options=None, force=False): - token_details = await self.__ensure_valid_auth_credentials(token_params, auth_options, force) + token_details = await self._ensure_valid_auth_credentials(token_params, auth_options, force) if self.ably._is_realtime: await self.ably.connection.connection_manager.on_auth_updated(token_details) return token_details - async def __ensure_valid_auth_credentials(self, token_params=None, auth_options=None, force=False): + async def _ensure_valid_auth_credentials(self, token_params=None, auth_options=None, force=False): self.__auth_mechanism = Auth.Method.TOKEN if token_params is None: token_params = dict(self.auth_options.default_token_params) @@ -122,6 +122,9 @@ def token_details_has_expired(self): if token_details is None: return True + if not self.__time_offset: + return False + expires = token_details.expires if expires is None: return False @@ -211,7 +214,7 @@ async def create_token_request(self, token_params=None, key_secret = key_secret or self.auth_options.key_secret if not key_name or not key_secret: log.debug('key_name or key_secret blank') - raise AblyException("No key specified: no means to generate a token", 401, 40101) + raise AblyException("No key specified: no means to generate a token", 401, 40171) token_request['key_name'] = key_name if token_params.get('timestamp'): diff --git a/ably/util/helper.py b/ably/util/helper.py index d45e39b5..2a767e83 100644 --- a/ably/util/helper.py +++ b/ably/util/helper.py @@ -21,8 +21,8 @@ def unix_time_ms(): return round(time.time_ns() / 1_000_000) -def is_token_error(code): - return 40140 <= code < 40150 +def is_token_error(exception): + return 40140 <= exception.code < 40150 class Timer: From 6ebac8eb7b1b733265028a82452446a52ae7c1dd Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 20 Feb 2023 16:56:00 +0000 Subject: [PATCH 416/481] update renew token tests --- test/ably/realtime/realtimeauth_test.py | 49 ++++++++----------------- 1 file changed, 16 insertions(+), 33 deletions(-) diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index 88357f42..2efc114a 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -7,7 +7,6 @@ from ably.types.channelstate import ChannelState from ably.types.connectionstate import ConnectionEvent from ably.types.tokendetails import TokenDetails -from ably.util.exceptions import AblyException from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, random_string import urllib.parse @@ -347,56 +346,40 @@ async def callback(params): await ably.connection.connection_manager.transport.on_protocol_message(msg) assert ably.auth.token_details is not original_token_details await ably.close() + await rest.close() # RTN14b async def test_renew_token_connection_attempt_fails(self): rest = await TestApp.get_ably_rest() + call_count = 0 async def callback(params): + nonlocal call_count + call_count += 1 + params = {"ttl": 1} token_details = await rest.auth.request_token(token_params=params) - return token_details.token + return token_details ably = await TestApp.get_ably_realtime(auth_callback=callback) - msg = { - "action": ProtocolMessageAction.ERROR, - "error": { - "code": 40142, - "statusCode": 401 - } - } - await ably.connection.once_async(ConnectionState.CONNECTED) - ably.connection.connection_manager.enact_state_change(ConnectionState.DISCONNECTED, - AblyException("token error", 401, 40143)) - await ably.connection.connection_manager.transport.on_protocol_message(msg) - state_change = await ably.connection.once_async(ConnectionState.DISCONNECTED) - assert ably.connection.error_reason == state_change.reason - assert state_change.reason.code == 40143 - assert state_change.reason.status_code == 401 + await ably.connection.once_async(ConnectionState.DISCONNECTED) + assert call_count == 2 + assert ably.connection.error_reason.code == 40142 + assert ably.connection.error_reason.status_code == 401 + await ably.close() + await rest.close() # RSA4a async def test_renew_token_no_renew_means_provided(self): rest = await TestApp.get_ably_rest() + token_details = await rest.auth.request_token(token_params={'ttl': 1}) - async def callback(params): - token_details = await rest.auth.request_token(token_params=params) - return token_details.token - - ably = await TestApp.get_ably_realtime(auth_callback=callback) - msg = { - "action": ProtocolMessageAction.ERROR, - "error": { - "code": 40171, - "statusCode": 401 - } - } + ably = await TestApp.get_ably_realtime(token_details=token_details) - await ably.connection.once_async(ConnectionState.CONNECTED) - await ably.connection.connection_manager.transport.on_protocol_message(msg) state_change = await ably.connection.once_async(ConnectionState.FAILED) - assert state_change.current == ConnectionState.FAILED - assert ably.connection.error_reason == state_change.reason + # assert ably.connection.error_reason == state_change.reason assert state_change.reason.code == 40171 assert state_change.reason.status_code == 401 await ably.close() + await rest.close() From 38d1b634ea95c76c030187268719c404d0a6026d Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 22 Feb 2023 17:05:06 +0000 Subject: [PATCH 417/481] update token code in rest --- ably/http/http.py | 15 +++------------ ably/realtime/connectionmanager.py | 4 ++-- ably/rest/auth.py | 10 +++++++--- 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/ably/http/http.py b/ably/http/http.py index 3d45b068..6032ddf5 100644 --- a/ably/http/http.py +++ b/ably/http/http.py @@ -10,7 +10,7 @@ from ably.rest.auth import Auth from ably.http.httputils import HttpUtils from ably.transport.defaults import Defaults -from ably.util.exceptions import AblyException, AblyAuthException +from ably.util.exceptions import AblyException from ably.util.helper import is_token_error log = logging.getLogger(__name__) @@ -26,7 +26,7 @@ async def wrapper(rest, *args, **kwargs): auth = rest.auth token_details = auth.token_details if token_details and auth.time_offset is not None and auth.token_details_has_expired(): - await rest.reauth() + await auth.authorize() retried = True else: retried = False @@ -35,7 +35,7 @@ async def wrapper(rest, *args, **kwargs): return await func(rest, *args, **kwargs) except AblyException as e: if is_token_error(e) and not retried: - await rest.reauth() + await auth.authorize() return await func(rest, *args, **kwargs) raise @@ -135,15 +135,6 @@ def dump_body(self, body): else: return json.dumps(body, separators=(',', ':')) - async def reauth(self): - try: - await self.auth.authorize() - except AblyAuthException as e: - if e.code == 40171: - e.message = ("The provided token is not renewable and there is" - " no means to generate a new token") - raise e - def get_rest_hosts(self): hosts = self.options.get_rest_hosts() host = self.__host or self.options.fallback_realtime_host diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index 827613a1..9d3091c7 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -195,11 +195,11 @@ def on_error_from_authorize(self, exception: AblyException): elif exception.status_code == 403: msg = 'Client configured authentication provider returned 403; failing the connection' log.error(f'ConnectionManager.on_error_from_authorize(): {msg}') - self.notify_state(ConnectionState.FAILED, AblyException(msg, 80019, 403)) + self.notify_state(ConnectionState.FAILED, AblyException(msg, 403, 80019)) else: msg = 'Client configured authentication provider request failed' log.warning = (f'ConnectionManager.on_error_from_authorize: {msg}') - self.notify_state(self.__fail_state, AblyException(msg, 80019, 401)) + self.notify_state(self.__fail_state, AblyException(msg, 401, 80019)) async def on_closed(self): if self.transport: diff --git a/ably/rest/auth.py b/ably/rest/auth.py index f40047d8..6823b02c 100644 --- a/ably/rest/auth.py +++ b/ably/rest/auth.py @@ -9,7 +9,7 @@ from ably.types.capability import Capability from ably.types.tokendetails import TokenDetails from ably.types.tokenrequest import TokenRequest -from ably.util.exceptions import AblyException, IncompatibleClientIdException +from ably.util.exceptions import AblyAuthException, AblyException, IncompatibleClientIdException __all__ = ["Auth"] @@ -178,10 +178,14 @@ async def request_token(self, token_params=None, token_request = await self.token_request_from_auth_url( auth_method, auth_url, token_params, auth_headers, auth_params) - else: + elif key_name is not None and key_secret is not None: token_request = await self.create_token_request( token_params, key_name=key_name, key_secret=key_secret, query_time=query_time) + else: + msg = "Need a new token but auth_options does not include a way to request one" + log.exception(msg) + raise AblyAuthException(msg, 403, 40171) if isinstance(token_request, TokenDetails): return token_request elif isinstance(token_request, dict) and 'issued' in token_request: @@ -214,7 +218,7 @@ async def create_token_request(self, token_params=None, key_secret = key_secret or self.auth_options.key_secret if not key_name or not key_secret: log.debug('key_name or key_secret blank') - raise AblyException("No key specified: no means to generate a token", 401, 40171) + raise AblyException("No key specified: no means to generate a token", 401, 40101) token_request['key_name'] = key_name if token_params.get('timestamp'): From e81ca87f79f2ad83f9c646b19498b699f8f1f9cc Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 22 Feb 2023 17:06:09 +0000 Subject: [PATCH 418/481] add to token issues time to fix failing test --- test/ably/rest/resttoken_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/ably/rest/resttoken_test.py b/test/ably/rest/resttoken_test.py index 0d40d202..a50c5ea4 100644 --- a/test/ably/rest/resttoken_test.py +++ b/test/ably/rest/resttoken_test.py @@ -39,7 +39,7 @@ async def test_request_token_null_params(self): token_details = await self.ably.auth.request_token() post_time = await self.server_time() assert token_details.token is not None, "Expected token" - assert token_details.issued >= pre_time, "Unexpected issued time" + assert token_details.issued + 300 >= pre_time, "Unexpected issued time" assert token_details.issued <= post_time, "Unexpected issued time" assert self.permit_all == str(token_details.capability), "Unexpected capability" @@ -48,7 +48,7 @@ async def test_request_token_explicit_timestamp(self): token_details = await self.ably.auth.request_token(token_params={'timestamp': pre_time}) post_time = await self.server_time() assert token_details.token is not None, "Expected token" - assert token_details.issued >= pre_time, "Unexpected issued time" + assert token_details.issued + 300 >= pre_time, "Unexpected issued time" assert token_details.issued <= post_time, "Unexpected issued time" assert self.permit_all == str(Capability(token_details.capability)), "Unexpected Capability" From f3e532127b0d6098cc78104f2dfadd2d3235853f Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 22 Feb 2023 17:06:59 +0000 Subject: [PATCH 419/481] update tests to use right token error code and message --- test/ably/realtime/realtimeauth_test.py | 3 +-- test/ably/rest/restauth_test.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index 2efc114a..293078a9 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -378,8 +378,7 @@ async def test_renew_token_no_renew_means_provided(self): ably = await TestApp.get_ably_realtime(token_details=token_details) state_change = await ably.connection.once_async(ConnectionState.FAILED) - # assert ably.connection.error_reason == state_change.reason assert state_change.reason.code == 40171 - assert state_change.reason.status_code == 401 + assert state_change.reason.status_code == 403 await ably.close() await rest.close() diff --git a/test/ably/rest/restauth_test.py b/test/ably/rest/restauth_test.py index 66695c70..d4092c9c 100644 --- a/test/ably/rest/restauth_test.py +++ b/test/ably/rest/restauth_test.py @@ -565,7 +565,7 @@ async def test_when_not_renewable(self): publish = self.ably.channels[self.channel].publish - match = "The provided token is not renewable and there is no means to generate a new token" + match = "Need a new token but auth_options does not include a way to request one" with pytest.raises(AblyAuthException, match=match): await publish('evt', 'msg') @@ -583,7 +583,7 @@ async def test_when_not_renewable_with_token_details(self): publish = self.ably.channels[self.channel].publish - match = "The provided token is not renewable and there is no means to generate a new token" + match = "Need a new token but auth_options does not include a way to request one" with pytest.raises(AblyAuthException, match=match): await publish('evt', 'msg') From 600ea29c51c73424e6d092851543ed8a3d9840c4 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 22 Feb 2023 17:12:39 +0000 Subject: [PATCH 420/481] fix: amend log.warning misuse --- ably/realtime/connectionmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index 9d3091c7..eb277e8e 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -198,7 +198,7 @@ def on_error_from_authorize(self, exception: AblyException): self.notify_state(ConnectionState.FAILED, AblyException(msg, 403, 80019)) else: msg = 'Client configured authentication provider request failed' - log.warning = (f'ConnectionManager.on_error_from_authorize: {msg}') + log.warning(f'ConnectionManager.on_error_from_authorize: {msg}') self.notify_state(self.__fail_state, AblyException(msg, 401, 80019)) async def on_closed(self): From fca361a968063856f2a062d4b378c99789cb9b68 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 22 Feb 2023 17:19:28 +0000 Subject: [PATCH 421/481] refactor: add `AblyException.cause` --- ably/util/exceptions.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ably/util/exceptions.py b/ably/util/exceptions.py index c59ab5f5..61864198 100644 --- a/ably/util/exceptions.py +++ b/ably/util/exceptions.py @@ -6,16 +6,17 @@ class AblyException(Exception): - def __new__(cls, message, status_code, code): + def __new__(cls, message, status_code, code, cause=None): if cls == AblyException and status_code == 401: - return AblyAuthException(message, status_code, code) - return super().__new__(cls, message, status_code, code) + return AblyAuthException(message, status_code, code, cause) + return super().__new__(cls, message, status_code, code, cause) - def __init__(self, message, status_code, code): + def __init__(self, message, status_code, code, cause=None): super().__init__() self.message = message self.code = code self.status_code = status_code + self.cause = cause def __str__(self): return '%s %s %s' % (self.code, self.status_code, self.message) From 21f746372a6b32cde57e19d8c84d4ebe67cf0666 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 22 Feb 2023 17:28:31 +0000 Subject: [PATCH 422/481] refactor: wrap auth_callback errors --- ably/rest/auth.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ably/rest/auth.py b/ably/rest/auth.py index 6823b02c..ed36c03a 100644 --- a/ably/rest/auth.py +++ b/ably/rest/auth.py @@ -172,7 +172,10 @@ async def request_token(self, token_params=None, log.debug("Token Params: %s" % token_params) if auth_callback: log.debug("using token auth with authCallback") - token_request = await auth_callback(token_params) + try: + token_request = await auth_callback(token_params) + except Exception as e: + raise AblyException("auth_callback raised an exception", 401, 40170, cause=e) elif auth_url: log.debug("using token auth with authUrl") From acd7ae8456e80f9ad73ba4030546b0194246e5bb Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 22 Feb 2023 17:28:45 +0000 Subject: [PATCH 423/481] fix: handle auth exceptions when requesting transport params --- ably/realtime/connectionmanager.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index eb277e8e..92211497 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -189,6 +189,7 @@ async def on_error(self, msg: dict, exception: AblyException): self.enact_state_change(ConnectionState.FAILED, exception) def on_error_from_authorize(self, exception: AblyException): + log.info("ConnectionManager.on_error_from_authorize(): err = %s", exception) # RSA4a if exception.code == 40171: self.notify_state(ConnectionState.FAILED, exception) @@ -287,7 +288,11 @@ async def connect_base(self): self.notify_state(self.__fail_state, reason=exception) async def try_host(self, host): - params = await self.__get_transport_params() + try: + params = await self.__get_transport_params() + except AblyException as e: + self.on_error_from_authorize(e) + return self.transport = WebSocketTransport(self, host, params) self._emit('transport.pending', self.transport) self.transport.connect() From c2245bc0bf36f4272a07b25905364907136dafd5 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 23 Feb 2023 12:55:39 +0000 Subject: [PATCH 424/481] refactor: improve validation of user auth provider responses --- ably/rest/auth.py | 28 +++++++++++++++++++++++++--- test/ably/rest/restauth_test.py | 7 +++++-- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/ably/rest/auth.py b/ably/rest/auth.py index ed36c03a..8dfbad35 100644 --- a/ably/rest/auth.py +++ b/ably/rest/auth.py @@ -194,9 +194,18 @@ async def request_token(self, token_params=None, elif isinstance(token_request, dict) and 'issued' in token_request: return TokenDetails.from_dict(token_request) elif isinstance(token_request, dict): - token_request = TokenRequest.from_json(token_request) + try: + token_request = TokenRequest.from_json(token_request) + except TypeError as e: + msg = "Expected token request callback to call back with a token string, token request object, or \ + token details object" + raise AblyAuthException(msg, 401, 40170, cause=e) elif isinstance(token_request, str): + if len(token_request) == 0: + raise AblyAuthException("Token string is empty", 401, 4017) return TokenDetails(token=token_request) + elif token_request is None: + raise AblyAuthException("Token string was None", 401, 40170) token_path = "/keys/%s/requestToken" % token_request.key_name @@ -381,8 +390,21 @@ async def token_request_from_auth_url(self, method, url, token_params, headers, response = Response(resp) AblyException.raise_for_response(response) - try: + + content_type = response.response.headers.get('content-type') + + if not content_type: + raise AblyAuthException("auth_url response missing a content-type header", 401, 40170) + + is_json = "application/json" in content_type + is_text = "application/jwt" in content_type or "text/plain" in content_type + + if is_json: token_request = response.to_native() - except ValueError: + elif is_text: token_request = response.text + else: + msg = 'auth_url responded with unacceptable content-type ' + content_type + \ + ', should be either text/plain, application/jwt or application/json', + raise AblyAuthException(msg, 401, 40170) return token_request diff --git a/test/ably/rest/restauth_test.py b/test/ably/rest/restauth_test.py index d4092c9c..9e5494c3 100644 --- a/test/ably/rest/restauth_test.py +++ b/test/ably/rest/restauth_test.py @@ -378,7 +378,10 @@ def call_back(request): assert parse_qs(request.content.decode('utf-8')) == {'foo': ['token'], 'spam': ['eggs']} return Response( status_code=200, - content="token_string" + content="token_string", + headers={ + "Content-Type": "text/plain", + } ) auth_route.side_effect = call_back @@ -452,7 +455,7 @@ async def test_when_auth_url_has_query_string(self): headers = {'foo': 'bar'} ably = await TestApp.get_ably_rest(key=None, auth_url=url) auth_route = respx.get('http://www.example.com', params={'with': 'query', 'spam': 'eggs'}).mock( - return_value=Response(status_code=200, content='token_string')) + return_value=Response(status_code=200, content='token_string', headers={"Content-Type": "text/plain"})) await ably.auth.request_token(auth_url=url, auth_headers=headers, auth_params={'spam': 'eggs'}) From 971e61044a2b73b7dced17d9c054b4622ba9bd61 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 23 Feb 2023 12:56:38 +0000 Subject: [PATCH 425/481] test: add tests for user auth provider validation --- test/ably/realtime/realtimeauth_test.py | 96 +++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index 293078a9..19bc32ce 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -2,6 +2,7 @@ import json import httpx +import pytest from ably.realtime.connection import ConnectionState from ably.transport.websockettransport import ProtocolMessageAction from ably.types.channelstate import ChannelState @@ -14,6 +15,22 @@ echo_url = 'https://echo.ably.io' +async def auth_callback_failure(options, expect_failure=False): + realtime = await TestApp.get_ably_realtime(**options) + + state_change = await realtime.connection.once_async() + + if expect_failure: + assert state_change.current == ConnectionState.FAILED + assert state_change.reason.status_code == 403 + else: + assert state_change.current == ConnectionState.DISCONNECTED + assert state_change.reason.status_code == 401 + assert state_change.reason.code == 80019 + + await realtime.close() + + class TestRealtimeAuth(BaseAsyncTestCase): async def test_auth_valid_api_key(self): ably = await TestApp.get_ably_realtime() @@ -382,3 +399,82 @@ async def test_renew_token_no_renew_means_provided(self): assert state_change.reason.status_code == 403 await ably.close() await rest.close() + + async def test_auth_callback_error(self): + async def auth_callback(_): + raise Exception("An error from client code that the authCallback might return") + + await auth_callback_failure({ + 'auth_callback': auth_callback + }) + + @pytest.mark.skip(reason="blocked by https://github.com/ably/ably-python/issues/461") + async def test_auth_callback_timeout(self): + async def auth_callback(_): + await asyncio.sleep(10_000) + + await auth_callback_failure({ + 'auth_callback': auth_callback, + 'realtime_request_timeout': 100, + }) + + async def test_auth_callback_nothing(self): + async def auth_callback(_): + return + + await auth_callback_failure({ + 'auth_callback': auth_callback, + }) + + async def test_auth_callback_malformed(self): + async def auth_callback(_): + return {"horse": "ebooks"} + + await auth_callback_failure({ + 'auth_callback': auth_callback, + }) + + async def test_auth_callback_empty_string(self): + async def auth_callback(_): + return "" + + await auth_callback_failure({ + 'auth_callback': auth_callback, + }) + + @pytest.mark.skip(reason="blocked by https://github.com/ably/ably-python/issues/461") + async def test_auth_url_timeout(self): + await auth_callback_failure({ + "auth_url": "http://10.255.255.1/" + }) + + async def test_auth_url_404(self): + await auth_callback_failure({ + "auth_url": "http://example.com/404" + }) + + async def test_auth_url_wrong_content_type(self): + await auth_callback_failure({ + "auth_url": "http://example.com/" + }) + + async def test_auth_url_401(self): + await auth_callback_failure({ + "auth_url": echo_url + '/respondwith?status=401' + }) + + async def test_auth_url_403(self): + await auth_callback_failure({ + "auth_url": echo_url + '/respondwith?status=403' + }, expect_failure=True) + + async def test_auth_url_403_custom_error(self): + error = json.dumps({ + "error": { + "some_custom": "error", + } + }) + + await auth_callback_failure({ + "auth_url": echo_url + '/respondwith?status=403&body=' + urllib.parse.quote_plus(error) + }, expect_failure=True) From 210bdb3c7c0ac7d876377046a06c77a5a8a4ce34 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 23 Feb 2023 13:01:27 +0000 Subject: [PATCH 426/481] refactor: use `Optional` type for ConnectionManager fields --- ably/realtime/connectionmanager.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index 92211497..7f4e69a0 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -24,18 +24,18 @@ def __init__(self, realtime, initial_state): self.__state = initial_state self.__ping_future = None self.__timeout_in_secs = self.options.realtime_request_timeout / 1000 - self.transport: WebSocketTransport or None = None + self.transport: Optional[WebSocketTransport] = None self.__connection_details = None self.connection_id = None self.__fail_state = ConnectionState.DISCONNECTED - self.transition_timer: Timer or None = None - self.suspend_timer: Timer or None = None - self.retry_timer: Timer or None = None - self.connect_base_task: asyncio.Task or None = None - self.disconnect_transport_task: asyncio.Task or None = None + self.transition_timer: Optional[Timer] = None + self.suspend_timer: Optional[Timer] = None + self.retry_timer: Optional[Timer] = None + self.connect_base_task: Optional[asyncio.Task] = None + self.disconnect_transport_task: Optional[asyncio.Task] = None self.__fallback_hosts = self.options.get_fallback_realtime_hosts() self.queued_messages = Queue() - self.__error_reason: AblyException or None = None + self.__error_reason: Optional[AblyException] = None super().__init__() def enact_state_change(self, state, reason=None): From 0bad28ee5ab2e10f2f0143b73ac43c04feee661f Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 23 Feb 2023 15:23:03 +0000 Subject: [PATCH 427/481] refactor: move token error handling to separate method --- ably/realtime/connectionmanager.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index 7f4e69a0..7e5cb64a 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -168,6 +168,18 @@ def on_disconnected(self, msg: dict): else: log.info("No fallback host to try for disconnected protocol message") + async def on_token_error(self, exception: AblyException): + if self.__error_reason is None or not is_token_error(self.__error_reason): + self.__error_reason = exception + try: + await self.ably.auth._ensure_valid_auth_credentials(force=True) + except Exception as e: + self.on_error_from_authorize(e) + return + self.notify_state(self.__fail_state, exception, retry_immediately=True) + return + self.notify_state(self.__fail_state, exception) + async def on_error(self, msg: dict, exception: AblyException): if msg.get("channel") is not None: # RTN15i self.on_channel_message(msg) @@ -175,16 +187,7 @@ async def on_error(self, msg: dict, exception: AblyException): if self.transport: await self.transport.dispose() if is_token_error(exception): # RTN14b - if self.__error_reason is None or not is_token_error(self.__error_reason): - self.__error_reason = exception - try: - await self.ably.auth._ensure_valid_auth_credentials(force=True) - except Exception as e: - self.on_error_from_authorize(e) - return - self.notify_state(self.__fail_state, exception, retry_immediately=True) - return - self.notify_state(self.__fail_state, exception) + await self.on_token_error(exception) else: self.enact_state_change(ConnectionState.FAILED, exception) From 89545fe49013f14e4bfc1546124ef95917e6ed68 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 23 Feb 2023 15:34:48 +0000 Subject: [PATCH 428/481] refactor: parse DISCONNECTED messages in ws transport --- ably/realtime/connectionmanager.py | 10 ++++------ ably/transport/websockettransport.py | 6 +++++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index 7e5cb64a..c1ba74b8 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -153,13 +153,11 @@ def on_connected(self, connection_details: ConnectionDetails, connection_id: str self.ably.channels._on_connected() - def on_disconnected(self, msg: dict): - error = msg.get("error") - exception = AblyException.from_dict(error) + def on_disconnected(self, exception: Optional[AblyException]): self.notify_state(ConnectionState.DISCONNECTED, exception) - if error: - error_status_code = error.get("statusCode") - if error_status_code >= 500 or error_status_code <= 504: # RTN17f1 + if exception: + status_code = exception.status_code + if status_code >= 500 or status_code <= 504: # RTN17f1 if len(self.__fallback_hosts) > 0: res = asyncio.create_task(self.connect_with_fallback_hosts(self.__fallback_hosts)) if not res: diff --git a/ably/transport/websockettransport.py b/ably/transport/websockettransport.py index 6ef27c7f..f7462d6c 100644 --- a/ably/transport/websockettransport.py +++ b/ably/transport/websockettransport.py @@ -113,7 +113,11 @@ async def on_protocol_message(self, msg): self.options.fallback_realtime_host = self.host self.connection_manager.on_connected(connection_details, connection_id, reason=exception) elif action == ProtocolMessageAction.DISCONNECTED: - self.connection_manager.on_disconnected(msg) + error = msg.get('error') + exception = None + if error is not None: + exception = AblyException.from_dict(error) + self.connection_manager.on_disconnected(exception) elif action == ProtocolMessageAction.AUTH: try: await self.connection_manager.ably.auth.authorize() From 0dfe10f4adea48ff216bff742d20519b386b9d5a Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 23 Feb 2023 15:49:14 +0000 Subject: [PATCH 429/481] test: update token renewal test to simulate ERROR before connection --- test/ably/realtime/realtimeauth_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index 19bc32ce..bececec9 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -358,9 +358,9 @@ async def callback(params): } } - await ably.connection.once_async(ConnectionState.CONNECTED) + transport = await ably.connection.connection_manager.once_async('transport.pending') original_token_details = ably.auth.token_details - await ably.connection.connection_manager.transport.on_protocol_message(msg) + await transport.on_protocol_message(msg) assert ably.auth.token_details is not original_token_details await ably.close() await rest.close() From a7eb85c7f445e3bc14b4d3a64f8ec160d381ee5d Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 23 Feb 2023 15:58:56 +0000 Subject: [PATCH 430/481] feat: token renewal upon DISCONNECTED message --- ably/realtime/connectionmanager.py | 23 ++++++++++++++++------- ably/transport/websockettransport.py | 2 +- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index c1ba74b8..ab6cdce6 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -153,18 +153,27 @@ def on_connected(self, connection_details: ConnectionDetails, connection_id: str self.ably.channels._on_connected() - def on_disconnected(self, exception: Optional[AblyException]): - self.notify_state(ConnectionState.DISCONNECTED, exception) + async def on_disconnected(self, exception: Optional[AblyException]): + # RTN15h + if self.transport: + await self.transport.dispose() if exception: status_code = exception.status_code - if status_code >= 500 or status_code <= 504: # RTN17f1 + if status_code >= 500 and status_code <= 504: # RTN17f1 if len(self.__fallback_hosts) > 0: - res = asyncio.create_task(self.connect_with_fallback_hosts(self.__fallback_hosts)) - if not res: - return - self.notify_state(self.__fail_state, reason=res) + try: + await self.connect_with_fallback_hosts(self.__fallback_hosts) + except Exception as e: + self.notify_state(self.__fail_state, reason=e) + return else: log.info("No fallback host to try for disconnected protocol message") + elif is_token_error(exception): + await self.on_token_error(exception) + else: + self.notify_state(ConnectionState.DISCONNECTED, exception) + else: + log.warn("DISCONNECTED message received without error") async def on_token_error(self, exception: AblyException): if self.__error_reason is None or not is_token_error(self.__error_reason): diff --git a/ably/transport/websockettransport.py b/ably/transport/websockettransport.py index f7462d6c..c8f8aef0 100644 --- a/ably/transport/websockettransport.py +++ b/ably/transport/websockettransport.py @@ -117,7 +117,7 @@ async def on_protocol_message(self, msg): exception = None if error is not None: exception = AblyException.from_dict(error) - self.connection_manager.on_disconnected(exception) + await self.connection_manager.on_disconnected(exception) elif action == ProtocolMessageAction.AUTH: try: await self.connection_manager.ably.auth.authorize() From 37191ee423e4ac828675aeaf7efbb8f275c486ae Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 23 Feb 2023 15:59:08 +0000 Subject: [PATCH 431/481] test: add test fixtures for DISCONNECTED token error handling --- test/ably/realtime/realtimeauth_test.py | 49 +++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index bececec9..7962f5d2 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -478,3 +478,52 @@ async def test_auth_url_403_custom_error(self): await auth_callback_failure({ "auth_url": echo_url + '/respondwith?status=403&body=' + urllib.parse.quote_plus(error) }, expect_failure=True) + + # RTN15h2 + async def test_renew_token_single_attempt_upon_disconnection(self): + rest = await TestApp.get_ably_rest() + + async def callback(params): + token_details = await rest.auth.request_token(token_params=params) + return token_details.token + + ably = await TestApp.get_ably_realtime(auth_callback=callback) + msg = { + "action": ProtocolMessageAction.DISCONNECTED, + "error": { + "code": 40142, + "statusCode": 401 + } + } + + await ably.connection.once_async(ConnectionState.CONNECTED) + original_token_details = ably.auth.token_details + assert ably.connection.connection_manager.transport + await ably.connection.connection_manager.transport.on_protocol_message(msg) + assert ably.auth.token_details is not original_token_details + await ably.close() + await rest.close() + + # RTN15h1 + async def test_renew_token_no_renew_means_provided_upon_disconnection(self): + rest = await TestApp.get_ably_rest() + token_details = await rest.auth.request_token() + + ably = await TestApp.get_ably_realtime(token_details=token_details) + + state_change = await ably.connection.once_async(ConnectionState.CONNECTED) + msg = { + "action": ProtocolMessageAction.DISCONNECTED, + "error": { + "code": 40142, + "statusCode": 401 + } + } + assert ably.connection.connection_manager.transport + await ably.connection.connection_manager.transport.on_protocol_message(msg) + + state_change = await ably.connection.once_async(ConnectionState.FAILED) + assert state_change.reason.code == 40171 + assert state_change.reason.status_code == 403 + await ably.close() + await rest.close() From b2af185af4fd82819fd757bdaf85fc945d329cbc Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Fri, 24 Feb 2023 11:37:54 +0000 Subject: [PATCH 432/481] test renew token on resume --- test/ably/realtime/realtimeauth_test.py | 30 +++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index 7962f5d2..e435ef39 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -527,3 +527,33 @@ async def test_renew_token_no_renew_means_provided_upon_disconnection(self): assert state_change.reason.status_code == 403 await ably.close() await rest.close() + + async def test_renew_token_single_attempt_on_resume(self): + rest = await TestApp.get_ably_rest() + + async def callback(params): + token_details = await rest.auth.request_token(token_params=params) + return token_details.token + + ably = await TestApp.get_ably_realtime(auth_callback=callback) + msg = { + "action": ProtocolMessageAction.ERROR, + "error": { + "code": 40142, + "statusCode": 401 + } + } + + await ably.connection.once_async(ConnectionState.CONNECTED) + connection_key = ably.connection.connection_details.connection_key + await ably.connection.connection_manager.transport.dispose() + ably.connection.connection_manager.notify_state(ConnectionState.DISCONNECTED) + + transport = await ably.connection.connection_manager.once_async('transport.pending') + assert ably.connection.connection_manager.transport.params["resume"] == connection_key + + original_token_details = ably.auth.token_details + await transport.on_protocol_message(msg) + assert ably.auth.token_details is not original_token_details + await ably.close() + await rest.close() From 4c30a963bf67f958f8dbe7d09dd491efbe12d78c Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Fri, 24 Feb 2023 11:38:40 +0000 Subject: [PATCH 433/481] test no means to renew on resume --- test/ably/realtime/realtimeauth_test.py | 31 +++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index e435ef39..4ba8eaca 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -557,3 +557,34 @@ async def callback(params): assert ably.auth.token_details is not original_token_details await ably.close() await rest.close() + + async def test_renew_token_no_renew_means_provided_on_resume(self): + rest = await TestApp.get_ably_rest() + token_details = await rest.auth.request_token() + + ably = await TestApp.get_ably_realtime(token_details=token_details) + + msg = { + "action": ProtocolMessageAction.DISCONNECTED, + "error": { + "code": 40142, + "statusCode": 401 + } + } + + await ably.connection.once_async(ConnectionState.CONNECTED) + connection_key = ably.connection.connection_details.connection_key + await ably.connection.connection_manager.transport.dispose() + ably.connection.connection_manager.notify_state(ConnectionState.DISCONNECTED) + + state_change = await ably.connection.once_async(ConnectionState.CONNECTED) + assert ably.connection.connection_manager.transport.params["resume"] == connection_key + + assert ably.connection.connection_manager.transport + await ably.connection.connection_manager.transport.on_protocol_message(msg) + + state_change = await ably.connection.once_async(ConnectionState.FAILED) + assert state_change.reason.code == 40171 + assert state_change.reason.status_code == 403 + await ably.close() + await rest.close() From d72f0d147890105429a7bdc2f0c85ec9fd1caa74 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Fri, 24 Feb 2023 12:14:49 +0000 Subject: [PATCH 434/481] refactor: add `ConnectionDetails.client_id` --- ably/types/connectiondetails.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ably/types/connectiondetails.py b/ably/types/connectiondetails.py index 8fc98cf4..a281daed 100644 --- a/ably/types/connectiondetails.py +++ b/ably/types/connectiondetails.py @@ -8,12 +8,13 @@ class ConnectionDetails: connection_key: str def __init__(self, connection_state_ttl: int, max_idle_interval: int, - connection_key: str): + connection_key: str, client_id: str): self.connection_state_ttl = connection_state_ttl self.max_idle_interval = max_idle_interval self.connection_key = connection_key + self.client_id = client_id @staticmethod def from_dict(json_dict: dict): return ConnectionDetails(json_dict.get('connectionStateTtl'), json_dict.get('maxIdleInterval'), - json_dict.get('connectionKey')) + json_dict.get('connectionKey'), json_dict.get('clientId')) From 679277552d8093df8ddf27dde904e8e87d1ab29c Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Fri, 24 Feb 2023 12:15:10 +0000 Subject: [PATCH 435/481] refactor: use correct error code for client_id mismatch --- ably/rest/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/rest/auth.py b/ably/rest/auth.py index 8dfbad35..b2671023 100644 --- a/ably/rest/auth.py +++ b/ably/rest/auth.py @@ -335,7 +335,7 @@ def _configure_client_id(self, new_client_id): if self.client_id is not None and self.client_id != '*' and new_client_id != self.client_id: raise IncompatibleClientIdException( "Client ID is immutable once configured for a client. " - "Client ID cannot be changed to '{}'".format(new_client_id), 400, 40012) + "Client ID cannot be changed to '{}'".format(new_client_id), 400, 40102) self.__client_id_validated = True self.__client_id = new_client_id From 556f4aa19655d89b3adbbb0f399905661fefad38 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Fri, 24 Feb 2023 12:15:33 +0000 Subject: [PATCH 436/481] feat: validate and set client_id from connection_details --- ably/realtime/connectionmanager.py | 9 ++++++++- ably/rest/auth.py | 6 +++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index ab6cdce6..b6998aac 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -6,7 +6,7 @@ from ably.types.connectionerrors import ConnectionErrors from ably.types.connectionstate import ConnectionEvent, ConnectionState, ConnectionStateChange from ably.types.tokendetails import TokenDetails -from ably.util.exceptions import AblyException +from ably.util.exceptions import AblyException, IncompatibleClientIdException from ably.util.eventemitter import EventEmitter from datetime import datetime from ably.util.helper import get_random_id, Timer, is_token_error @@ -144,6 +144,13 @@ def on_connected(self, connection_details: ConnectionDetails, connection_id: str self.__connection_details = connection_details self.connection_id = connection_id + if connection_details.client_id: + try: + self.ably.auth._configure_client_id(connection_details.client_id) + except IncompatibleClientIdException as e: + self.notify_state(ConnectionState.FAILED, reason=e) + return + if self.__state == ConnectionState.CONNECTED: state_change = ConnectionStateChange(ConnectionState.CONNECTED, ConnectionState.CONNECTED, ConnectionEvent.UPDATE) diff --git a/ably/rest/auth.py b/ably/rest/auth.py index b2671023..5eec9906 100644 --- a/ably/rest/auth.py +++ b/ably/rest/auth.py @@ -25,10 +25,10 @@ class Method: def __init__(self, ably, options): self.__ably = ably self.__auth_options = options - if options.token_details: + + self.__client_id = options.client_id + if not self.__client_id and options.token_details: self.__client_id = options.token_details.client_id - else: - self.__client_id = options.client_id self.__client_id_validated = False self.__basic_credentials = None From 2bfe192a5c102e1ce81e6063536e908f4691f18a Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Fri, 24 Feb 2023 12:15:51 +0000 Subject: [PATCH 437/481] test: add tests for client_id validation/mismatch --- test/ably/realtime/realtimeauth_test.py | 70 +++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index 4ba8eaca..7c7a5886 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -588,3 +588,73 @@ async def test_renew_token_no_renew_means_provided_on_resume(self): assert state_change.reason.status_code == 403 await ably.close() await rest.close() + + # Request a token using client_id, then initialize a connection without one, + # and check that the connection inherits the client_id from the token_details + async def test_auth_client_id_inheritance_auth_callback(self): + rest = await TestApp.get_ably_rest() + client_id = 'test_client_id' + + async def auth_callback(_): + return await rest.auth.request_token({"client_id": client_id}) + + realtime = await TestApp.get_ably_realtime(auth_callback=auth_callback) + + await realtime.connection.once_async(ConnectionState.CONNECTED) + + assert realtime.auth.client_id == client_id + + await realtime.close() + await rest.close() + + # Rest token generation with client_id, then connecting with a + # different client_id, should fail with a library-generated message + # (RSA15a, RSA15c) + async def test_auth_client_id_mismatch(self): + rest = await TestApp.get_ably_rest() + client_id = 'test_client_id' + + token_details = await rest.auth.request_token({"client_id": client_id}) + + realtime = await TestApp.get_ably_realtime(token_details=token_details, client_id="WRONG") + + state_change = await realtime.connection.once_async(ConnectionState.FAILED) + + assert state_change.reason.code == 40102 + + await realtime.close() + await rest.close() + + # Rest token generation with clientId '*', then connecting with just the + # token string and a different clientId, should succeed (RSA15b) + async def test_auth_client_id_wildcard_token(self): + rest = await TestApp.get_ably_rest() + client_id = 'test_client_id' + + token_details = await rest.auth.request_token({"client_id": "*"}) + + realtime = await TestApp.get_ably_realtime(token_details=token_details, client_id=client_id) + + await realtime.connection.once_async(ConnectionState.CONNECTED) + + assert realtime.auth.client_id == client_id + + await realtime.close() + await rest.close() + + # Request a token using clientId, then initialize a connection using just the token string, + # and check that the connection inherits the clientId from the connectionDetails + async def test_auth_client_id_inheritance_token(self): + rest = await TestApp.get_ably_rest() + client_id = 'test_client_id' + + token_details = await rest.auth.request_token({"client_id": client_id}) + + realtime = await TestApp.get_ably_realtime(token_details=token_details) + + await realtime.connection.once_async(ConnectionState.CONNECTED) + + assert realtime.auth.client_id == client_id + + await realtime.close() + await rest.close() From 144668b6b72e049f289d2b444ef47197d43f13b2 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 27 Feb 2023 13:04:46 +0000 Subject: [PATCH 438/481] refactor: set Client._is_realtime before Auth instantiated --- ably/realtime/realtime.py | 3 ++- ably/rest/rest.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 55fc0c63..54f561cd 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -91,9 +91,10 @@ def __init__(self, key=None, loop=None, **kwargs): except RuntimeError: log.warning('Realtime client created outside event loop') + self._is_realtime = True + # RTC1 super().__init__(key, loop=loop, **kwargs) - self._is_realtime = True self.key = key self.__connection = Connection(self) diff --git a/ably/rest/rest.py b/ably/rest/rest.py index 79c7a960..59380cf4 100644 --- a/ably/rest/rest.py +++ b/ably/rest/rest.py @@ -60,7 +60,10 @@ def __init__(self, key=None, token=None, token_details=None, **kwargs): else: options = Options(**kwargs) - self._is_realtime = False + try: + self._is_realtime + except AttributeError: + self._is_realtime = False self.__http = Http(self, options) self.__auth = Auth(self, options) From 763749e13b4849fe291fd47265f7290b742c680e Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 27 Feb 2023 17:19:00 +0000 Subject: [PATCH 439/481] refactor: don't set client_id for realtime clients until connected --- ably/rest/auth.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/ably/rest/auth.py b/ably/rest/auth.py index 5eec9906..7fdcfe59 100644 --- a/ably/rest/auth.py +++ b/ably/rest/auth.py @@ -26,9 +26,12 @@ def __init__(self, ably, options): self.__ably = ably self.__auth_options = options - self.__client_id = options.client_id - if not self.__client_id and options.token_details: - self.__client_id = options.token_details.client_id + if not self.ably._is_realtime: + self.__client_id = options.client_id + if not self.__client_id and options.token_details: + self.__client_id = options.token_details.client_id + else: + self.__client_id = None self.__client_id_validated = False self.__basic_credentials = None @@ -325,14 +328,18 @@ def time_offset(self): return self.__time_offset def _configure_client_id(self, new_client_id): + log.debug("Auth._configure_client_id(): new client_id = %s", new_client_id) + original_client_id = self.client_id or self.auth_options.client_id + # If new client ID from Ably is a wildcard, but preconfigured clientId is set, # then keep the existing clientId - if self.client_id != '*' and new_client_id == '*': + if original_client_id != '*' and new_client_id == '*': self.__client_id_validated = True + self.__client_id = original_client_id return # If client_id is defined and not a wildcard, prevent it changing, this is not supported - if self.client_id is not None and self.client_id != '*' and new_client_id != self.client_id: + if original_client_id is not None and original_client_id != '*' and new_client_id != original_client_id: raise IncompatibleClientIdException( "Client ID is immutable once configured for a client. " "Client ID cannot be changed to '{}'".format(new_client_id), 400, 40102) @@ -341,12 +348,14 @@ def _configure_client_id(self, new_client_id): self.__client_id = new_client_id def can_assume_client_id(self, assumed_client_id): + original_client_id = self.client_id or self.auth_options.client_id + if self.__client_id_validated: return self.client_id == '*' or self.client_id == assumed_client_id - elif self.client_id is None or self.client_id == '*': + elif original_client_id is None or original_client_id == '*': return True # client ID is unknown else: - return self.client_id == assumed_client_id + return original_client_id == assumed_client_id async def _get_auth_headers(self): if self.__auth_mechanism == Auth.Method.BASIC: From 148304ee85ffccd77d3570e3ed42dc2bff7c4571 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 27 Feb 2023 17:19:19 +0000 Subject: [PATCH 440/481] test: add assertions for RTC4a (client_id is None until connection) --- test/ably/realtime/realtimeauth_test.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index 7c7a5886..39213c72 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -600,6 +600,9 @@ async def auth_callback(_): realtime = await TestApp.get_ably_realtime(auth_callback=auth_callback) + # RTC4a + assert realtime.auth.client_id is None + await realtime.connection.once_async(ConnectionState.CONNECTED) assert realtime.auth.client_id == client_id @@ -618,6 +621,8 @@ async def test_auth_client_id_mismatch(self): realtime = await TestApp.get_ably_realtime(token_details=token_details, client_id="WRONG") + assert realtime.auth.client_id is None + state_change = await realtime.connection.once_async(ConnectionState.FAILED) assert state_change.reason.code == 40102 @@ -635,6 +640,8 @@ async def test_auth_client_id_wildcard_token(self): realtime = await TestApp.get_ably_realtime(token_details=token_details, client_id=client_id) + assert realtime.auth.client_id is None + await realtime.connection.once_async(ConnectionState.CONNECTED) assert realtime.auth.client_id == client_id @@ -652,6 +659,8 @@ async def test_auth_client_id_inheritance_token(self): realtime = await TestApp.get_ably_realtime(token_details=token_details) + assert realtime.auth.client_id is None + await realtime.connection.once_async(ConnectionState.CONNECTED) assert realtime.auth.client_id == client_id From 4037d17422f2fcc47cb72f8d7791bcaf5fbe82fb Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 27 Feb 2023 15:56:10 +0000 Subject: [PATCH 441/481] pass client id as query param --- ably/rest/auth.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ably/rest/auth.py b/ably/rest/auth.py index 5eec9906..47ea8bae 100644 --- a/ably/rest/auth.py +++ b/ably/rest/auth.py @@ -82,7 +82,10 @@ async def get_auth_transport_param(self): return {"key": f"{key_name}:{key_secret}"} elif self.__auth_mechanism == Auth.Method.TOKEN: token_details = await self._ensure_valid_auth_credentials() - return {"accessToken": token_details.token} + auth_credentials = {"accessToken": token_details.token} + if token_details.client_id: + auth_credentials["client_id"] = token_details.client_id + return auth_credentials async def __authorize_when_necessary(self, token_params=None, auth_options=None, force=False): token_details = await self._ensure_valid_auth_credentials(token_params, auth_options, force) From 73736f78c5e8f51c827d26e744db3edc21a375b7 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 27 Feb 2023 19:10:42 +0000 Subject: [PATCH 442/481] update params to use options client id --- ably/rest/auth.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ably/rest/auth.py b/ably/rest/auth.py index 47ea8bae..22329a9e 100644 --- a/ably/rest/auth.py +++ b/ably/rest/auth.py @@ -76,16 +76,17 @@ def __init__(self, ably, options): "auth_callback, auth_url, key, token or a TokenDetail") async def get_auth_transport_param(self): + auth_credentials = {} + if self.__client_id: + auth_credentials["client_id"] = self.__client_id if self.__auth_mechanism == Auth.Method.BASIC: key_name = self.__auth_options.key_name key_secret = self.__auth_options.key_secret - return {"key": f"{key_name}:{key_secret}"} + auth_credentials["key"] = f"{key_name}:{key_secret}" elif self.__auth_mechanism == Auth.Method.TOKEN: token_details = await self._ensure_valid_auth_credentials() - auth_credentials = {"accessToken": token_details.token} - if token_details.client_id: - auth_credentials["client_id"] = token_details.client_id - return auth_credentials + auth_credentials["accessToken"] = token_details.token + return auth_credentials async def __authorize_when_necessary(self, token_params=None, auth_options=None, force=False): token_details = await self._ensure_valid_auth_credentials(token_params, auth_options, force) From 14c9828d339598760b5721c4f6905228cf30c626 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 27 Feb 2023 19:12:06 +0000 Subject: [PATCH 443/481] move client id param test --- test/ably/realtime/realtimeconnection_test.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/ably/realtime/realtimeconnection_test.py b/test/ably/realtime/realtimeconnection_test.py index 93ba9bd2..29f690ad 100644 --- a/test/ably/realtime/realtimeconnection_test.py +++ b/test/ably/realtime/realtimeconnection_test.py @@ -323,3 +323,33 @@ async def on_transport_pending(transport): await ably.connection.once_async(ConnectionState.CONNECTED) assert ably.connection.connection_manager.transport.host == fallback_host await ably.close() + + # RTN2d + async def test_connection_client_id_query_params_using_token_auth(self): + rest = await TestApp.get_ably_rest() + client_id = 'test_client_id' + + token_details = await rest.auth.request_token({"client_id": client_id}) + + realtime = await TestApp.get_ably_realtime(token_details=token_details) + + await realtime.connection.once_async(ConnectionState.CONNECTED) + assert realtime.connection.connection_manager.transport.params["client_id"] == client_id + assert realtime.auth.client_id == client_id + + await realtime.close() + await rest.close() + + async def test_connection_null_client_id_query_params_using_token_auth(self): + rest = await TestApp.get_ably_rest() + + token_details = await rest.auth.request_token() + + realtime = await TestApp.get_ably_realtime(token_details=token_details) + + await realtime.connection.once_async(ConnectionState.CONNECTED) + assert realtime.connection.connection_manager.transport.params.get("client_id") is None + assert realtime.auth.client_id is None + + await realtime.close() + await rest.close() From 692a47a2861af886606751f5f802a0c14507d6f9 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 27 Feb 2023 19:13:19 +0000 Subject: [PATCH 444/481] add test for client_id param using api key --- test/ably/realtime/realtimeconnection_test.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/ably/realtime/realtimeconnection_test.py b/test/ably/realtime/realtimeconnection_test.py index 29f690ad..164e948c 100644 --- a/test/ably/realtime/realtimeconnection_test.py +++ b/test/ably/realtime/realtimeconnection_test.py @@ -353,3 +353,23 @@ async def test_connection_null_client_id_query_params_using_token_auth(self): await realtime.close() await rest.close() + + async def test_connection_client_id_query_params_using_api_key(self): + client_id = 'test_client_id' + + ably = await TestApp.get_ably_realtime(client_id=client_id) + + await ably.connection.once_async(ConnectionState.CONNECTED) + assert ably.connection.connection_manager.transport.params["client_id"] == client_id + assert ably.auth.client_id == client_id + + await ably.close() + + async def test_connection_null_client_id_query_params_using_api_key(self): + + ably = await TestApp.get_ably_realtime() + + await ably.connection.once_async(ConnectionState.CONNECTED) + assert ably.connection.connection_manager.transport.params.get("client_id") is None + assert ably.auth.client_id is None + await ably.close() From df4f7a3eb17055f941e53fdc3cf6df0d1e8b3da3 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 28 Feb 2023 12:23:26 +0000 Subject: [PATCH 445/481] update client_id source --- ably/rest/auth.py | 4 +-- test/ably/realtime/realtimeconnection_test.py | 28 ++----------------- 2 files changed, 4 insertions(+), 28 deletions(-) diff --git a/ably/rest/auth.py b/ably/rest/auth.py index 22329a9e..0e7c7472 100644 --- a/ably/rest/auth.py +++ b/ably/rest/auth.py @@ -77,8 +77,8 @@ def __init__(self, ably, options): async def get_auth_transport_param(self): auth_credentials = {} - if self.__client_id: - auth_credentials["client_id"] = self.__client_id + if self.auth_options.client_id: + auth_credentials["client_id"] = self.auth_options.client_id if self.__auth_mechanism == Auth.Method.BASIC: key_name = self.__auth_options.key_name key_secret = self.__auth_options.key_secret diff --git a/test/ably/realtime/realtimeconnection_test.py b/test/ably/realtime/realtimeconnection_test.py index 164e948c..2017f1c9 100644 --- a/test/ably/realtime/realtimeconnection_test.py +++ b/test/ably/realtime/realtimeconnection_test.py @@ -325,22 +325,7 @@ async def on_transport_pending(transport): await ably.close() # RTN2d - async def test_connection_client_id_query_params_using_token_auth(self): - rest = await TestApp.get_ably_rest() - client_id = 'test_client_id' - - token_details = await rest.auth.request_token({"client_id": client_id}) - - realtime = await TestApp.get_ably_realtime(token_details=token_details) - - await realtime.connection.once_async(ConnectionState.CONNECTED) - assert realtime.connection.connection_manager.transport.params["client_id"] == client_id - assert realtime.auth.client_id == client_id - - await realtime.close() - await rest.close() - - async def test_connection_null_client_id_query_params_using_token_auth(self): + async def test_connection_null_client_id_query_params(self): rest = await TestApp.get_ably_rest() token_details = await rest.auth.request_token() @@ -354,7 +339,7 @@ async def test_connection_null_client_id_query_params_using_token_auth(self): await realtime.close() await rest.close() - async def test_connection_client_id_query_params_using_api_key(self): + async def test_connection_client_id_query_params(self): client_id = 'test_client_id' ably = await TestApp.get_ably_realtime(client_id=client_id) @@ -364,12 +349,3 @@ async def test_connection_client_id_query_params_using_api_key(self): assert ably.auth.client_id == client_id await ably.close() - - async def test_connection_null_client_id_query_params_using_api_key(self): - - ably = await TestApp.get_ably_realtime() - - await ably.connection.once_async(ConnectionState.CONNECTED) - assert ably.connection.connection_manager.transport.params.get("client_id") is None - assert ably.auth.client_id is None - await ably.close() From 28c7abbb78ee797db1b85254d198450b4dc844f6 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 1 Mar 2023 13:58:46 +0000 Subject: [PATCH 446/481] chore: fix capabilities.yaml key ordering --- .ably/capabilities.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ably/capabilities.yaml b/.ably/capabilities.yaml index 4617785e..5d8199aa 100644 --- a/.ably/capabilities.yaml +++ b/.ably/capabilities.yaml @@ -20,8 +20,8 @@ compliance: Realtime: Channel: Attach: - Subscribe: State Events: + Subscribe: Connection: Disconnected Retry Timeout: Lifecycle control: From ed8d777ff3811b320c1c11cc5c8c3ab73ab7cc60 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 1 Mar 2023 14:01:47 +0000 Subject: [PATCH 447/481] chore: fix key name in capabilities.yaml --- .ably/capabilities.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ably/capabilities.yaml b/.ably/capabilities.yaml index 5d8199aa..9f124310 100644 --- a/.ably/capabilities.yaml +++ b/.ably/capabilities.yaml @@ -24,7 +24,7 @@ compliance: Subscribe: Connection: Disconnected Retry Timeout: - Lifecycle control: + Lifecycle Control: Ping: State Events: Suspended Retry Timeout: From 8298aae80fe609282df229dbaaa38b1d3e3372b4 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 1 Mar 2023 14:12:57 +0000 Subject: [PATCH 448/481] chore: bump version for 2.0.0-beta.4 release --- ably/__init__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ably/__init__.py b/ably/__init__.py index 88c0f542..33265da9 100644 --- a/ably/__init__.py +++ b/ably/__init__.py @@ -15,4 +15,4 @@ logger.addHandler(logging.NullHandler()) api_version = '1.2' -lib_version = '2.0.0-beta.3' +lib_version = '2.0.0-beta.4' diff --git a/pyproject.toml b/pyproject.toml index 5d16edbe..0ec9f9da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "ably" -version = "2.0.0-beta.3" +version = "2.0.0-beta.4" description = "Python REST and Realtime client library SDK for Ably realtime messaging service" license = "Apache-2.0" authors = ["Ably <support@ably.com>"] From c4f96d1d209338226f17c9da6cc60dd10332cf49 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 1 Mar 2023 14:19:50 +0000 Subject: [PATCH 449/481] chore: update CHANGELOG for 2.0.0-beta.4 release --- CHANGELOG.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3614d251..0fbcfadf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Change Log +## [v2.0.0-beta.4](https://github.com/ably/ably-python/tree/v2.0.0-beta.4) + +This new beta release of the ably-python realtime client implements token authentication for realtime connections, allowing you to use all currently supported token options to authenticate a realtime client (auth_url, auth_callback, jwt, etc). The client will reauthenticate when the token expires or otherwise becomes invalid. + +[Full Changelog](https://github.com/ably/ably-python/compare/v2.0.0-beta.3...v2.0.0-beta.4) + +- Allow token auth methods for realtime constructor [\#425](https://github.com/ably/ably-python/issues/425) +- Send `AUTH` protocol message when `Auth.authorize` called on realtime client [\#427](https://github.com/ably/ably-python/issues/427) +- Reauth upon inbound `AUTH` protocol message [\#428](https://github.com/ably/ably-python/issues/428) +- Handle connection request failure due to token error [\#445](https://github.com/ably/ably-python/issues/445) +- Handle token `ERROR` response to a resume request [\#444](https://github.com/ably/ably-python/issues/444) +- Handle `DISCONNECTED` messages containing token errors [\#443](https://github.com/ably/ably-python/issues/443) +- Pass `clientId` as query string param when opening a new connection [\#449](https://github.com/ably/ably-python/issues/449) +- Validate `clientId` in `ClientOptions` [\#448](https://github.com/ably/ably-python/issues/448) +- Apply `Auth#clientId` only after a realtime connection has been established [\#409](https://github.com/ably/ably-python/issues/409) +- Channels should transition to `INITIALIZED` if `Connection.connect` called from terminal state [\#411](https://github.com/ably/ably-python/issues/411) +- Calling connect while `CLOSING` should start connect on a new transport [\#410](https://github.com/ably/ably-python/issues/410) +- Handle realtime channel errors [\#455](https://github.com/ably/ably-python/issues/455) + ## [v2.0.0-beta.3](https://github.com/ably/ably-python/tree/v2.0.0-beta.3) This new beta release of the ably-python realtime client implements a number of new features to improve the stability of realtime connections, allowing the client to reconnect during a temporary disconnection, use fallback hosts when necessary, and catch up on messages missed while the client was disconnected. From 86df591154ff16968bbb6d09ad93335f673e7393 Mon Sep 17 00:00:00 2001 From: Andy Ford <andy.ford@ably.com> Date: Fri, 3 Mar 2023 12:11:34 +0000 Subject: [PATCH 450/481] Remove `del` implementation Removing the magic method as it is not part of the specification. --- ably/rest/channel.py | 3 --- test/ably/rest/restchannels_test.py | 7 ------- 2 files changed, 10 deletions(-) diff --git a/ably/rest/channel.py b/ably/rest/channel.py index 5ea8efd3..f4c5de30 100644 --- a/ably/rest/channel.py +++ b/ably/rest/channel.py @@ -221,6 +221,3 @@ def __iter__(self) -> Iterator[str]: def release(self, key): del self.__all[key] - - def __delitem__(self, key): - return self.release(key) diff --git a/test/ably/rest/restchannels_test.py b/test/ably/rest/restchannels_test.py index c6a791fe..4fee4a1d 100644 --- a/test/ably/rest/restchannels_test.py +++ b/test/ably/rest/restchannels_test.py @@ -77,13 +77,6 @@ def test_channels_release(self): with pytest.raises(KeyError): self.ably.channels.release('new_channel') - def test_channels_del(self): - self.ably.channels.get('new_channel') - del self.ably.channels['new_channel'] - - with pytest.raises(KeyError): - del self.ably.channels['new_channel'] - def test_channel_has_presence(self): channel = self.ably.channels.get('new_channnel') assert channel.presence From a820b46e4e903e0f5d3ec0f096e449ab5f166e93 Mon Sep 17 00:00:00 2001 From: Andy Ford <andy.ford@ably.com> Date: Fri, 3 Mar 2023 12:13:33 +0000 Subject: [PATCH 451/481] fix: `RSN4b` compliance on rest channels The rest channels were not compliant with the aformentioned spec point as releasing a non-existent channel would raise an error. This change implements RSN4b for the rest client. --- ably/rest/channel.py | 18 ++++++++++++++++-- test/ably/rest/restchannels_test.py | 5 ++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/ably/rest/channel.py b/ably/rest/channel.py index f4c5de30..a22f68e5 100644 --- a/ably/rest/channel.py +++ b/ably/rest/channel.py @@ -219,5 +219,19 @@ def __contains__(self, item): def __iter__(self) -> Iterator[str]: return iter(self.__all.values()) - def release(self, key): - del self.__all[key] + #RSN4 + def release(self, name): + """Releases a Channel object, deleting it, and enabling it to be garbage collected. + If the channel does not exist, nothing happens. + + It also removes any listeners associated with the channel. + + Parameters + ---------- + name: str + Channel name + """ + + if name not in self.__all: + return + del self.__all[name] diff --git a/test/ably/rest/restchannels_test.py b/test/ably/rest/restchannels_test.py index 4fee4a1d..b567781f 100644 --- a/test/ably/rest/restchannels_test.py +++ b/test/ably/rest/restchannels_test.py @@ -70,12 +70,11 @@ def test_channels_iteration(self): assert isinstance(channel, Channel) assert name == channel.name + # RSN4a, RSN4b def test_channels_release(self): self.ably.channels.get('new_channel') self.ably.channels.release('new_channel') - - with pytest.raises(KeyError): - self.ably.channels.release('new_channel') + self.ably.channels.release('new_channel') def test_channel_has_presence(self): channel = self.ably.channels.get('new_channnel') From 1b09ee5bcb338e2a341dbf640224f6fab823b908 Mon Sep 17 00:00:00 2001 From: Andy Ford <andy.ford@ably.com> Date: Fri, 3 Mar 2023 12:18:21 +0000 Subject: [PATCH 452/481] lovely formatting --- ably/rest/channel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/rest/channel.py b/ably/rest/channel.py index a22f68e5..45f6ceff 100644 --- a/ably/rest/channel.py +++ b/ably/rest/channel.py @@ -219,7 +219,7 @@ def __contains__(self, item): def __iter__(self) -> Iterator[str]: return iter(self.__all.values()) - #RSN4 + # RSN4 def release(self, name): """Releases a Channel object, deleting it, and enabling it to be garbage collected. If the channel does not exist, nothing happens. From 54842128fc751456b5327cdba87fec5c96a25e8a Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 8 Mar 2023 12:15:20 +0000 Subject: [PATCH 453/481] add typings to realtime --- ably/realtime/connection.py | 33 +++--- ably/realtime/connectionmanager.py | 94 +++++++++-------- ably/realtime/realtime.py | 115 ++------------------- ably/realtime/realtime_channel.py | 159 ++++++++++++++++++++++++----- ably/rest/channel.py | 2 +- 5 files changed, 210 insertions(+), 193 deletions(-) diff --git a/ably/realtime/connection.py b/ably/realtime/connection.py index 93f59462..a27d0835 100644 --- a/ably/realtime/connection.py +++ b/ably/realtime/connection.py @@ -1,8 +1,15 @@ +from __future__ import annotations import functools import logging from ably.realtime.connectionmanager import ConnectionManager -from ably.types.connectionstate import ConnectionEvent, ConnectionState +from ably.types.connectiondetails import ConnectionDetails +from ably.types.connectionstate import ConnectionEvent, ConnectionState, ConnectionStateChange from ably.util.eventemitter import EventEmitter +from ably.util.exceptions import AblyException +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from ably.realtime.realtime import AblyRealtime log = logging.getLogger(__name__) @@ -30,9 +37,9 @@ class Connection(EventEmitter): # RTN4 Pings a realtime connection """ - def __init__(self, realtime): + def __init__(self, realtime: AblyRealtime): self.__realtime = realtime - self.__error_reason = None + self.__error_reason: Optional[AblyException] = None self.__state = ConnectionState.CONNECTING if realtime.options.auto_connect else ConnectionState.INITIALIZED self.__connection_manager = ConnectionManager(self.__realtime, self.state) self.__connection_manager.on('connectionstate', self._on_state_update) # RTN4a @@ -40,7 +47,7 @@ def __init__(self, realtime): super().__init__() # RTN11 - def connect(self): + def connect(self) -> None: """Establishes a realtime connection. Causes the connection to open, entering the connecting state @@ -48,7 +55,7 @@ def connect(self): self.__error_reason = None self.connection_manager.request_state(ConnectionState.CONNECTING) - async def close(self): + async def close(self) -> None: """Causes the connection to close, entering the closing state. Once closed, the library will not attempt to re-establish the @@ -58,7 +65,7 @@ async def close(self): await self.once_async(ConnectionState.CLOSED) # RTN13 - async def ping(self): + async def ping(self) -> float: """Send a ping to the realtime connection When connected, sends a heartbeat ping to the Ably server and executes @@ -77,36 +84,36 @@ async def ping(self): """ return await self.__connection_manager.ping() - def _on_state_update(self, state_change): + def _on_state_update(self, state_change: ConnectionStateChange) -> None: log.info(f'Connection state changing from {self.state} to {state_change.current}') self.__state = state_change.current if state_change.reason is not None: self.__error_reason = state_change.reason self.__realtime.options.loop.call_soon(functools.partial(self._emit, state_change.current, state_change)) - def _on_connection_update(self, state_change): + def _on_connection_update(self, state_change: ConnectionStateChange) -> None: self.__realtime.options.loop.call_soon(functools.partial(self._emit, ConnectionEvent.UPDATE, state_change)) # RTN4d @property - def state(self): + def state(self) -> ConnectionState: """The current connection state of the connection""" return self.__state # RTN25 @property - def error_reason(self): + def error_reason(self) -> Optional[AblyException]: """An object describing the last error which occurred on the channel, if any.""" return self.__error_reason @state.setter - def state(self, value): + def state(self, value: ConnectionState) -> None: self.__state = value @property - def connection_manager(self): + def connection_manager(self) -> ConnectionManager: return self.__connection_manager @property - def connection_details(self): + def connection_details(self) -> Optional[ConnectionDetails]: return self.__connection_manager.connection_details diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index b6998aac..8696b8cd 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -1,3 +1,4 @@ +from __future__ import annotations import logging import asyncio import httpx @@ -10,35 +11,38 @@ from ably.util.eventemitter import EventEmitter from datetime import datetime from ably.util.helper import get_random_id, Timer, is_token_error -from typing import Optional +from typing import Optional, TYPE_CHECKING from ably.types.connectiondetails import ConnectionDetails from queue import Queue +if TYPE_CHECKING: + from ably.realtime.realtime import AblyRealtime + log = logging.getLogger(__name__) class ConnectionManager(EventEmitter): - def __init__(self, realtime, initial_state): + def __init__(self, realtime: AblyRealtime, initial_state): self.options = realtime.options self.__ably = realtime - self.__state = initial_state - self.__ping_future = None - self.__timeout_in_secs = self.options.realtime_request_timeout / 1000 + self.__state: ConnectionState = initial_state + self.__ping_future: Optional[asyncio.Future] = None + self.__timeout_in_secs: float = self.options.realtime_request_timeout / 1000 self.transport: Optional[WebSocketTransport] = None - self.__connection_details = None - self.connection_id = None + self.__connection_details: Optional[ConnectionDetails] = None + self.connection_id: Optional[str] = None self.__fail_state = ConnectionState.DISCONNECTED self.transition_timer: Optional[Timer] = None self.suspend_timer: Optional[Timer] = None self.retry_timer: Optional[Timer] = None self.connect_base_task: Optional[asyncio.Task] = None self.disconnect_transport_task: Optional[asyncio.Task] = None - self.__fallback_hosts = self.options.get_fallback_realtime_hosts() - self.queued_messages = Queue() + self.__fallback_hosts: list[str] = self.options.get_fallback_realtime_hosts() + self.queued_messages: Queue = Queue() self.__error_reason: Optional[AblyException] = None super().__init__() - def enact_state_change(self, state, reason=None): + def enact_state_change(self, state: ConnectionState, reason: Optional[AblyException] = None) -> None: current_state = self.__state log.info(f'ConnectionManager.enact_state_change(): {current_state} -> {state}') self.__state = state @@ -46,7 +50,7 @@ def enact_state_change(self, state, reason=None): self.__error_reason = reason self._emit('connectionstate', ConnectionStateChange(current_state, state, state, reason)) - def check_connection(self): + def check_connection(self) -> bool: try: response = httpx.get(self.options.connectivity_check_url) return 200 <= response.status_code < 300 and \ @@ -54,10 +58,10 @@ def check_connection(self): except httpx.HTTPError: return False - def get_state_error(self): + def get_state_error(self) -> AblyException: return ConnectionErrors[self.state] - async def __get_transport_params(self): + async def __get_transport_params(self) -> dict: protocol_version = Defaults.protocol_version params = await self.ably.auth.get_auth_transport_param() params["v"] = protocol_version @@ -65,7 +69,7 @@ async def __get_transport_params(self): params["resume"] = self.connection_details.connection_key return params - async def close_impl(self): + async def close_impl(self) -> None: log.debug('ConnectionManager.close_impl()') self.cancel_suspend_timer() @@ -80,7 +84,7 @@ async def close_impl(self): self.notify_state(ConnectionState.CLOSED) - async def send_protocol_message(self, protocol_message): + async def send_protocol_message(self, protocol_message: dict) -> None: if self.state in ( ConnectionState.DISCONNECTED, ConnectionState.CONNECTING, @@ -99,12 +103,12 @@ async def send_protocol_message(self, protocol_message): raise AblyException(f"ConnectionManager.send_protocol_message(): called in {self.state}", 500, 50000) - def send_queued_messages(self): + def send_queued_messages(self) -> None: log.info(f'ConnectionManager.send_queued_messages(): sending {self.queued_messages.qsize()} message(s)') while not self.queued_messages.empty(): asyncio.create_task(self.send_protocol_message(self.queued_messages.get())) - def fail_queued_messages(self, err): + def fail_queued_messages(self, err) -> None: log.info( f"ConnectionManager.fail_queued_messages(): discarding {self.queued_messages.qsize()} messages;" + f" reason = {err}" @@ -113,7 +117,7 @@ def fail_queued_messages(self, err): msg = self.queued_messages.get() log.exception(f"ConnectionManager.fail_queued_messages(): Failed to send protocol message: {msg}") - async def ping(self): + async def ping(self) -> float: if self.__ping_future: try: response = await self.__ping_future @@ -138,7 +142,8 @@ async def ping(self): response_time_ms = (ping_end_time - ping_start_time) * 1000 return round(response_time_ms, 2) - def on_connected(self, connection_details: ConnectionDetails, connection_id: str, reason=None): + def on_connected(self, connection_details: ConnectionDetails, connection_id: str, + reason: Optional[AblyException] = None) -> None: self.__fail_state = ConnectionState.DISCONNECTED self.__connection_details = connection_details @@ -160,7 +165,7 @@ def on_connected(self, connection_details: ConnectionDetails, connection_id: str self.ably.channels._on_connected() - async def on_disconnected(self, exception: Optional[AblyException]): + async def on_disconnected(self, exception: AblyException) -> None: # RTN15h if self.transport: await self.transport.dispose() @@ -182,7 +187,7 @@ async def on_disconnected(self, exception: Optional[AblyException]): else: log.warn("DISCONNECTED message received without error") - async def on_token_error(self, exception: AblyException): + async def on_token_error(self, exception: AblyException) -> None: if self.__error_reason is None or not is_token_error(self.__error_reason): self.__error_reason = exception try: @@ -194,7 +199,7 @@ async def on_token_error(self, exception: AblyException): return self.notify_state(self.__fail_state, exception) - async def on_error(self, msg: dict, exception: AblyException): + async def on_error(self, msg: dict, exception: AblyException) -> None: if msg.get("channel") is not None: # RTN15i self.on_channel_message(msg) return @@ -205,7 +210,7 @@ async def on_error(self, msg: dict, exception: AblyException): else: self.enact_state_change(ConnectionState.FAILED, exception) - def on_error_from_authorize(self, exception: AblyException): + def on_error_from_authorize(self, exception: AblyException) -> None: log.info("ConnectionManager.on_error_from_authorize(): err = %s", exception) # RSA4a if exception.code == 40171: @@ -219,16 +224,16 @@ def on_error_from_authorize(self, exception: AblyException): log.warning(f'ConnectionManager.on_error_from_authorize: {msg}') self.notify_state(self.__fail_state, AblyException(msg, 401, 80019)) - async def on_closed(self): + async def on_closed(self) -> None: if self.transport: await self.transport.dispose() if self.connect_base_task: self.connect_base_task.cancel() - def on_channel_message(self, msg: dict): + def on_channel_message(self, msg: dict) -> None: self.__ably.channels._on_channel_message(msg) - def on_heartbeat(self, id: Optional[str]): + def on_heartbeat(self, id: Optional[str]) -> None: if self.__ping_future: # Resolve on heartbeat from ping request. if self.__ping_id == id: @@ -236,11 +241,11 @@ def on_heartbeat(self, id: Optional[str]): self.__ping_future.set_result(None) self.__ping_future = None - def deactivate_transport(self, reason=None): + def deactivate_transport(self, reason: Optional[AblyException] = None): self.transport = None self.enact_state_change(ConnectionState.DISCONNECTED, reason) - def request_state(self, state: ConnectionState, force=False): + def request_state(self, state: ConnectionState, force=False) -> None: log.info(f'ConnectionManager.request_state(): state = {state}') if not force and state == self.state: @@ -265,12 +270,12 @@ def request_state(self, state: ConnectionState, force=False): if state == ConnectionState.CLOSING: asyncio.create_task(self.close_impl()) - def start_connect(self): + def start_connect(self) -> None: self.start_suspend_timer() self.start_transition_timer(ConnectionState.CONNECTING) self.connect_base_task = asyncio.create_task(self.connect_base()) - async def connect_with_fallback_hosts(self, fallback_hosts: list): + async def connect_with_fallback_hosts(self, fallback_hosts: list) -> Optional[Exception]: for host in fallback_hosts: try: if self.check_connection(): @@ -288,7 +293,7 @@ async def connect_with_fallback_hosts(self, fallback_hosts: list): log.exception("No more fallback hosts to try") return exception - async def connect_base(self): + async def connect_base(self) -> None: fallback_hosts = self.__fallback_hosts primary_host = self.options.get_realtime_host() try: @@ -304,7 +309,7 @@ async def connect_base(self): exception = resp self.notify_state(self.__fail_state, reason=exception) - async def try_host(self, host): + async def try_host(self, host) -> None: try: params = await self.__get_transport_params() except AblyException as e: @@ -338,7 +343,8 @@ async def on_transport_failed(exception): except asyncio.CancelledError: return - def notify_state(self, state: ConnectionState, reason=None, retry_immediately=None): + def notify_state(self, state: ConnectionState, reason: Optional[AblyException] = None, + retry_immediately: Optional[bool] = None) -> None: # RTN15a retry_immediately = (retry_immediately is not False) and ( state == ConnectionState.DISCONNECTED and self.__state == ConnectionState.CONNECTED) @@ -377,7 +383,7 @@ def notify_state(self, state: ConnectionState, reason=None, retry_immediately=No self.fail_queued_messages(reason) self.ably.channels._propagate_connection_interruption(state, reason) - def start_transition_timer(self, state: ConnectionState, fail_state=None): + def start_transition_timer(self, state: ConnectionState, fail_state: Optional[ConnectionState] = None) -> None: log.debug(f'ConnectionManager.start_transition_timer(): transition state = {state}') if self.transition_timer: @@ -408,12 +414,12 @@ def cancel_transition_timer(self): self.transition_timer.cancel() self.transition_timer = None - def start_suspend_timer(self): + def start_suspend_timer(self) -> None: log.debug('ConnectionManager.start_suspend_timer()') if self.suspend_timer: return - def on_suspend_timer_expire(): + def on_suspend_timer_expire() -> None: if self.suspend_timer: self.suspend_timer = None log.info('ConnectionManager suspend timer expired, requesting new state: suspended') @@ -426,7 +432,7 @@ def on_suspend_timer_expire(): self.suspend_timer = Timer(Defaults.connection_state_ttl, on_suspend_timer_expire) - def check_suspend_timer(self, state: ConnectionState): + def check_suspend_timer(self, state: ConnectionState) -> None: if state not in ( ConnectionState.CONNECTING, ConnectionState.DISCONNECTED, @@ -434,14 +440,14 @@ def check_suspend_timer(self, state: ConnectionState): ): self.cancel_suspend_timer() - def cancel_suspend_timer(self): + def cancel_suspend_timer(self) -> None: log.debug('ConnectionManager.cancel_suspend_timer()') self.__fail_state = ConnectionState.DISCONNECTED if self.suspend_timer: self.suspend_timer.cancel() self.suspend_timer = None - def start_retry_timer(self, interval: int): + def start_retry_timer(self, interval: int) -> None: def on_retry_timeout(): log.info('ConnectionManager retry timer expired, retrying') self.retry_timer = None @@ -449,12 +455,12 @@ def on_retry_timeout(): self.retry_timer = Timer(interval, on_retry_timeout) - def cancel_retry_timer(self): + def cancel_retry_timer(self) -> None: if self.retry_timer: self.retry_timer.cancel() self.retry_timer = None - def disconnect_transport(self): + def disconnect_transport(self) -> None: log.info('ConnectionManager.disconnect_transport()') if self.transport: self.disconnect_transport_task = asyncio.create_task(self.transport.dispose()) @@ -484,7 +490,7 @@ async def on_auth_updated(self, token_details: TokenDetails): if self.state != ConnectionState.CONNECTED: future = asyncio.Future() - def on_state_change(state_change): + def on_state_change(state_change: ConnectionStateChange) -> None: if state_change.current == ConnectionState.CONNECTED: self.off('connectionstate', on_state_change) future.set_result(token_details) @@ -510,9 +516,9 @@ def ably(self): return self.__ably @property - def state(self): + def state(self) -> ConnectionState: return self.__state @property - def connection_details(self): + def connection_details(self) -> Optional[ConnectionDetails]: return self.__connection_details diff --git a/ably/realtime/realtime.py b/ably/realtime/realtime.py index 54f561cd..ea454df1 100644 --- a/ably/realtime/realtime.py +++ b/ably/realtime/realtime.py @@ -1,9 +1,9 @@ import logging import asyncio +from typing import Optional +from ably.realtime.realtime_channel import Channels from ably.realtime.connection import Connection, ConnectionState from ably.rest.rest import AblyRest -from ably.rest.channel import Channels as RestChannels -from ably.realtime.realtime_channel import ChannelState, RealtimeChannel log = logging.getLogger(__name__) @@ -34,7 +34,7 @@ class AblyRealtime(AblyRest): Closes the realtime connection """ - def __init__(self, key=None, loop=None, **kwargs): + def __init__(self, key: Optional[str] = None, loop: Optional[asyncio.AbstractEventLoop] = None, **kwargs): """Constructs a RealtimeClient object using an Ably API key. Parameters @@ -91,7 +91,7 @@ def __init__(self, key=None, loop=None, **kwargs): except RuntimeError: log.warning('Realtime client created outside event loop') - self._is_realtime = True + self._is_realtime: bool = True # RTC1 super().__init__(key, loop=loop, **kwargs) @@ -105,7 +105,7 @@ def __init__(self, key=None, loop=None, **kwargs): self.connection.connection_manager.request_state(ConnectionState.CONNECTING, force=True) # RTC15 - def connect(self): + def connect(self) -> None: """Establishes a realtime connection. Explicitly calling connect() is unnecessary unless the autoConnect attribute of the ClientOptions object @@ -117,7 +117,7 @@ def connect(self): self.connection.connect() # RTC16 - async def close(self): + async def close(self) -> None: """Causes the connection to close, entering the closing state. Once closed, the library will not attempt to re-establish the connection without an explicit call to connect() @@ -129,111 +129,12 @@ async def close(self): # RTC2 @property - def connection(self): + def connection(self) -> Connection: """Returns the realtime connection object""" return self.__connection # RTC3, RTS1 @property - def channels(self): + def channels(self) -> Channels: """Returns the realtime channel object""" return self.__channels - - -class Channels(RestChannels): - """Creates and destroys RealtimeChannel objects. - - Methods - ------- - get(name) - Gets a channel - release(name) - Releases a channel - """ - - # RTS3 - def get(self, name) -> RealtimeChannel: - """Creates a new RealtimeChannel object, or returns the existing channel object. - - Parameters - ---------- - - name: str - Channel name - """ - if name not in self.__all: - channel = self.__all[name] = RealtimeChannel(self.__ably, name) - else: - channel = self.__all[name] - return channel - - # RTS4 - def release(self, name): - """Releases a RealtimeChannel object, deleting it, and enabling it to be garbage collected - - It also removes any listeners associated with the channel. - To release a channel, the channel state must be INITIALIZED, DETACHED, or FAILED. - - - Parameters - ---------- - name: str - Channel name - """ - if name not in self.__all: - return - del self.__all[name] - - def _on_channel_message(self, msg): - channel_name = msg.get('channel') - if not channel_name: - log.error( - 'Channels.on_channel_message()', - f'received event without channel, action = {msg.get("action")}' - ) - return - - channel = self.__all[channel_name] - if not channel: - log.warning( - 'Channels.on_channel_message()', - f'receieved event for non-existent channel: {channel_name}' - ) - return - - channel._on_message(msg) - - def _propagate_connection_interruption(self, state: ConnectionState, reason): - from_channel_states = ( - ChannelState.ATTACHING, - ChannelState.ATTACHED, - ChannelState.DETACHING, - ChannelState.SUSPENDED, - ) - - connection_to_channel_state = { - ConnectionState.CLOSING: ChannelState.DETACHED, - ConnectionState.CLOSED: ChannelState.DETACHED, - ConnectionState.FAILED: ChannelState.FAILED, - ConnectionState.SUSPENDED: ChannelState.SUSPENDED, - } - - for channel_name in self.__all: - channel = self.__all[channel_name] - if channel.state in from_channel_states: - channel._notify_state(connection_to_channel_state[state], reason) - - def _on_connected(self): - for channel_name in self.__all: - channel = self.__all[channel_name] - if channel.state == ChannelState.ATTACHING or channel.state == ChannelState.DETACHING: - channel._check_pending_state() - elif channel.state == ChannelState.SUSPENDED: - asyncio.create_task(channel.attach()) - elif channel.state == ChannelState.ATTACHED: - channel._request_state(ChannelState.ATTACHING) - - def _initialize_channels(self): - for channel_name in self.__all: - channel = self.__all[channel_name] - channel._request_state(ChannelState.INITIALIZED) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index 8a27a771..f9b757d6 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -1,17 +1,20 @@ +from __future__ import annotations import asyncio import logging - +from typing import Optional, TYPE_CHECKING from ably.realtime.connection import ConnectionState from ably.transport.websockettransport import ProtocolMessageAction -from ably.rest.channel import Channel +from ably.rest.channel import Channel, Channels as RestChannels from ably.types.channelstate import ChannelState, ChannelStateChange from ably.types.flags import Flag, has_flag from ably.types.message import Message from ably.util.eventemitter import EventEmitter from ably.util.exceptions import AblyException - from ably.util.helper import Timer, is_callable_or_coroutine +if TYPE_CHECKING: + from ably.realtime.realtime import AblyRealtime + log = logging.getLogger(__name__) @@ -40,17 +43,17 @@ class RealtimeChannel(EventEmitter, Channel): Unsubscribe to messages from a channel """ - def __init__(self, realtime, name): + def __init__(self, realtime: AblyRealtime, name: str): EventEmitter.__init__(self) self.__name = name self.__realtime = realtime self.__state = ChannelState.INITIALIZED self.__message_emitter = EventEmitter() - self.__state_timer: Timer | None = None + self.__state_timer: Optional[Timer] = None self.__attach_resume = False - self.__channel_serial: str | None = None - self.__retry_timer: Timer | None = None - self.__error_reason: AblyException | None = None + self.__channel_serial: Optional[str] = None + self.__retry_timer: Optional[Timer] = None + self.__error_reason: Optional[AblyException] = None # Used to listen to state changes internally, if we use the public event emitter interface then internals # will be disrupted if the user called .off() to remove all listeners @@ -59,7 +62,7 @@ def __init__(self, realtime, name): Channel.__init__(self, realtime, name, {}) # RTL4 - async def attach(self): + async def attach(self) -> None: """Attach to channel Attach to this channel ensuring the channel is created in the Ably system and all messages published @@ -116,7 +119,7 @@ def _attach_impl(self): self._send_message(attach_msg) # RTL5 - async def detach(self): + async def detach(self) -> None: """Detach from channel Any resulting channel state change is emitted to any listeners registered @@ -165,7 +168,7 @@ async def detach(self): else: raise state_change.reason - def _detach_impl(self): + def _detach_impl(self) -> None: log.info("RealtimeChannel.detach_impl(): sending DETACH protocol message") # RTL5d @@ -177,7 +180,7 @@ def _detach_impl(self): self._send_message(detach_msg) # RTL7 - async def subscribe(self, *args): + async def subscribe(self, *args) -> None: """Subscribe to a channel Registers a listener for messages on the channel. @@ -232,7 +235,7 @@ async def subscribe(self, *args): await self.attach() # RTL8 - def unsubscribe(self, *args): + def unsubscribe(self, *args) -> None: """Unsubscribe from a channel Deregister the given listener for (for any/all event names). @@ -285,7 +288,7 @@ def unsubscribe(self, *args): # RTL8a self.__message_emitter.off(listener) - def _on_message(self, msg): + def _on_message(self, msg: dict) -> None: action = msg.get('action') # RTL4c1 @@ -329,12 +332,13 @@ def _on_message(self, msg): error = AblyException.from_dict(msg.get('error')) self._notify_state(ChannelState.FAILED, reason=error) - def _request_state(self, state: ChannelState): + def _request_state(self, state: ChannelState) -> None: log.info(f'RealtimeChannel._request_state(): state = {state}') self._notify_state(state) self._check_pending_state() - def _notify_state(self, state: ChannelState, reason=None, resumed=False): + def _notify_state(self, state: ChannelState, reason: Optional[AblyException] = None, + resumed: bool = False) -> None: log.info(f'RealtimeChannel._notify_state(): state = {state}') self.__clear_state_timer() @@ -369,7 +373,7 @@ def _notify_state(self, state: ChannelState, reason=None, resumed=False): self._emit(state, state_change) self.__internal_state_emitter._emit(state, state_change) - def _send_message(self, msg): + def _send_message(self, msg: dict) -> None: asyncio.create_task(self.__realtime.connection.connection_manager.send_protocol_message(msg)) def _check_pending_state(self): @@ -386,21 +390,21 @@ def _check_pending_state(self): self.__start_state_timer() self._detach_impl() - def __start_state_timer(self): + def __start_state_timer(self) -> None: if not self.__state_timer: - def on_timeout(): + def on_timeout() -> None: log.info('RealtimeChannel.start_state_timer(): timer expired') self.__state_timer = None self.__timeout_pending_state() self.__state_timer = Timer(self.__realtime.options.realtime_request_timeout, on_timeout) - def __clear_state_timer(self): + def __clear_state_timer(self) -> None: if self.__state_timer: self.__state_timer.cancel() self.__state_timer = None - def __timeout_pending_state(self): + def __timeout_pending_state(self) -> None: if self.state == ChannelState.ATTACHING: self._notify_state( ChannelState.SUSPENDED, reason=AblyException("Channel attach timed out", 408, 90007)) @@ -409,18 +413,18 @@ def __timeout_pending_state(self): else: self._check_pending_state() - def __start_retry_timer(self): + def __start_retry_timer(self) -> None: if self.__retry_timer: return self.__retry_timer = Timer(self.ably.options.channel_retry_timeout, self.__on_retry_timer_expire) - def __cancel_retry_timer(self): + def __cancel_retry_timer(self) -> None: if self.__retry_timer: self.__retry_timer.cancel() self.__retry_timer = None - def __on_retry_timer_expire(self): + def __on_retry_timer_expire(self) -> None: if self.state == ChannelState.SUSPENDED and self.ably.connection.state == ConnectionState.CONNECTED: self.__retry_timer = None log.info("RealtimeChannel retry timer expired, attempting a new attach") @@ -428,22 +432,121 @@ def __on_retry_timer_expire(self): # RTL23 @property - def name(self): + def name(self) -> str: """Returns channel name""" return self.__name # RTL2b @property - def state(self): + def state(self) -> ChannelState: """Returns channel state""" return self.__state @state.setter - def state(self, state: ChannelState): + def state(self, state: ChannelState) -> None: self.__state = state # RTL24 @property - def error_reason(self): + def error_reason(self) -> Optional[AblyException]: """An AblyException instance describing the last error which occurred on the channel, if any.""" return self.__error_reason + + +class Channels(RestChannels): + """Creates and destroys RealtimeChannel objects. + + Methods + ------- + get(name) + Gets a channel + release(name) + Releases a channel + """ + + # RTS3 + def get(self, name: str) -> RealtimeChannel: + """Creates a new RealtimeChannel object, or returns the existing channel object. + + Parameters + ---------- + + name: str + Channel name + """ + if name not in self.__all: + channel = self.__all[name] = RealtimeChannel(self.__ably, name) + else: + channel = self.__all[name] + return channel + + # RTS4 + def release(self, name: str) -> None: + """Releases a RealtimeChannel object, deleting it, and enabling it to be garbage collected + + It also removes any listeners associated with the channel. + To release a channel, the channel state must be INITIALIZED, DETACHED, or FAILED. + + + Parameters + ---------- + name: str + Channel name + """ + if name not in self.__all: + return + del self.__all[name] + + def _on_channel_message(self, msg: dict) -> None: + channel_name = msg.get('channel') + if not channel_name: + log.error( + 'Channels.on_channel_message()', + f'received event without channel, action = {msg.get("action")}' + ) + return + + channel = self.__all[channel_name] + if not channel: + log.warning( + 'Channels.on_channel_message()', + f'receieved event for non-existent channel: {channel_name}' + ) + return + + channel._on_message(msg) + + def _propagate_connection_interruption(self, state: ConnectionState, reason: Optional[AblyException]) -> None: + from_channel_states = ( + ChannelState.ATTACHING, + ChannelState.ATTACHED, + ChannelState.DETACHING, + ChannelState.SUSPENDED, + ) + + connection_to_channel_state = { + ConnectionState.CLOSING: ChannelState.DETACHED, + ConnectionState.CLOSED: ChannelState.DETACHED, + ConnectionState.FAILED: ChannelState.FAILED, + ConnectionState.SUSPENDED: ChannelState.SUSPENDED, + } + + for channel_name in self.__all: + channel = self.__all[channel_name] + if channel.state in from_channel_states: + channel._notify_state(connection_to_channel_state[state], reason) + + def _on_connected(self) -> None: + for channel_name in self.__all: + channel = self.__all[channel_name] + if channel.state == ChannelState.ATTACHING or channel.state == ChannelState.DETACHING: + channel._check_pending_state() + elif channel.state == ChannelState.SUSPENDED: + asyncio.create_task(channel.attach()) + elif channel.state == ChannelState.ATTACHED: + channel._request_state(ChannelState.ATTACHING) + + def _initialize_channels(self) -> None: + for channel_name in self.__all: + channel = self.__all[channel_name] + channel._request_state(ChannelState.INITIALIZED) diff --git a/ably/rest/channel.py b/ably/rest/channel.py index 45f6ceff..2ca220b5 100644 --- a/ably/rest/channel.py +++ b/ably/rest/channel.py @@ -185,7 +185,7 @@ def options(self, options): class Channels: def __init__(self, rest): self.__ably = rest - self.__all = OrderedDict() + self.__all: dict = OrderedDict() def get(self, name, **kwargs): if isinstance(name, bytes): From 2873fe8cda650a86fa2d63455a1ff89481742ca4 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Tue, 7 Mar 2023 15:00:06 +0000 Subject: [PATCH 454/481] add typings to rest --- ably/rest/auth.py | 15 ++++++++------- ably/rest/channel.py | 4 ++-- ably/rest/rest.py | 10 +++++----- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/ably/rest/auth.py b/ably/rest/auth.py index a9594428..66e961fe 100644 --- a/ably/rest/auth.py +++ b/ably/rest/auth.py @@ -142,7 +142,7 @@ def token_details_has_expired(self): return expires < timestamp + token_details.TOKEN_EXPIRY_BUFFER - async def authorize(self, token_params=None, auth_options=None): + async def authorize(self, token_params: dict = None, auth_options=None): return await self.__authorize_when_necessary(token_params, auth_options, force=True) async def authorise(self, *args, **kwargs): @@ -151,10 +151,10 @@ async def authorise(self, *args, **kwargs): DeprecationWarning) return await self.authorize(*args, **kwargs) - async def request_token(self, token_params=None, + async def request_token(self, token_params: dict = None, # auth_options - key_name=None, key_secret=None, auth_callback=None, - auth_url=None, auth_method=None, auth_headers=None, + key_name: str = None, key_secret: str = None, auth_callback=None, + auth_url: str = None, auth_method: str = None, auth_headers: dict = None, auth_params=None, query_time=None): token_params = token_params or {} token_params = dict(self.auth_options.default_token_params, @@ -228,8 +228,8 @@ async def request_token(self, token_params=None, log.debug("Token: %s" % str(response_dict.get("token"))) return TokenDetails.from_dict(response_dict) - async def create_token_request(self, token_params=None, - key_name=None, key_secret=None, query_time=None): + async def create_token_request(self, token_params: dict = None, + key_name: str = None, key_secret: str = None, query_time=None): token_params = token_params or {} token_request = {} @@ -385,7 +385,8 @@ def _timestamp(self): def _random_nonce(self): return uuid.uuid4().hex[:16] - async def token_request_from_auth_url(self, method, url, token_params, headers, auth_params): + async def token_request_from_auth_url(self, method: str, url: str, token_params, + headers, auth_params): body = None params = None if method == 'GET': diff --git a/ably/rest/channel.py b/ably/rest/channel.py index 2ca220b5..df84043e 100644 --- a/ably/rest/channel.py +++ b/ably/rest/channel.py @@ -30,7 +30,7 @@ def __init__(self, ably, name, options): self.__presence = Presence(self) @catch_all - async def history(self, direction=None, limit=None, start=None, end=None): + async def history(self, direction=None, limit: int = None, start=None, end=None): """Returns the history for this channel""" params = format_params({}, direction=direction, start=start, end=end, limit=limit) path = self.__base_path + 'messages' + params @@ -220,7 +220,7 @@ def __iter__(self) -> Iterator[str]: return iter(self.__all.values()) # RSN4 - def release(self, name): + def release(self, name: str): """Releases a Channel object, deleting it, and enabling it to be garbage collected. If the channel does not exist, nothing happens. diff --git a/ably/rest/rest.py b/ably/rest/rest.py index 59380cf4..cf4f3b2e 100644 --- a/ably/rest/rest.py +++ b/ably/rest/rest.py @@ -18,7 +18,7 @@ class AblyRest: """Ably Rest Client""" - def __init__(self, key=None, token=None, token_details=None, **kwargs): + def __init__(self, key: str = None, token: str = None, token_details: TokenDetails = None, **kwargs): """Create an AblyRest instance. :Parameters: @@ -77,8 +77,8 @@ async def __aenter__(self): return self @catch_all - async def stats(self, direction=None, start=None, end=None, params=None, - limit=None, paginated=None, unit=None, timeout=None): + async def stats(self, direction: str = None, start=None, end=None, params: dict = None, + limit: int = None, paginated=None, unit=None, timeout=None): """Returns the stats for this application""" params = format_params(params, direction=direction, start=start, end=end, limit=limit, unit=unit) url = '/stats' + params @@ -93,7 +93,7 @@ async def time(self, timeout=None): return r.to_native()[0] @property - def client_id(self): + def client_id(self) -> str: return self.options.client_id @property @@ -117,7 +117,7 @@ def options(self): def push(self): return self.__push - async def request(self, method, path, params=None, body=None, headers=None): + async def request(self, method: str, path: str, params: dict = None, body=None, headers=None): url = path if params: url += '?' + urlencode(params) From 411475f338889d29f4723bc478280bd8f125ac22 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Wed, 8 Mar 2023 10:52:47 +0000 Subject: [PATCH 455/481] add more typings --- ably/rest/auth.py | 42 +++++++++++++++++++++++++----------------- ably/rest/rest.py | 16 +++++++++------- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/ably/rest/auth.py b/ably/rest/auth.py index 66e961fe..15eaf166 100644 --- a/ably/rest/auth.py +++ b/ably/rest/auth.py @@ -1,11 +1,18 @@ +from __future__ import annotations import base64 from datetime import timedelta import logging import time +from typing import Optional, TYPE_CHECKING, Union import uuid import warnings import httpx +from ably.types.options import Options +if TYPE_CHECKING: + from ably.rest.rest import AblyRest + from ably.realtime.realtime import AblyRealtime + from ably.types.capability import Capability from ably.types.tokendetails import TokenDetails from ably.types.tokenrequest import TokenRequest @@ -22,7 +29,7 @@ class Method: BASIC = "BASIC" TOKEN = "TOKEN" - def __init__(self, ably, options): + def __init__(self, ably: Union[AblyRest, AblyRealtime], options: Options): self.__ably = ably self.__auth_options = options @@ -32,12 +39,12 @@ def __init__(self, ably, options): self.__client_id = options.token_details.client_id else: self.__client_id = None - self.__client_id_validated = False + self.__client_id_validated: bool = False - self.__basic_credentials = None - self.__auth_params = None - self.__token_details = None - self.__time_offset = None + self.__basic_credentials: Optional[str] = None + self.__auth_params: Optional[dict] = None + self.__token_details: Optional[TokenDetails] = None + self.__time_offset: Optional[int] = None must_use_token_auth = options.use_token_auth is True must_not_use_token_auth = options.use_token_auth is False @@ -142,7 +149,7 @@ def token_details_has_expired(self): return expires < timestamp + token_details.TOKEN_EXPIRY_BUFFER - async def authorize(self, token_params: dict = None, auth_options=None): + async def authorize(self, token_params: Optional[dict] = None, auth_options=None): return await self.__authorize_when_necessary(token_params, auth_options, force=True) async def authorise(self, *args, **kwargs): @@ -151,11 +158,12 @@ async def authorise(self, *args, **kwargs): DeprecationWarning) return await self.authorize(*args, **kwargs) - async def request_token(self, token_params: dict = None, + async def request_token(self, token_params: Optional[dict] = None, # auth_options - key_name: str = None, key_secret: str = None, auth_callback=None, - auth_url: str = None, auth_method: str = None, auth_headers: dict = None, - auth_params=None, query_time=None): + key_name: Optional[str] = None, key_secret: Optional[str] = None, auth_callback=None, + auth_url: Optional[str] = None, auth_method: Optional[str] = None, + auth_headers: Optional[dict] = None, auth_params: Optional[dict] = None, + query_time=None): token_params = token_params or {} token_params = dict(self.auth_options.default_token_params, **token_params) @@ -228,8 +236,8 @@ async def request_token(self, token_params: dict = None, log.debug("Token: %s" % str(response_dict.get("token"))) return TokenDetails.from_dict(response_dict) - async def create_token_request(self, token_params: dict = None, - key_name: str = None, key_secret: str = None, query_time=None): + async def create_token_request(self, token_params: Optional[dict] = None, key_name: Optional[str] = None, + key_secret: Optional[str] = None, query_time=None): token_params = token_params or {} token_request = {} @@ -279,18 +287,18 @@ async def create_token_request(self, token_params: dict = None, # simply for testing purposes token_request["nonce"] = token_params.get('nonce') or self._random_nonce() - token_request = TokenRequest(**token_request) + token_req = TokenRequest(**token_request) if token_params.get('mac') is None: # Note: There is no expectation that the client # specifies the mac; this is done by the library # However, this can be overridden by the client # simply for testing purposes. - token_request.sign_request(key_secret.encode('utf8')) + token_req.sign_request(key_secret.encode('utf8')) else: - token_request.mac = token_params['mac'] + token_req.mac = token_params['mac'] - return token_request + return token_req @property def ably(self): diff --git a/ably/rest/rest.py b/ably/rest/rest.py index cf4f3b2e..dffb9948 100644 --- a/ably/rest/rest.py +++ b/ably/rest/rest.py @@ -1,4 +1,5 @@ import logging +from typing import Optional from urllib.parse import urlencode from ably.http.http import Http @@ -18,7 +19,8 @@ class AblyRest: """Ably Rest Client""" - def __init__(self, key: str = None, token: str = None, token_details: TokenDetails = None, **kwargs): + def __init__(self, key: Optional[str] = None, token: Optional[str] = None, + token_details: Optional[TokenDetails] = None, **kwargs): """Create an AblyRest instance. :Parameters: @@ -77,11 +79,11 @@ async def __aenter__(self): return self @catch_all - async def stats(self, direction: str = None, start=None, end=None, params: dict = None, - limit: int = None, paginated=None, unit=None, timeout=None): + async def stats(self, direction: Optional[str] = None, start=None, end=None, params: Optional[dict] = None, + limit: Optional[int] = None, paginated=None, unit=None, timeout=None): """Returns the stats for this application""" - params = format_params(params, direction=direction, start=start, end=end, limit=limit, unit=unit) - url = '/stats' + params + formatted_params = format_params(params, direction=direction, start=start, end=end, limit=limit, unit=unit) + url = '/stats' + formatted_params return await PaginatedResult.paginated_query( self.http, url=url, response_processor=stats_response_processor) @@ -93,7 +95,7 @@ async def time(self, timeout=None): return r.to_native()[0] @property - def client_id(self) -> str: + def client_id(self) -> Optional[str]: return self.options.client_id @property @@ -117,7 +119,7 @@ def options(self): def push(self): return self.__push - async def request(self, method: str, path: str, params: dict = None, body=None, headers=None): + async def request(self, method: str, path: str, params: Optional[dict] = None, body=None, headers=None): url = path if params: url += '?' + urlencode(params) From 2255f27c38ddd3f9bd0d22af3fc6513c82e5251f Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Fri, 10 Mar 2023 10:56:03 +0000 Subject: [PATCH 456/481] add typings for push admin --- ably/rest/push.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ably/rest/push.py b/ably/rest/push.py index e63aeeb1..d3cf0e03 100644 --- a/ably/rest/push.py +++ b/ably/rest/push.py @@ -1,3 +1,4 @@ +from typing import Optional from ably.http.paginatedresult import PaginatedResult, format_params from ably.types.device import DeviceDetails, device_details_response_processor from ably.types.channelsubscription import PushChannelSubscription, channel_subscriptions_response_processor @@ -34,7 +35,7 @@ def device_registrations(self): def channel_subscriptions(self): return self.__channel_subscriptions - async def publish(self, recipient, data, timeout=None): + async def publish(self, recipient: dict, data: dict, timeout: Optional[float] = None): """Publish a push notification to a single device. :Parameters: @@ -67,7 +68,7 @@ def __init__(self, ably): def ably(self): return self.__ably - async def get(self, device_id): + async def get(self, device_id: str): """Returns a DeviceDetails object if the device id is found or results in a not found error if the device cannot be found. @@ -91,7 +92,7 @@ async def list(self, **params): self.ably.http, url=path, response_processor=device_details_response_processor) - async def save(self, device): + async def save(self, device: dict): """Creates or updates the device. Returns a DeviceDetails object. :Parameters: @@ -104,7 +105,7 @@ async def save(self, device): obj = response.to_native() return DeviceDetails.from_dict(obj) - async def remove(self, device_id): + async def remove(self, device_id: str): """Deletes the registered device identified by the given device id. :Parameters: @@ -154,7 +155,7 @@ async def list_channels(self, **params): return await PaginatedResult.paginated_query(self.ably.http, url=path, response_processor=channels_response_processor) - async def save(self, subscription): + async def save(self, subscription: dict): """Creates or updates the subscription. Returns a PushChannelSubscription object. @@ -168,7 +169,7 @@ async def save(self, subscription): obj = response.to_native() return PushChannelSubscription.from_dict(obj) - async def remove(self, subscription): + async def remove(self, subscription: dict): """Deletes the given subscription. :Parameters: From 73f6733e42ce6adc3e3e4c27183286fab6810a57 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Fri, 10 Mar 2023 10:56:24 +0000 Subject: [PATCH 457/481] add typings to Rest.time --- ably/rest/rest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/rest/rest.py b/ably/rest/rest.py index dffb9948..7662392c 100644 --- a/ably/rest/rest.py +++ b/ably/rest/rest.py @@ -88,7 +88,7 @@ async def stats(self, direction: Optional[str] = None, start=None, end=None, par self.http, url=url, response_processor=stats_response_processor) @catch_all - async def time(self, timeout=None): + async def time(self, timeout: Optional[float] = None): """Returns the current server time in ms since the unix epoch""" r = await self.http.get('/time', skip_auth=True, timeout=timeout) AblyException.raise_for_response(r) From 1117b3bf96bea41d8b6ea1d313a892a3fcf48e08 Mon Sep 17 00:00:00 2001 From: moyosore <mohyour01@gmail.com> Date: Mon, 13 Mar 2023 13:32:09 +0000 Subject: [PATCH 458/481] add return value to rest.time --- ably/rest/rest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/rest/rest.py b/ably/rest/rest.py index 7662392c..64b2c683 100644 --- a/ably/rest/rest.py +++ b/ably/rest/rest.py @@ -88,7 +88,7 @@ async def stats(self, direction: Optional[str] = None, start=None, end=None, par self.http, url=url, response_processor=stats_response_processor) @catch_all - async def time(self, timeout: Optional[float] = None): + async def time(self, timeout: Optional[float] = None) -> float: """Returns the current server time in ms since the unix epoch""" r = await self.http.get('/time', skip_auth=True, timeout=timeout) AblyException.raise_for_response(r) From f7a31a2491eff5ac476dde1e223a296c126b0cba Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 16 Mar 2023 17:48:19 +0000 Subject: [PATCH 459/481] refactor!: remove `client_id` and `extras` args from `publish_name_data` --- ably/rest/channel.py | 12 ++---------- test/ably/rest/restchannelpublish_test.py | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/ably/rest/channel.py b/ably/rest/channel.py index df84043e..d7995607 100644 --- a/ably/rest/channel.py +++ b/ably/rest/channel.py @@ -5,7 +5,6 @@ import os from typing import Iterator from urllib import parse -import warnings from methoddispatch import SingleDispatch, singledispatch import msgpack @@ -100,15 +99,8 @@ async def publish_messages(self, messages, params=None, timeout=None): return await self.ably.http.post(path, body=request_body, timeout=timeout) @_publish.register(str) - async def publish_name_data(self, name, data, client_id=None, extras=None, timeout=None): - # RSL1h - if client_id or extras: - warnings.warn( - "Support for client_id and extras will be removed in 2.0", - DeprecationWarning - ) - - messages = [Message(name, data, client_id, extras=extras)] + async def publish_name_data(self, name, data, timeout=None): + messages = [Message(name, data)] return await self.publish_messages(messages, timeout=timeout) async def publish(self, *args, **kwargs): diff --git a/test/ably/rest/restchannelpublish_test.py b/test/ably/rest/restchannelpublish_test.py index ed415527..48f18c3b 100644 --- a/test/ably/rest/restchannelpublish_test.py +++ b/test/ably/rest/restchannelpublish_test.py @@ -279,9 +279,8 @@ async def test_publish_message_with_client_id_on_identified_client(self): # works if same channel = self.ably_with_client_id.channels[ self.get_channel_name('persisted:with_client_id_identified_client')] - await channel.publish(name='publish', - data='test', - client_id=self.ably_with_client_id.client_id) + message = Message(name='publish', data='test', client_id=self.ably_with_client_id.client_id) + await channel.publish(message) history = await channel.history() messages = history.items @@ -291,9 +290,10 @@ async def test_publish_message_with_client_id_on_identified_client(self): assert messages[0].client_id == self.ably_with_client_id.client_id + message = Message(name='publish', data='test', client_id='invalid') # fails if different with pytest.raises(IncompatibleClientIdException): - await channel.publish(name='publish', data='test', client_id='invalid') + await channel.publish(message) async def test_publish_message_with_wrong_client_id_on_implicit_identified_client(self): new_token = await self.ably.auth.authorize(token_params={'client_id': uuid.uuid4().hex}) @@ -304,8 +304,9 @@ async def test_publish_message_with_wrong_client_id_on_implicit_identified_clien channel = new_ably.channels[ self.get_channel_name('persisted:wrong_client_id_implicit_client')] + message = Message(name='publish', data='test', client_id='invalid') with pytest.raises(AblyException) as excinfo: - await channel.publish(name='publish', data='test', client_id='invalid') + await channel.publish(message) assert 400 == excinfo.value.status_code assert 40012 == excinfo.value.code @@ -324,8 +325,8 @@ async def test_wildcard_client_id_can_publish_as_others(self): self.get_channel_name('persisted:wildcard_client_id')] await channel.publish(name='publish1', data='no client_id') some_client_id = uuid.uuid4().hex - await channel.publish(name='publish2', data='some client_id', - client_id=some_client_id) + message = Message(name='publish2', data='some client_id', client_id=some_client_id) + await channel.publish(message) history = await channel.history() messages = history.items @@ -358,7 +359,8 @@ async def test_publish_extras(self): 'notification': {"title": "Testing"}, } } - await channel.publish(name='test-name', data='test-data', extras=extras) + message = Message(name='test-name', data='test-data', extras=extras) + await channel.publish(message) # Get the history for this channel history = await channel.history() From 59666315b04a1ebf5638d0283cccc69cb4a106c6 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 20 Mar 2023 10:12:46 +0000 Subject: [PATCH 460/481] test: use 'sandbox' environment instead of explicit hosts --- ably/types/options.py | 1 + test/ably/realtime/realtimeconnection_test.py | 55 +++++++++++++------ test/ably/rest/restauth_test.py | 16 +++--- test/ably/rest/restchannelhistory_test.py | 2 +- test/ably/testapp.py | 30 +++++----- 5 files changed, 62 insertions(+), 42 deletions(-) diff --git a/ably/types/options.py b/ably/types/options.py index 4db971e8..676b5473 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -317,6 +317,7 @@ def __get_rest_hosts(self): def __get_realtime_hosts(self): if self.realtime_host is not None: host = self.realtime_host + return [host] elif self.environment != "production": host = f'{self.environment}-{Defaults.realtime_host}' else: diff --git a/test/ably/realtime/realtimeconnection_test.py b/test/ably/realtime/realtimeconnection_test.py index 2017f1c9..76ba1d1f 100644 --- a/test/ably/realtime/realtimeconnection_test.py +++ b/test/ably/realtime/realtimeconnection_test.py @@ -288,40 +288,61 @@ async def test_retry_immediately_upon_unexpected_disconnection(self): await ably.close() async def test_fallback_host(self): - fallback_host = 'sandbox-realtime.ably.io' - ably = await TestApp.get_ably_realtime(realtime_host="iamnotahost", disconnected_retry_timeout=400000, - fallback_hosts=[fallback_host]) + ably = await TestApp.get_ably_realtime() + + await ably.connection.connection_manager.once_async('transport.pending') + assert ably.connection.connection_manager.transport + ably.connection.connection_manager.transport._emit('failed', AblyException("test exception", 502, 50200)) + await ably.connection.once_async(ConnectionState.CONNECTED) - assert ably.connection.connection_manager.transport.host == fallback_host - assert ably.options.fallback_realtime_host == fallback_host + + assert ably.connection.connection_manager.transport.host != self.test_vars["realtime_host"] + assert ably.options.fallback_realtime_host != self.test_vars["realtime_host"] await ably.close() async def test_fallback_host_no_connection(self): - fallback_host = 'sandbox-realtime.ably.io' - ably = await TestApp.get_ably_realtime(realtime_host="iamnotahost", disconnected_retry_timeout=400000, - fallback_hosts=[fallback_host]) + ably = await TestApp.get_ably_realtime() + + await ably.connection.connection_manager.once_async('transport.pending') + assert ably.connection.connection_manager.transport def check_connection(): return False ably.connection.connection_manager.check_connection = check_connection + asyncio.create_task(ably.connection.connection_manager.transport.on_protocol_message({ + "action": ProtocolMessageAction.DISCONNECTED, + "error": { + "statusCode": 502, + "code": 50200, + "message": "test exception" + } + })) + await ably.connection.once_async(ConnectionState.DISCONNECTED) - assert ably.connection.connection_manager.transport.host == "iamnotahost" + + assert ably.options.fallback_realtime_host is None await ably.close() async def test_fallback_host_disconnected_protocol_msg(self): - fallback_host = 'sandbox-realtime.ably.io' - ably = await TestApp.get_ably_realtime(realtime_host="iamnotahost", disconnected_retry_timeout=400000, - fallback_hosts=[fallback_host]) - - async def on_transport_pending(transport): - await transport.on_protocol_message({'action': 6, "error": {"statusCode": 500, "code": 500}}) + ably = await TestApp.get_ably_realtime() - ably.connection.connection_manager.once('transport.pending', on_transport_pending) + await ably.connection.connection_manager.once_async('transport.pending') + assert ably.connection.connection_manager.transport + asyncio.create_task(ably.connection.connection_manager.transport.on_protocol_message({ + "action": ProtocolMessageAction.DISCONNECTED, + "error": { + "statusCode": 502, + "code": 50200, + "message": "test exception" + } + })) await ably.connection.once_async(ConnectionState.CONNECTED) - assert ably.connection.connection_manager.transport.host == fallback_host + + assert ably.connection.connection_manager.transport.host != self.test_vars["realtime_host"] + assert ably.options.fallback_realtime_host != self.test_vars["realtime_host"] await ably.close() # RTN2d diff --git a/test/ably/rest/restauth_test.py b/test/ably/rest/restauth_test.py index 9e5494c3..d2dd834b 100644 --- a/test/ably/rest/restauth_test.py +++ b/test/ably/rest/restauth_test.py @@ -496,14 +496,14 @@ class TestRenewToken(BaseAsyncTestCase): async def asyncSetUp(self): self.test_vars = await TestApp.get_test_vars() - self.ably = await TestApp.get_ably_rest(use_binary_protocol=False) + self.host = 'fake-host.ably.io' + self.ably = await TestApp.get_ably_rest(use_binary_protocol=False, rest_host=self.host) # with headers self.publish_attempts = 0 self.channel = uuid.uuid4().hex - host = self.test_vars['host'] tokens = ['a_token', 'another_token'] headers = {'Content-Type': 'application/json'} - self.mocked_api = respx.mock(base_url='https://{}'.format(host)) + self.mocked_api = respx.mock(base_url='https://{}'.format(self.host)) self.request_token_route = self.mocked_api.post( "/keys/{}/requestToken".format(self.test_vars["keys"][0]['key_name']), name="request_token_route") @@ -561,6 +561,7 @@ async def test_when_not_renewable(self): self.ably = await TestApp.get_ably_rest( key=None, + rest_host=self.host, token='token ID cannot be used to create a new token', use_binary_protocol=False) await self.ably.channels[self.channel].publish('evt', 'msg') @@ -579,6 +580,7 @@ async def test_when_not_renewable_with_token_details(self): token_details = TokenDetails(token='a_dummy_token') self.ably = await TestApp.get_ably_rest( key=None, + rest_host=self.host, token_details=token_details, use_binary_protocol=False) await self.ably.channels[self.channel].publish('evt', 'msg') @@ -600,11 +602,11 @@ async def asyncSetUp(self): self.publish_attempts = 0 self.channel = uuid.uuid4().hex - host = self.test_vars['host'] + self.host = 'fake-host.ably.io' key = self.test_vars["keys"][0]['key_name'] headers = {'Content-Type': 'application/json'} - self.mocked_api = respx.mock(base_url='https://{}'.format(host)) + self.mocked_api = respx.mock(base_url='https://{}'.format(self.host)) self.request_token_route = self.mocked_api.post("/keys/{}/requestToken".format(key), name="request_token_route") self.request_token_route.return_value = Response( @@ -648,7 +650,7 @@ async def asyncTearDown(self): # RSA4b1 async def test_query_time_false(self): - ably = await TestApp.get_ably_rest() + ably = await TestApp.get_ably_rest(rest_host=self.host) await ably.auth.authorize() self.publish_fail = True await ably.channels[self.channel].publish('evt', 'msg') @@ -657,7 +659,7 @@ async def test_query_time_false(self): # RSA4b1 async def test_query_time_true(self): - ably = await TestApp.get_ably_rest(query_time=True) + ably = await TestApp.get_ably_rest(query_time=True, rest_host=self.host) await ably.auth.authorize() self.publish_fail = False await ably.channels[self.channel].publish('evt', 'msg') diff --git a/test/ably/rest/restchannelhistory_test.py b/test/ably/rest/restchannelhistory_test.py index 30d94e91..d1ea1591 100644 --- a/test/ably/rest/restchannelhistory_test.py +++ b/test/ably/rest/restchannelhistory_test.py @@ -14,7 +14,7 @@ class TestRestChannelHistory(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): async def asyncSetUp(self): - self.ably = await TestApp.get_ably_rest() + self.ably = await TestApp.get_ably_rest(fallback_hosts=[]) self.test_vars = await TestApp.get_test_vars() async def asyncTearDown(self): diff --git a/test/ably/testapp.py b/test/ably/testapp.py index 80bfe925..86741f3c 100644 --- a/test/ably/testapp.py +++ b/test/ably/testapp.py @@ -14,20 +14,21 @@ app_spec_local = json.loads(f.read()) tls = (os.environ.get('ABLY_TLS') or "true").lower() == "true" -host = os.environ.get('ABLY_HOST', 'sandbox-rest.ably.io') -realtime_host = os.environ.get('ABLY_HOST', 'sandbox-realtime.ably.io') -environment = os.environ.get('ABLY_ENV') +rest_host = os.environ.get('ABLY_REST_HOST', 'sandbox-rest.ably.io') +realtime_host = os.environ.get('ABLY_REALTIME_HOST', 'sandbox-realtime.ably.io') + +environment = os.environ.get('ABLY_ENV', 'sandbox') port = 80 tls_port = 443 -if host and not host.endswith("rest.ably.io"): - tls = tls and host != "localhost" +if rest_host and not rest_host.endswith("rest.ably.io"): + tls = tls and rest_host != "localhost" port = 8080 tls_port = 8081 -ably = AblyRest(token='not_a_real_token', rest_host=host, +ably = AblyRest(token='not_a_real_token', port=port, tls_port=tls_port, tls=tls, environment=environment, use_binary_protocol=False) @@ -48,7 +49,7 @@ async def get_test_vars(): test_vars = { "app_id": app_id, - "host": host, + "host": rest_host, "port": port, "tls_port": tls_port, "tls": tls, @@ -71,14 +72,7 @@ async def get_test_vars(): @staticmethod async def get_ably_rest(**kw): test_vars = await TestApp.get_test_vars() - options = { - 'key': test_vars["keys"][0]["key_str"], - 'rest_host': test_vars["host"], - 'port': test_vars["port"], - 'tls_port': test_vars["tls_port"], - 'tls': test_vars["tls"], - 'environment': test_vars["environment"], - } + options = TestApp.get_options(test_vars, **kw) options.update(kw) return AblyRest(**options) @@ -91,8 +85,6 @@ async def get_ably_realtime(**kw): @staticmethod def get_options(test_vars, **kwargs): options = { - 'realtime_host': test_vars["realtime_host"], - 'rest_host': test_vars["host"], 'port': test_vars["port"], 'tls_port': test_vars["tls_port"], 'tls': test_vars["tls"], @@ -102,7 +94,11 @@ def get_options(test_vars, **kwargs): if not any(x in kwargs for x in auth_methods): options["key"] = test_vars["keys"][0]["key_str"] + if any(x in kwargs for x in ["rest_host", "realtime_host"]): + options["environment"] = None + options.update(kwargs) + return options @staticmethod From cfd5d2a36a0713f942f2beb41948d259fe933b78 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 16 Mar 2023 17:49:00 +0000 Subject: [PATCH 461/481] refactor!: remove `Auth.authorise` --- ably/rest/auth.py | 7 ------- test/ably/rest/restauth_test.py | 19 ++----------------- 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/ably/rest/auth.py b/ably/rest/auth.py index 15eaf166..06af2438 100644 --- a/ably/rest/auth.py +++ b/ably/rest/auth.py @@ -5,7 +5,6 @@ import time from typing import Optional, TYPE_CHECKING, Union import uuid -import warnings import httpx from ably.types.options import Options @@ -152,12 +151,6 @@ def token_details_has_expired(self): async def authorize(self, token_params: Optional[dict] = None, auth_options=None): return await self.__authorize_when_necessary(token_params, auth_options, force=True) - async def authorise(self, *args, **kwargs): - warnings.warn( - "authorise is deprecated and will be removed in v2.0, please use authorize", - DeprecationWarning) - return await self.authorize(*args, **kwargs) - async def request_token(self, token_params: Optional[dict] = None, # auth_options key_name: Optional[str] = None, key_secret: Optional[str] = None, auth_callback=None, diff --git a/test/ably/rest/restauth_test.py b/test/ably/rest/restauth_test.py index d2dd834b..a6ac0ceb 100644 --- a/test/ably/rest/restauth_test.py +++ b/test/ably/rest/restauth_test.py @@ -4,7 +4,6 @@ import uuid import base64 -import warnings from urllib.parse import parse_qs import mock import pytest @@ -183,7 +182,7 @@ async def test_if_authorize_changes_auth_mechanism_to_token(self): await self.ably.auth.authorize() - assert Auth.Method.TOKEN == self.ably.auth.auth_mechanism, "Authorise should change the Auth method" + assert Auth.Method.TOKEN == self.ably.auth.auth_mechanism, "Authorize should change the Auth method" # RSA10a @dont_vary_protocol @@ -217,7 +216,7 @@ async def test_authorize_adheres_to_request_token(self): token_called, auth_called = request_mock.call_args assert token_called[0] == token_params - # Authorise may call request_token with some default auth_options. + # Authorize may call request_token with some default auth_options. for arg, value in auth_params.items(): assert auth_called[arg] == value, "%s called with wrong value: %s" % (arg, value) @@ -319,20 +318,6 @@ async def test_client_id_precedence(self): assert history.items[0].client_id == client_id await ably.close() - # RSA10l - @dont_vary_protocol - async def test_authorise(self): - with warnings.catch_warnings(record=True) as ws: - # Cause all warnings to always be triggered - warnings.simplefilter("always") - - token = await self.ably.auth.authorise() - assert isinstance(token, TokenDetails) - - # Verify warning is raised - ws = [w for w in ws if issubclass(w.category, DeprecationWarning)] - assert len(ws) == 1 - class TestRequestToken(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): From 82f874bcf4c3a103e14120910d254acd9e8dfd56 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 16 Mar 2023 17:53:06 +0000 Subject: [PATCH 462/481] refactor!: remove `Options.fallback_hosts_use_default` --- ably/types/options.py | 28 ++++------------------------ test/ably/rest/resthttp_test.py | 4 +--- test/ably/rest/restinit_test.py | 17 ----------------- test/ably/rest/restrequest_test.py | 27 ++++++++++++++------------- 4 files changed, 19 insertions(+), 57 deletions(-) diff --git a/ably/types/options.py b/ably/types/options.py index 676b5473..61b1a848 100644 --- a/ably/types/options.py +++ b/ably/types/options.py @@ -1,5 +1,4 @@ import random -import warnings import logging from ably.transport.defaults import Defaults @@ -13,9 +12,9 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti tls_port=0, use_binary_protocol=True, queue_messages=False, recover=False, environment=None, http_open_timeout=None, http_request_timeout=None, realtime_request_timeout=None, http_max_retry_count=None, http_max_retry_duration=None, fallback_hosts=None, - fallback_hosts_use_default=None, fallback_retry_timeout=None, disconnected_retry_timeout=None, - idempotent_rest_publishing=None, loop=None, auto_connect=True, suspended_retry_timeout=None, - connectivity_check_url=None, channel_retry_timeout=Defaults.channel_retry_timeout, **kwargs): + fallback_retry_timeout=None, disconnected_retry_timeout=None, idempotent_rest_publishing=None, + loop=None, auto_connect=True, suspended_retry_timeout=None, connectivity_check_url=None, + channel_retry_timeout=Defaults.channel_retry_timeout, **kwargs): super().__init__(**kwargs) # TODO check these defaults @@ -66,7 +65,6 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti self.__http_max_retry_count = http_max_retry_count self.__http_max_retry_duration = http_max_retry_duration self.__fallback_hosts = fallback_hosts - self.__fallback_hosts_use_default = fallback_hosts_use_default self.__fallback_retry_timeout = fallback_retry_timeout self.__disconnected_retry_timeout = disconnected_retry_timeout self.__channel_retry_timeout = channel_retry_timeout @@ -206,10 +204,6 @@ def http_max_retry_duration(self, value): def fallback_hosts(self): return self.__fallback_hosts - @property - def fallback_hosts_use_default(self): - return self.__fallback_hosts_use_default - @property def fallback_retry_timeout(self): return self.__fallback_retry_timeout @@ -283,27 +277,13 @@ def __get_rest_hosts(self): # Fallback hosts fallback_hosts = self.fallback_hosts if fallback_hosts is None: - if host == Defaults.rest_host or self.fallback_hosts_use_default: + if host == Defaults.rest_host: fallback_hosts = Defaults.fallback_hosts elif environment != 'production': fallback_hosts = Defaults.get_environment_fallback_hosts(environment) else: fallback_hosts = [] - # Explicit warning about deprecating the option - if self.fallback_hosts_use_default: - if environment != Defaults.environment: - warnings.warn( - "It is no longer required to set 'fallback_hosts_use_default', the correct fallback hosts " - "are now inferred from the environment, 'fallback_hosts': {}" - .format(','.join(fallback_hosts)), DeprecationWarning - ) - else: - warnings.warn( - "It is no longer required to set 'fallback_hosts_use_default': 'fallback_hosts': {}" - .format(','.join(fallback_hosts)), DeprecationWarning - ) - # Shuffle fallback_hosts = list(fallback_hosts) random.shuffle(fallback_hosts) diff --git a/test/ably/rest/resthttp_test.py b/test/ably/rest/resthttp_test.py index bab45344..db219b53 100644 --- a/test/ably/rest/resthttp_test.py +++ b/test/ably/rest/resthttp_test.py @@ -94,11 +94,9 @@ async def test_no_host_fallback_nor_retries_if_custom_host(self): await ably.close() # RSC15f - # Ignore library warning regarding fallback_hosts_use_default - @pytest.mark.filterwarnings('ignore::DeprecationWarning') async def test_cached_fallback(self): timeout = 2000 - ably = await TestApp.get_ably_rest(fallback_hosts_use_default=True, fallback_retry_timeout=timeout) + ably = await TestApp.get_ably_rest(fallback_retry_timeout=timeout) host = ably.options.get_rest_host() state = {'errors': 0} diff --git a/test/ably/rest/restinit_test.py b/test/ably/rest/restinit_test.py index 88a433da..10dd8282 100644 --- a/test/ably/rest/restinit_test.py +++ b/test/ably/rest/restinit_test.py @@ -1,4 +1,3 @@ -import warnings from mock import patch import pytest from httpx import AsyncClient @@ -91,8 +90,6 @@ def test_rest_host_and_environment(self): # RSC15 @dont_vary_protocol - # Ignore library warning regarding fallback_hosts_use_default - @pytest.mark.filterwarnings('ignore::DeprecationWarning') def test_fallback_hosts(self): # Specify the fallback_hosts (RSC15a) fallback_hosts = [ @@ -114,26 +111,12 @@ def test_fallback_hosts(self): ably = AblyRest(token='foo', http_max_retry_count=10) assert sorted(Defaults.fallback_hosts) == sorted(ably.options.get_fallback_rest_hosts()) - # Specify environment and fallback_hosts_use_default, no fallback hosts (RSC15g4) - # We specify http_max_retry_count=10 so all the fallback hosts get in the list - ably = AblyRest(token='foo', environment='not_considered', fallback_hosts_use_default=True, - http_max_retry_count=10) - assert sorted(Defaults.fallback_hosts) == sorted(ably.options.get_fallback_rest_hosts()) - # RSC15f ably = AblyRest(token='foo') assert 600000 == ably.options.fallback_retry_timeout ably = AblyRest(token='foo', fallback_retry_timeout=1000) assert 1000 == ably.options.fallback_retry_timeout - with warnings.catch_warnings(record=True) as ws: - # Cause all warnings to always be triggered - warnings.simplefilter("always") - AblyRest(token='foo', fallback_hosts_use_default=True) - # Verify warning is raised for fallback_hosts_use_default - ws = [w for w in ws if issubclass(w.category, DeprecationWarning)] - assert len(ws) == 1 - @dont_vary_protocol def test_specified_realtime_host(self): ably = AblyRest(token='foo', realtime_host="some.other.host") diff --git a/test/ably/rest/restrequest_test.py b/test/ably/rest/restrequest_test.py index 78702bc5..2824d570 100644 --- a/test/ably/rest/restrequest_test.py +++ b/test/ably/rest/restrequest_test.py @@ -1,5 +1,6 @@ import httpx import pytest +import respx from ably import AblyRest from ably.http.paginatedresult import HttpPaginatedResponse @@ -90,8 +91,6 @@ async def test_headers(self): # RSC19e @dont_vary_protocol - # Ignore library warning regarding fallback_hosts_use_default - @pytest.mark.filterwarnings('ignore::DeprecationWarning') async def test_timeout(self): # Timeout timeout = 0.000001 @@ -101,17 +100,19 @@ async def test_timeout(self): await ably.request('GET', '/time') await ably.close() - # Bad host, use fallback - ably = AblyRest(key=self.test_vars["keys"][0]["key_str"], - rest_host='some.other.host', - port=self.test_vars["port"], - tls_port=self.test_vars["tls_port"], - tls=self.test_vars["tls"], - fallback_hosts_use_default=True) - result = await ably.request('GET', '/time') - assert isinstance(result, HttpPaginatedResponse) - assert len(result.items) == 1 - assert isinstance(result.items[0], int) + default_endpoint = 'https://sandbox-rest.ably.io/time' + fallback_host = 'sandbox-a-fallback.ably-realtime.com' + fallback_endpoint = f'https://{fallback_host}/time' + ably = await TestApp.get_ably_rest(fallback_hosts=[fallback_host]) + with respx.mock: + default_route = respx.get(default_endpoint) + fallback_route = respx.get(fallback_endpoint) + headers = { + "Content-Type": "application/json" + } + default_route.side_effect = httpx.ConnectError('') + fallback_route.return_value = httpx.Response(200, headers=headers, text='[123]') + await ably.request('GET', '/time') await ably.close() # Bad host, no Fallback From f6b8c18ed9c2366c4aee48bc6a9dd2f88aff4033 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 16 Mar 2023 17:56:49 +0000 Subject: [PATCH 463/481] refactor!: raise exception when `get_default_params` called without params --- ably/util/crypto.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ably/util/crypto.py b/ably/util/crypto.py index 3ed24f24..acd558b6 100644 --- a/ably/util/crypto.py +++ b/ably/util/crypto.py @@ -142,10 +142,8 @@ def generate_random_key(length=DEFAULT_KEYLENGTH): def get_default_params(params=None): - # Backwards compatibility if type(params) in [str, bytes]: - log.warning("Calling get_default_params with a key directly is deprecated, it expects a params dict") - return get_default_params({'key': params}) + raise ValueError("Calling get_default_params with a key directly is deprecated, it expects a params dict") key = params.get('key') algorithm = params.get('algorithm') or 'AES' From 3307d89dd8bda03be7f40dce0303e1275eb3f47d Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 20 Mar 2023 15:17:04 +0000 Subject: [PATCH 464/481] test: fix flakey idempotent publishing test --- test/ably/rest/restchannelpublish_test.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/ably/rest/restchannelpublish_test.py b/test/ably/rest/restchannelpublish_test.py index 48f18c3b..6cf458eb 100644 --- a/test/ably/rest/restchannelpublish_test.py +++ b/test/ably/rest/restchannelpublish_test.py @@ -529,10 +529,8 @@ def get_ably_rest(self, *args, **kwargs): # RSL1k4 async def test_idempotent_library_generated_retry(self): - ably = await self.get_ably_rest(idempotent_rest_publishing=True) - if not ably.options.fallback_hosts: - host = ably.options.get_rest_host() - ably = await self.get_ably_rest(idempotent_rest_publishing=True, fallback_hosts=[host] * 3) + test_vars = await TestApp.get_test_vars() + ably = await self.get_ably_rest(idempotent_rest_publishing=True, fallback_hosts=[test_vars["host"]] * 3) channel = ably.channels[self.get_channel_name()] state = {'failures': 0} From 1e7c7dc66c39abedd9ea3a529626e776eb75f8cc Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 27 Mar 2023 17:58:50 +0100 Subject: [PATCH 465/481] chore: bump version for 2.0.0-beta.5 release --- ably/__init__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ably/__init__.py b/ably/__init__.py index 33265da9..08d5fa5c 100644 --- a/ably/__init__.py +++ b/ably/__init__.py @@ -15,4 +15,4 @@ logger.addHandler(logging.NullHandler()) api_version = '1.2' -lib_version = '2.0.0-beta.4' +lib_version = '2.0.0-beta.5' diff --git a/pyproject.toml b/pyproject.toml index 0ec9f9da..307d6c69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "ably" -version = "2.0.0-beta.4" +version = "2.0.0-beta.5" description = "Python REST and Realtime client library SDK for Ably realtime messaging service" license = "Apache-2.0" authors = ["Ably <support@ably.com>"] From e0449e27f6eba3ce7b6bffb9832fce0dd1fd2552 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 27 Mar 2023 18:05:43 +0100 Subject: [PATCH 466/481] docs: update CHANGELOG for 2.0.0-beta.5 release --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fbcfadf..4b8ad216 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Change Log +## [v2.0.0-beta.5](https://github.com/ably/ably-python/tree/v2.0.0-beta.5) + +The latest beta release of ably-python 2.0 makes some minor breaking changes, removing already soft-deprecated features from the 1.x branch. Most users will not be affected by these changes since the library was already warning that these features were deprecated. For information on how to migrate, please consult the [migration guide](https://github.com/ably/ably-python/blob/main/UPDATING.md). + +[Full Changelog](https://github.com/ably/ably-python/compare/v2.0.0-beta.4...v2.0.0-beta.5) + +- Remove soft-deprecated APIs [\#482](https://github.com/ably/ably-python/issues/482) +- Improve realtime client typings [\#476](https://github.com/ably/ably-python/issues/476) +- Improve REST client typings [\#477](https://github.com/ably/ably-python/issues/477) +- Stop raising `KeyError` when releasing a channel which doesn't exist [\#474](https://github.com/ably/ably-python/issues/474) + ## [v2.0.0-beta.4](https://github.com/ably/ably-python/tree/v2.0.0-beta.4) This new beta release of the ably-python realtime client implements token authentication for realtime connections, allowing you to use all currently supported token options to authenticate a realtime client (auth_url, auth_callback, jwt, etc). The client will reauthenticate when the token expires or otherwise becomes invalid. From 2c203e676e0710dc7b5a8e1866e1e97b97cc70f7 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 3 May 2023 14:14:52 +0100 Subject: [PATCH 467/481] refactor(ConnectionManager): log reason for all state changes --- ably/realtime/connectionmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index 8696b8cd..e31ae25e 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -44,7 +44,7 @@ def __init__(self, realtime: AblyRealtime, initial_state): def enact_state_change(self, state: ConnectionState, reason: Optional[AblyException] = None) -> None: current_state = self.__state - log.info(f'ConnectionManager.enact_state_change(): {current_state} -> {state}') + log.info(f'ConnectionManager.enact_state_change(): {current_state} -> {state}; reason = {reason}') self.__state = state if reason: self.__error_reason = reason From 09bcf750ae2580a717301ccc14dc1d3cbdfde4c4 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 3 May 2023 15:56:01 +0100 Subject: [PATCH 468/481] fix(ConnectionManager): notify state upon transport deactiviation --- ably/realtime/connectionmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index 8696b8cd..114f2bf8 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -243,7 +243,7 @@ def on_heartbeat(self, id: Optional[str]) -> None: def deactivate_transport(self, reason: Optional[AblyException] = None): self.transport = None - self.enact_state_change(ConnectionState.DISCONNECTED, reason) + self.notify_state(ConnectionState.DISCONNECTED, reason) def request_state(self, state: ConnectionState, force=False) -> None: log.info(f'ConnectionManager.request_state(): state = {state}') From aa97420d4ef3fc0fb16cd33a4c547687f2729a5d Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 3 May 2023 16:48:06 +0100 Subject: [PATCH 469/481] test: add test for reconnection after loss of connectivity --- test/ably/realtime/realtimeconnection_test.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/ably/realtime/realtimeconnection_test.py b/test/ably/realtime/realtimeconnection_test.py index 76ba1d1f..31628b97 100644 --- a/test/ably/realtime/realtimeconnection_test.py +++ b/test/ably/realtime/realtimeconnection_test.py @@ -370,3 +370,30 @@ async def test_connection_client_id_query_params(self): assert ably.auth.client_id == client_id await ably.close() + + async def test_lost_connection_lifecycle(self): + ably = await TestApp.get_ably_realtime(realtime_request_timeout=2000, disconnected_retry_timeout=2000) + + # when client connectivity is lost, the transport will become aware of a connectivity issue + # when it stops seeing activity from realtime within maxIdleInterval, therefore setting the max idle + # interval arbitrarily low will simulate client behaviour when connectivity is lost. + def on_transport_pending(transport): + original_on_protocol_message = transport.on_protocol_message + + async def on_protocol_message(msg): + if msg["action"] == ProtocolMessageAction.CONNECTED: + msg["connectionDetails"]["maxIdleInterval"] = 1000 + + await original_on_protocol_message(msg) + + transport.on_protocol_message = on_protocol_message + + ably.connection.connection_manager.once('transport.pending', on_transport_pending) + + # should transition to disconnected due to lack of activity from realtime + await ably.connection.once_async(ConnectionState.DISCONNECTED) + + # should re-establish connection after disconnected_retry_timeout + await ably.connection.once_async(ConnectionState.CONNECTED) + + await ably.close() From 341dcd14a936751c761063d876dc0787d79ee812 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 4 May 2023 17:17:40 +0100 Subject: [PATCH 470/481] chore: bump version for 2.0.0-beta.6 release --- ably/__init__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ably/__init__.py b/ably/__init__.py index 08d5fa5c..4ceb30c5 100644 --- a/ably/__init__.py +++ b/ably/__init__.py @@ -15,4 +15,4 @@ logger.addHandler(logging.NullHandler()) api_version = '1.2' -lib_version = '2.0.0-beta.5' +lib_version = '2.0.0-beta.6' diff --git a/pyproject.toml b/pyproject.toml index 307d6c69..ac5e6c7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "ably" -version = "2.0.0-beta.5" +version = "2.0.0-beta.6" description = "Python REST and Realtime client library SDK for Ably realtime messaging service" license = "Apache-2.0" authors = ["Ably <support@ably.com>"] From 151550efd95170a44fab5879932311f479a27f1b Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 4 May 2023 17:34:55 +0100 Subject: [PATCH 471/481] docs: update changelog for 2.0.0-beta.6 release --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b8ad216..7075b008 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Change Log +## [v2.0.0-beta.6](https://github.com/ably/ably-python/tree/v2.0.0-beta.6) + +[Full Changelog](https://github.com/ably/ably-python/compare/v2.0.0-beta.5...v2.0.0-beta.6) + +- Improve logger output upon disconnection [\#492](https://github.com/ably/ably-python/issues/492) +- Fix an issue where in some cases the client was unable to recover after loss of connectivity [\#493](https://github.com/ably/ably-python/issues/493) + ## [v2.0.0-beta.5](https://github.com/ably/ably-python/tree/v2.0.0-beta.5) The latest beta release of ably-python 2.0 makes some minor breaking changes, removing already soft-deprecated features from the 1.x branch. Most users will not be affected by these changes since the library was already warning that these features were deprecated. For information on how to migrate, please consult the [migration guide](https://github.com/ably/ably-python/blob/main/UPDATING.md). From 7d0cb8f4f11d96d303f5c3a4a425aadbde25350b Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 15 May 2023 18:16:55 +0100 Subject: [PATCH 472/481] feat!: add mandatory version param to `Rest.request` --- ably/http/http.py | 15 ++++++++++----- ably/http/httputils.py | 14 ++++++++------ ably/http/paginatedresult.py | 10 +++++----- ably/rest/rest.py | 8 ++++++-- test/ably/rest/resthttp_test.py | 2 +- test/ably/rest/restrequest_test.py | 24 +++++++++++++++--------- 6 files changed, 45 insertions(+), 28 deletions(-) diff --git a/ably/http/http.py b/ably/http/http.py index 054fe00c..440bf0c6 100644 --- a/ably/http/http.py +++ b/ably/http/http.py @@ -44,18 +44,19 @@ async def wrapper(rest, *args, **kwargs): class Request: - def __init__(self, method='GET', url='/', headers=None, body=None, + def __init__(self, method='GET', url='/', version=None, headers=None, body=None, skip_auth=False, raise_on_error=True): self.__method = method self.__headers = headers or {} self.__body = body self.__skip_auth = skip_auth self.__url = url + self.__version = version self.raise_on_error = raise_on_error def with_relative_url(self, relative_url): url = urljoin(self.url, relative_url) - return Request(self.method, url, self.headers, self.body, + return Request(self.method, url, self.version, self.headers, self.body, self.skip_auth, self.raise_on_error) @property @@ -78,6 +79,10 @@ def body(self): def skip_auth(self): return self.__skip_auth + @property + def version(self): + return self.__version + class Response: """ @@ -152,16 +157,16 @@ def get_rest_hosts(self): return hosts @reauth_if_expired - async def make_request(self, method, path, headers=None, body=None, + async def make_request(self, method, path, version=None, headers=None, body=None, skip_auth=False, timeout=None, raise_on_error=True): if body is not None and type(body) not in (bytes, str): body = self.dump_body(body) if body: - all_headers = HttpUtils.default_post_headers(self.options.use_binary_protocol) + all_headers = HttpUtils.default_post_headers(self.options.use_binary_protocol, version=version) else: - all_headers = HttpUtils.default_get_headers(self.options.use_binary_protocol) + all_headers = HttpUtils.default_get_headers(self.options.use_binary_protocol, version=version) if not skip_auth: if self.auth.auth_mechanism == Auth.Method.BASIC and self.preferred_scheme.lower() == 'http': diff --git a/ably/http/httputils.py b/ably/http/httputils.py index 53a583a1..20c7131e 100644 --- a/ably/http/httputils.py +++ b/ably/http/httputils.py @@ -14,8 +14,8 @@ class HttpUtils: } @staticmethod - def default_get_headers(binary=False): - headers = HttpUtils.default_headers() + def default_get_headers(binary=False, version=None): + headers = HttpUtils.default_headers(version=version) if binary: headers["Accept"] = HttpUtils.mime_types['binary'] else: @@ -23,8 +23,8 @@ def default_get_headers(binary=False): return headers @staticmethod - def default_post_headers(binary=False): - headers = HttpUtils.default_get_headers(binary=binary) + def default_post_headers(binary=False, version=None): + headers = HttpUtils.default_get_headers(binary=binary, version=version) headers["Content-Type"] = headers["Accept"] return headers @@ -35,8 +35,10 @@ def get_host_header(host): } @staticmethod - def default_headers(): + def default_headers(version=None): + if version is None: + version = ably.api_version return { - "X-Ably-Version": ably.api_version, + "X-Ably-Version": version, "Ably-Agent": 'ably-python/%s python/%s' % (ably.lib_version, platform.python_version()) } diff --git a/ably/http/paginatedresult.py b/ably/http/paginatedresult.py index fffcabf1..6421251b 100644 --- a/ably/http/paginatedresult.py +++ b/ably/http/paginatedresult.py @@ -77,11 +77,11 @@ async def __get_rel(self, rel_req): return await self.paginated_query_with_request(self.__http, rel_req, self.__response_processor) @classmethod - async def paginated_query(cls, http, method='GET', url='/', body=None, + async def paginated_query(cls, http, method='GET', url='/', version=None, body=None, headers=None, response_processor=None, raise_on_error=True): headers = headers or {} - req = Request(method, url, body=body, headers=headers, skip_auth=False, + req = Request(method, url, version=version, body=body, headers=headers, skip_auth=False, raise_on_error=raise_on_error) return await cls.paginated_query_with_request(http, req, response_processor) @@ -89,9 +89,9 @@ async def paginated_query(cls, http, method='GET', url='/', body=None, async def paginated_query_with_request(cls, http, request, response_processor, raise_on_error=True): response = await http.make_request( - request.method, request.url, headers=request.headers, - body=request.body, skip_auth=request.skip_auth, - raise_on_error=request.raise_on_error) + request.method, request.url, version=request.version, + headers=request.headers, body=request.body, + skip_auth=request.skip_auth, raise_on_error=request.raise_on_error) items = response_processor(response) diff --git a/ably/rest/rest.py b/ably/rest/rest.py index 64b2c683..a42ba2fd 100644 --- a/ably/rest/rest.py +++ b/ably/rest/rest.py @@ -119,7 +119,11 @@ def options(self): def push(self): return self.__push - async def request(self, method: str, path: str, params: Optional[dict] = None, body=None, headers=None): + async def request(self, method: str, path: str, version: str, params: + Optional[dict] = None, body=None, headers=None): + if version is None: + raise AblyException("No version parameter", 400, 40000) + url = path if params: url += '?' + urlencode(params) @@ -133,7 +137,7 @@ def response_processor(response): return items return await HttpPaginatedResponse.paginated_query( - self.http, method, url, body=body, headers=headers, + self.http, method, url, version=version, body=body, headers=headers, response_processor=response_processor, raise_on_error=False) diff --git a/test/ably/rest/resthttp_test.py b/test/ably/rest/resthttp_test.py index db219b53..4929bdf3 100644 --- a/test/ably/rest/resthttp_test.py +++ b/test/ably/rest/resthttp_test.py @@ -25,7 +25,7 @@ async def test_max_retry_attempts_and_timeouts_defaults(self): with mock.patch('httpx.AsyncClient.send', side_effect=httpx.RequestError('')) as send_mock: with pytest.raises(httpx.RequestError): - await ably.http.make_request('GET', '/', skip_auth=True) + await ably.http.make_request('GET', '/', version=Defaults.protocol_version, skip_auth=True) assert send_mock.call_count == Defaults.http_max_retry_count assert send_mock.call_args == mock.call(mock.ANY) diff --git a/test/ably/rest/restrequest_test.py b/test/ably/rest/restrequest_test.py index 2824d570..d0c9ad9d 100644 --- a/test/ably/rest/restrequest_test.py +++ b/test/ably/rest/restrequest_test.py @@ -4,6 +4,7 @@ from ably import AblyRest from ably.http.paginatedresult import HttpPaginatedResponse +from ably.transport.defaults import Defaults from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase from test.ably.utils import VaryByProtocolTestsMetaclass, dont_vary_protocol @@ -21,7 +22,7 @@ async def asyncSetUp(self): self.path = '/channels/%s/messages' % self.channel for i in range(20): body = {'name': 'event%s' % i, 'data': 'lorem ipsum %s' % i} - await self.ably.request('POST', self.path, body=body) + await self.ably.request('POST', self.path, body=body, version=Defaults.protocol_version) async def asyncTearDown(self): await self.ably.close() @@ -32,7 +33,7 @@ def per_protocol_setup(self, use_binary_protocol): async def test_post(self): body = {'name': 'test-post', 'data': 'lorem ipsum'} - result = await self.ably.request('POST', self.path, body=body) + result = await self.ably.request('POST', self.path, body=body, version=Defaults.protocol_version) assert isinstance(result, HttpPaginatedResponse) # RSC19d # HP3 @@ -43,7 +44,7 @@ async def test_post(self): async def test_get(self): params = {'limit': 10, 'direction': 'forwards'} - result = await self.ably.request('GET', self.path, params=params) + result = await self.ably.request('GET', self.path, params=params, version=Defaults.protocol_version) assert isinstance(result, HttpPaginatedResponse) # RSC19d @@ -68,7 +69,7 @@ async def test_get(self): @dont_vary_protocol async def test_not_found(self): - result = await self.ably.request('GET', '/not-found') + result = await self.ably.request('GET', '/not-found', version=Defaults.protocol_version) assert isinstance(result, HttpPaginatedResponse) # RSC19d assert result.status_code == 404 # HP4 assert result.success is False # HP5 @@ -76,7 +77,7 @@ async def test_not_found(self): @dont_vary_protocol async def test_error(self): params = {'limit': 'abc'} - result = await self.ably.request('GET', self.path, params=params) + result = await self.ably.request('GET', self.path, params=params, version=Defaults.protocol_version) assert isinstance(result, HttpPaginatedResponse) # RSC19d assert result.status_code == 400 # HP4 assert not result.success @@ -86,7 +87,7 @@ async def test_error(self): async def test_headers(self): key = 'X-Test' value = 'lorem ipsum' - result = await self.ably.request('GET', '/time', headers={key: value}) + result = await self.ably.request('GET', '/time', headers={key: value}, version=Defaults.protocol_version) assert result.response.request.headers[key] == value # RSC19e @@ -97,7 +98,7 @@ async def test_timeout(self): ably = AblyRest(token="foo", http_request_timeout=timeout) assert ably.http.http_request_timeout == timeout with pytest.raises(httpx.ReadTimeout): - await ably.request('GET', '/time') + await ably.request('GET', '/time', version=Defaults.protocol_version) await ably.close() default_endpoint = 'https://sandbox-rest.ably.io/time' @@ -112,7 +113,7 @@ async def test_timeout(self): } default_route.side_effect = httpx.ConnectError('') fallback_route.return_value = httpx.Response(200, headers=headers, text='[123]') - await ably.request('GET', '/time') + await ably.request('GET', '/time', version=Defaults.protocol_version) await ably.close() # Bad host, no Fallback @@ -122,5 +123,10 @@ async def test_timeout(self): tls_port=self.test_vars["tls_port"], tls=self.test_vars["tls"]) with pytest.raises(httpx.ConnectError): - await ably.request('GET', '/time') + await ably.request('GET', '/time', version=Defaults.protocol_version) await ably.close() + + async def test_version(self): + version = "150" # chosen arbitrarily + result = await self.ably.request('GET', '/time', "150") + assert result.response.request.headers["X-Ably-Version"] == version From a70c47d1d719f346f2359ad8723246dfb718a673 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 25 May 2023 12:44:54 +0100 Subject: [PATCH 473/481] feat: bump api_version to 2.0, add DeviceDetails.deviceSecret --- ably/__init__.py | 2 +- ably/types/device.py | 9 +++++++-- test/ably/rest/resthttp_test.py | 2 +- test/ably/rest/restpush_test.py | 1 + 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ably/__init__.py b/ably/__init__.py index 4ceb30c5..793818f3 100644 --- a/ably/__init__.py +++ b/ably/__init__.py @@ -14,5 +14,5 @@ logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) -api_version = '1.2' +api_version = '2.0' lib_version = '2.0.0-beta.6' diff --git a/ably/types/device.py b/ably/types/device.py index ea35c269..337de002 100644 --- a/ably/types/device.py +++ b/ably/types/device.py @@ -10,7 +10,7 @@ class DeviceDetails: def __init__(self, id, client_id=None, form_factor=None, metadata=None, platform=None, push=None, update_token=None, app_id=None, - device_identity_token=None, modified=None): + device_identity_token=None, modified=None, device_secret=None): if push: recipient = push.get('recipient') @@ -35,6 +35,7 @@ def __init__(self, id, client_id=None, form_factor=None, metadata=None, self.__app_id = app_id self.__device_identity_token = device_identity_token self.__modified = modified + self.__device_secret = device_secret @property def id(self): @@ -76,9 +77,13 @@ def device_identity_token(self): def modified(self): return self.__modified + @property + def device_secret(self): + return self.__device_secret + def as_dict(self): keys = ['id', 'client_id', 'form_factor', 'metadata', 'platform', - 'push', 'update_token', 'app_id', 'device_identity_token', 'modified'] + 'push', 'update_token', 'app_id', 'device_identity_token', 'modified', 'device_secret'] obj = {} for key in keys: diff --git a/test/ably/rest/resthttp_test.py b/test/ably/rest/resthttp_test.py index 4929bdf3..9aa512f2 100644 --- a/test/ably/rest/resthttp_test.py +++ b/test/ably/rest/resthttp_test.py @@ -187,7 +187,7 @@ async def test_request_headers(self): # API assert 'X-Ably-Version' in r.request.headers - assert r.request.headers['X-Ably-Version'] == '1.2' + assert r.request.headers['X-Ably-Version'] == '2.0' # Agent assert 'Ably-Agent' in r.request.headers diff --git a/test/ably/rest/restpush_test.py b/test/ably/rest/restpush_test.py index acbe05a7..f4a6a81a 100644 --- a/test/ably/rest/restpush_test.py +++ b/test/ably/rest/restpush_test.py @@ -57,6 +57,7 @@ def gen_device_data(self, data=None, **kw): 'clientId': self.get_client_id(), 'platform': random.choice(['android', 'ios']), 'formFactor': 'phone', + 'deviceSecret': 'test-secret', 'push': { 'recipient': { 'transportType': 'apns', From 31776d05bcb3f386b6ff646bd1689134b43ad568 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 25 May 2023 16:28:03 +0100 Subject: [PATCH 474/481] refactor: include cause in AblyException.__str__ result --- ably/util/exceptions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ably/util/exceptions.py b/ably/util/exceptions.py index 61864198..8b98c5ee 100644 --- a/ably/util/exceptions.py +++ b/ably/util/exceptions.py @@ -19,7 +19,10 @@ def __init__(self, message, status_code, code, cause=None): self.cause = cause def __str__(self): - return '%s %s %s' % (self.code, self.status_code, self.message) + str = '%s %s %s' % (self.code, self.status_code, self.message) + if self.cause is not None: + str += ' (cause: %s)' % self.cause + return str @property def is_server_error(self): From 022f772449397d10d3d47bfbd94efa8f94c6c9cc Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Mon, 5 Jun 2023 14:58:38 +0100 Subject: [PATCH 475/481] refactor: use integer api_version --- ably/__init__.py | 2 +- test/ably/rest/resthttp_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ably/__init__.py b/ably/__init__.py index 793818f3..37e2acb5 100644 --- a/ably/__init__.py +++ b/ably/__init__.py @@ -14,5 +14,5 @@ logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) -api_version = '2.0' +api_version = '2' lib_version = '2.0.0-beta.6' diff --git a/test/ably/rest/resthttp_test.py b/test/ably/rest/resthttp_test.py index 9aa512f2..79daffee 100644 --- a/test/ably/rest/resthttp_test.py +++ b/test/ably/rest/resthttp_test.py @@ -187,7 +187,7 @@ async def test_request_headers(self): # API assert 'X-Ably-Version' in r.request.headers - assert r.request.headers['X-Ably-Version'] == '2.0' + assert r.request.headers['X-Ably-Version'] == '2' # Agent assert 'Ably-Agent' in r.request.headers From 2d65f1c44744161ac15ee55798e0ad244bf018e9 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 25 May 2023 12:04:59 +0100 Subject: [PATCH 476/481] feat!: use api v3 and untyped stats --- ably/__init__.py | 2 +- ably/types/stats.py | 133 +++---------------------------- test/ably/rest/resthttp_test.py | 2 +- test/ably/rest/reststats_test.py | 63 +++++++-------- 4 files changed, 42 insertions(+), 158 deletions(-) diff --git a/ably/__init__.py b/ably/__init__.py index 37e2acb5..c708a2f8 100644 --- a/ably/__init__.py +++ b/ably/__init__.py @@ -14,5 +14,5 @@ logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) -api_version = '2' +api_version = '3' lib_version = '2.0.0-beta.6' diff --git a/ably/types/stats.py b/ably/types/stats.py index 02b6d4d4..ead5e548 100644 --- a/ably/types/stats.py +++ b/ably/types/stats.py @@ -4,137 +4,28 @@ log = logging.getLogger(__name__) -class ResourceCount: - def __init__(self, opened=0, peak=0, mean=0, min=0, refused=0): - self.opened = opened - self.peak = peak - self.mean = mean - self.min = min - self.refused = refused - - @staticmethod - def from_dict(rc_dict): - rc_dict = rc_dict or {} - expected = ['opened', 'peak', 'mean', 'min', 'refused'] - kwargs = {k: rc_dict[k] for k in rc_dict if (k in expected)} - - return ResourceCount(**kwargs) - - -class ConnectionTypes: - def __init__(self, all=None, plain=None, tls=None): - self.all = all or ResourceCount() - self.plain = plain or ResourceCount() - self.tls = tls or ResourceCount() - - @staticmethod - def from_dict(ct_dict): - ct_dict = ct_dict or {} - kwargs = { - "all": ResourceCount.from_dict(ct_dict.get("all")), - "plain": ResourceCount.from_dict(ct_dict.get("plain")), - "tls": ResourceCount.from_dict(ct_dict.get("tls")), - } - return ConnectionTypes(**kwargs) - - -class MessageCount: - def __init__(self, count=0, data=0): - self.count = count - self.data = data - - @staticmethod - def from_dict(mc_dict): - mc_dict = mc_dict or {} - expected = ['count', 'data'] - kwargs = {k: mc_dict[k] for k in mc_dict if (k in expected)} - return MessageCount(**kwargs) - - -class MessageTypes: - def __init__(self, all=None, messages=None, presence=None): - self.all = all or MessageCount() - self.messages = messages or MessageCount() - self.presence = presence or MessageCount() - - @staticmethod - def from_dict(mt_dict): - mt_dict = mt_dict or {} - kwargs = { - "all": MessageCount.from_dict(mt_dict.get("all")), - "messages": MessageCount.from_dict(mt_dict.get("messages")), - "presence": MessageCount.from_dict(mt_dict.get("presence")), - } - return MessageTypes(**kwargs) - - -class MessageTraffic: - def __init__(self, all=None, realtime=None, rest=None, webhook=None): - self.all = all or MessageTypes() - self.realtime = realtime or MessageTypes() - self.rest = rest or MessageTypes() - self.webhook = webhook or MessageTypes() - - @staticmethod - def from_dict(mt_dict): - mt_dict = mt_dict or {} - kwargs = { - "all": MessageTypes.from_dict(mt_dict.get("all")), - "realtime": MessageTypes.from_dict(mt_dict.get("realtime")), - "rest": MessageTypes.from_dict(mt_dict.get("rest")), - "webhook": MessageTypes.from_dict(mt_dict.get("webhook")), - } - return MessageTraffic(**kwargs) - - -class RequestCount: - def __init__(self, succeeded=0, failed=0, refused=0): - self.succeeded = succeeded - self.failed = failed - self.refused = refused - - @staticmethod - def from_dict(rc_dict): - rc_dict = rc_dict or {} - expected = ['succeeded', 'failed', 'refused'] - kwargs = {k: rc_dict[k] for k in rc_dict if (k in expected)} - return RequestCount(**kwargs) - - class Stats: - def __init__(self, all=None, inbound=None, outbound=None, persisted=None, - connections=None, channels=None, api_requests=None, - token_requests=None, interval_granularity=None, - interval_id=None): - self.all = all or MessageTypes() - self.inbound = inbound or MessageTraffic() - self.outbound = outbound or MessageTraffic() - self.persisted = persisted or MessageTypes() - self.connections = connections or ConnectionTypes() - self.channels = channels or ResourceCount() - self.api_requests = api_requests or RequestCount() - self.token_requests = token_requests or RequestCount() + def __init__(self, entries=None, unit=None, interval_id=None, in_progress=None, app_id=None, schema=None): self.interval_id = interval_id or '' - self.interval_granularity = (interval_granularity or - granularity_from_interval_id(self.interval_id)) + self.entries = entries + self.unit = unit self.interval_time = interval_from_interval_id(self.interval_id) + self.in_progress = in_progress + self.app_id = app_id + self.schema = schema @classmethod def from_dict(cls, stats_dict): stats_dict = stats_dict or {} kwargs = { - "all": MessageTypes.from_dict(stats_dict.get("all")), - "inbound": MessageTraffic.from_dict(stats_dict.get("inbound")), - "outbound": MessageTraffic.from_dict(stats_dict.get("outbound")), - "persisted": MessageTypes.from_dict(stats_dict.get("persisted")), - "connections": ConnectionTypes.from_dict(stats_dict.get("connections")), - "channels": ResourceCount.from_dict(stats_dict.get("channels")), - "api_requests": RequestCount.from_dict(stats_dict.get("apiRequests")), - "token_requests": RequestCount.from_dict(stats_dict.get("tokenRequests")), - "interval_granularity": stats_dict.get("unit"), - "interval_id": stats_dict.get("intervalId") + "entries": stats_dict.get("entries"), + "unit": stats_dict.get("unit"), + "interval_id": stats_dict.get("intervalId"), + "in_progress": stats_dict.get("inProgress"), + "app_id": stats_dict.get("appId"), + "schema": stats_dict.get("schema"), } return cls(**kwargs) diff --git a/test/ably/rest/resthttp_test.py b/test/ably/rest/resthttp_test.py index 79daffee..7230829b 100644 --- a/test/ably/rest/resthttp_test.py +++ b/test/ably/rest/resthttp_test.py @@ -187,7 +187,7 @@ async def test_request_headers(self): # API assert 'X-Ably-Version' in r.request.headers - assert r.request.headers['X-Ably-Version'] == '2' + assert r.request.headers['X-Ably-Version'] == '3' # Agent assert 'Ably-Agent' in r.request.headers diff --git a/test/ably/rest/reststats_test.py b/test/ably/rest/reststats_test.py index 2b612ade..ca0547b8 100644 --- a/test/ably/rest/reststats_test.py +++ b/test/ably/rest/reststats_test.py @@ -98,14 +98,14 @@ async def test_stats_are_forward(self): stats_pages = await self.ably.stats(**self.get_params()) stats = stats_pages.items stat = stats[0] - assert stat.inbound.realtime.all.count == 50 + assert stat.entries["messages.inbound.realtime.all.count"] == 50 async def test_three_pages(self): stats_pages = await self.ably.stats(**self.get_params()) assert not stats_pages.is_last() page2 = await stats_pages.next() page3 = await page2.next() - assert page3.items[0].inbound.realtime.all.count == 70 + assert page3.items[0].entries["messages.inbound.realtime.all.count"] == 70 class TestDirectionBackwards(TestRestAppStatsSetup, BaseAsyncTestCase, @@ -123,7 +123,7 @@ async def test_stats_are_forward(self): stats_pages = await self.ably.stats(**self.get_params()) stats = stats_pages.items stat = stats[0] - assert stat.inbound.realtime.all.count == 70 + assert stat.entries["messages.inbound.realtime.all.count"] == 70 async def test_three_pages(self): stats_pages = await self.ably.stats(**self.get_params()) @@ -131,7 +131,7 @@ async def test_three_pages(self): page2 = await stats_pages.next() page3 = await page2.next() assert not stats_pages.is_last() - assert page3.items[0].inbound.realtime.all.count == 50 + assert page3.items[0].entries["messages.inbound.realtime.all.count"] == 50 class TestOnlyLastYear(TestRestAppStatsSetup, BaseAsyncTestCase, @@ -147,8 +147,8 @@ def get_params(self): async def test_default_is_backwards(self): stats_pages = await self.ably.stats(**self.get_params()) stats = stats_pages.items - assert stats[0].inbound.realtime.messages.count == 70 - assert stats[-1].inbound.realtime.messages.count == 50 + assert stats[0].entries["messages.inbound.realtime.messages.count"] == 70 + assert stats[-1].entries["messages.inbound.realtime.messages.count"] == 50 class TestPreviousYear(TestRestAppStatsSetup, BaseAsyncTestCase, @@ -194,8 +194,8 @@ async def test_units(self): stats_pages = await self.ably.stats(**params) stat = stats_pages.items[0] assert len(stats_pages.items) == 1 - assert stat.all.messages.count == 50 + 20 + 60 + 10 + 70 + 40 - assert stat.all.messages.data == 5000 + 2000 + 6000 + 1000 + 7000 + 4000 + assert stat.entries["messages.all.messages.count"] == 50 + 20 + 60 + 10 + 70 + 40 + assert stat.entries["messages.all.messages.data"] == 5000 + 2000 + 6000 + 1000 + 7000 + 4000 @dont_vary_protocol async def test_when_argument_start_is_after_end(self): @@ -222,96 +222,89 @@ async def test_no_arguments(self): } stats_pages = await self.ably.stats(**params) self.stat = stats_pages.items[0] - assert self.stat.interval_granularity == 'minute' + assert self.stat.unit == 'minute' async def test_got_1_record(self): stats_pages = await self.ably.stats(**self.get_params()) assert 1 == len(stats_pages.items), "Expected 1 record" - async def test_zero_by_default(self): - stats_pages = await self.ably.stats(**self.get_params()) - stats = stats_pages.items - stat = stats[0] - assert stat.channels.refused == 0 - assert stat.outbound.webhook.all.count == 0 - async def test_return_aggregated_message_data(self): # returns aggregated message data stats_pages = await self.ably.stats(**self.get_params()) stats = stats_pages.items stat = stats[0] - assert stat.all.messages.count == 70 + 40 - assert stat.all.messages.data == 7000 + 4000 + assert stat.entries["messages.all.messages.count"] == 70 + 40 + assert stat.entries["messages.all.messages.data"] == 7000 + 4000 async def test_inbound_realtime_all_data(self): # returns inbound realtime all data stats_pages = await self.ably.stats(**self.get_params()) stats = stats_pages.items stat = stats[0] - assert stat.inbound.realtime.all.count == 70 - assert stat.inbound.realtime.all.data == 7000 + assert stat.entries["messages.inbound.realtime.all.count"] == 70 + assert stat.entries["messages.inbound.realtime.all.data"] == 7000 async def test_inboud_realtime_message_data(self): # returns inbound realtime message data stats_pages = await self.ably.stats(**self.get_params()) stats = stats_pages.items stat = stats[0] - assert stat.inbound.realtime.messages.count == 70 - assert stat.inbound.realtime.messages.data == 7000 + assert stat.entries["messages.inbound.realtime.messages.count"] == 70 + assert stat.entries["messages.inbound.realtime.messages.data"] == 7000 async def test_outbound_realtime_all_data(self): # returns outboud realtime all data stats_pages = await self.ably.stats(**self.get_params()) stats = stats_pages.items stat = stats[0] - assert stat.outbound.realtime.all.count == 40 - assert stat.outbound.realtime.all.data == 4000 + assert stat.entries["messages.outbound.realtime.all.count"] == 40 + assert stat.entries["messages.outbound.realtime.all.data"] == 4000 async def test_persisted_data(self): # returns persisted presence all data stats_pages = await self.ably.stats(**self.get_params()) stats = stats_pages.items stat = stats[0] - assert stat.persisted.all.count == 20 - assert stat.persisted.all.data == 2000 + assert stat.entries["messages.persisted.all.count"] == 20 + assert stat.entries["messages.persisted.all.data"] == 2000 async def test_connections_data(self): # returns connections all data stats_pages = await self.ably.stats(**self.get_params()) stats = stats_pages.items stat = stats[0] - assert stat.connections.tls.peak == 20 - assert stat.connections.tls.opened == 10 + assert stat.entries["connections.all.peak"] == 20 + assert stat.entries["connections.all.opened"] == 10 async def test_channels_all_data(self): # returns channels all data stats_pages = await self.ably.stats(**self.get_params()) stats = stats_pages.items stat = stats[0] - assert stat.channels.peak == 50 - assert stat.channels.opened == 30 + assert stat.entries["channels.peak"] == 50 + assert stat.entries["channels.opened"] == 30 async def test_api_requests_data(self): # returns api_requests data stats_pages = await self.ably.stats(**self.get_params()) stats = stats_pages.items stat = stats[0] - assert stat.api_requests.succeeded == 50 - assert stat.api_requests.failed == 10 + assert stat.entries["apiRequests.other.succeeded"] == 50 + assert stat.entries["apiRequests.other.failed"] == 10 async def test_token_requests(self): # returns token_requests data stats_pages = await self.ably.stats(**self.get_params()) stats = stats_pages.items stat = stats[0] - assert stat.token_requests.succeeded == 60 - assert stat.token_requests.failed == 20 + assert stat.entries["apiRequests.tokenRequests.succeeded"] == 60 + assert stat.entries["apiRequests.tokenRequests.failed"] == 20 async def test_interval(self): # interval stats_pages = await self.ably.stats(**self.get_params()) stats = stats_pages.items stat = stats[0] - assert stat.interval_granularity == 'minute' + assert stat.unit == 'minute' assert stat.interval_id == self.last_interval.strftime('%Y-%m-%d:%H:%M') assert stat.interval_time == self.last_interval From 68e578d922d23d3069b8c6efa9fcfc9c140ab1e1 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Wed, 21 Jun 2023 15:59:52 +0100 Subject: [PATCH 477/481] refactor: adjust log levels for connection/channel modules --- ably/realtime/connectionmanager.py | 8 ++++---- ably/realtime/realtime_channel.py | 12 ++++++------ ably/transport/websockettransport.py | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ably/realtime/connectionmanager.py b/ably/realtime/connectionmanager.py index c729176b..eb49b2d6 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/realtime/connectionmanager.py @@ -44,7 +44,7 @@ def __init__(self, realtime: AblyRealtime, initial_state): def enact_state_change(self, state: ConnectionState, reason: Optional[AblyException] = None) -> None: current_state = self.__state - log.info(f'ConnectionManager.enact_state_change(): {current_state} -> {state}; reason = {reason}') + log.debug(f'ConnectionManager.enact_state_change(): {current_state} -> {state}; reason = {reason}') self.__state = state if reason: self.__error_reason = reason @@ -246,7 +246,7 @@ def deactivate_transport(self, reason: Optional[AblyException] = None): self.notify_state(ConnectionState.DISCONNECTED, reason) def request_state(self, state: ConnectionState, force=False) -> None: - log.info(f'ConnectionManager.request_state(): state = {state}') + log.debug(f'ConnectionManager.request_state(): state = {state}') if not force and state == self.state: return @@ -322,7 +322,7 @@ async def try_host(self, host) -> None: future = asyncio.Future() def on_transport_connected(): - log.info('ConnectionManager.try_a_host(): transport connected') + log.debug('ConnectionManager.try_a_host(): transport connected') if self.transport: self.transport.off('failed', on_transport_failed) if not future.done(): @@ -349,7 +349,7 @@ def notify_state(self, state: ConnectionState, reason: Optional[AblyException] = retry_immediately = (retry_immediately is not False) and ( state == ConnectionState.DISCONNECTED and self.__state == ConnectionState.CONNECTED) - log.info( + log.debug( f'ConnectionManager.notify_state(): new state: {state}' + ('; will retry immediately' if retry_immediately else '') ) diff --git a/ably/realtime/realtime_channel.py b/ably/realtime/realtime_channel.py index f9b757d6..1b132c00 100644 --- a/ably/realtime/realtime_channel.py +++ b/ably/realtime/realtime_channel.py @@ -103,7 +103,7 @@ async def attach(self) -> None: raise state_change.reason def _attach_impl(self): - log.info("RealtimeChannel.attach_impl(): sending ATTACH protocol message") + log.debug("RealtimeChannel.attach_impl(): sending ATTACH protocol message") # RTL4c attach_msg = { @@ -169,7 +169,7 @@ async def detach(self) -> None: raise state_change.reason def _detach_impl(self) -> None: - log.info("RealtimeChannel.detach_impl(): sending DETACH protocol message") + log.debug("RealtimeChannel.detach_impl(): sending DETACH protocol message") # RTL5d detach_msg = { @@ -333,13 +333,13 @@ def _on_message(self, msg: dict) -> None: self._notify_state(ChannelState.FAILED, reason=error) def _request_state(self, state: ChannelState) -> None: - log.info(f'RealtimeChannel._request_state(): state = {state}') + log.debug(f'RealtimeChannel._request_state(): state = {state}') self._notify_state(state) self._check_pending_state() def _notify_state(self, state: ChannelState, reason: Optional[AblyException] = None, resumed: bool = False) -> None: - log.info(f'RealtimeChannel._notify_state(): state = {state}') + log.debug(f'RealtimeChannel._notify_state(): state = {state}') self.__clear_state_timer() @@ -380,7 +380,7 @@ def _check_pending_state(self): connection_state = self.__realtime.connection.connection_manager.state if connection_state is not ConnectionState.CONNECTED: - log.info(f"RealtimeChannel._check_pending_state(): connection state = {connection_state}") + log.debug(f"RealtimeChannel._check_pending_state(): connection state = {connection_state}") return if self.state == ChannelState.ATTACHING: @@ -393,7 +393,7 @@ def _check_pending_state(self): def __start_state_timer(self) -> None: if not self.__state_timer: def on_timeout() -> None: - log.info('RealtimeChannel.start_state_timer(): timer expired') + log.debug('RealtimeChannel.start_state_timer(): timer expired') self.__state_timer = None self.__timeout_pending_state() diff --git a/ably/transport/websockettransport.py b/ably/transport/websockettransport.py index c8f8aef0..7c7886fa 100644 --- a/ably/transport/websockettransport.py +++ b/ably/transport/websockettransport.py @@ -93,7 +93,7 @@ async def ws_connect(self, ws_url, headers): async def on_protocol_message(self, msg): self.on_activity() - log.info(f'WebSocketTransport.on_protocol_message(): received protocol message: {msg}') + log.debug(f'WebSocketTransport.on_protocol_message(): received protocol message: {msg}') action = msg.get('action') if action == ProtocolMessageAction.CONNECTED: connection_id = msg.get('connectionId') From 62072c7cd2b915d2f01a8e9f11ba570a1efa7039 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 29 Jun 2023 17:09:08 +0100 Subject: [PATCH 478/481] docs: update README for 2.0 general availability --- README.md | 50 +++++++++++++++++++++----------------------------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index e206e72c..cd12649e 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ ably-python ## Overview -This is a Python client library for Ably. The library currently targets the [Ably 1.1 client library specification](https://ably.com/docs/client-lib-development-guide/features). +This is a Python client library for Ably. The library currently targets the [Ably 2.0 client library specification](https://sdk.ably.com/builds/ably/specification/main/features/). ## Running example @@ -197,27 +197,9 @@ await client.time() await client.close() ``` -## Realtime client (beta) +## Using the realtime client -We currently have a preview version of our first ever Python realtime client available for beta testing. -Currently the realtime client supports basic and token-based authentication and message subscription. -Realtime publishing and realtime presence are upcoming but not yet supported. -The 2.0 beta version contains a few minor breaking changes, removing already soft-deprecated features from the 1.x branch. -Most users will not be affected by these changes since the library was already warning that these features were deprecated. -For information on how to migrate, please consult the [migration guide](./UPDATING.md). -Check out the [roadmap](./roadmap.md) to see our plan for the realtime client. - -### Installing the realtime client - -The beta realtime client is available as a [PyPI](https://pypi.org/project/ably/2.0.0b6/) package. - -``` -pip install ably==2.0.0b6 -``` - -### Using the realtime client - -#### Creating a client using API key +### Create a client using an API key ```python from ably import AblyRealtime @@ -228,7 +210,7 @@ async def main(): client = AblyRealtime('api:key') ``` -#### Create a client using an token auth +### Create a client using token auth ```python # Create a client using kwargs, which must contain at least one auth option @@ -242,7 +224,7 @@ async def main(): client = AblyRealtime(token_details=token_details) ``` -#### Subscribe to connection state changes +### Subscribe to connection state changes ```python # subscribe to 'failed' connection state @@ -278,11 +260,14 @@ await client.connection.once_async() await client.connection.once_async('connected') ``` -#### Get a realtime channel instance +### Get a realtime channel instance + ```python channel = client.channels.get('channel_name') ``` -#### Subscribing to messages on a channel + +### Subscribing to messages on a channel + ```python def listener(message): @@ -294,9 +279,11 @@ await channel.subscribe('event', listener) # Subscribe to all messages on a channel await channel.subscribe(listener) ``` + Note that `channel.subscribe` is a coroutine function and will resolve when the channel is attached -#### Unsubscribing from messages on a channel +### Unsubscribing from messages on a channel + ```python # unsubscribe the listener from the channel channel.unsubscribe('event', listener) @@ -305,16 +292,20 @@ channel.unsubscribe('event', listener) channel.unsubscribe() ``` -#### Attach to a channel +### Attach to a channel + ```python await channel.attach() ``` -#### Detach from a channel + +### Detach from a channel + ```python await channel.detach() ``` -#### Managing a connection +### Managing a connection + ```python # Establish a realtime connection. # Explicitly calling connect() is unnecessary unless the autoConnect attribute of the ClientOptions object is false @@ -326,6 +317,7 @@ await client.close() # Send a ping time_in_ms = await client.connection.ping() ``` + ## Resources Visit https://ably.com/docs for a complete API reference and more examples. From 04ad0b900cb94474dca583c631485cdefb8fdeb9 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 29 Jun 2023 17:17:56 +0100 Subject: [PATCH 479/481] docs: update CHANGELOG for 2.0 release --- CHANGELOG.md | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7075b008..f14adf10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,76 @@ # Change Log +## [v2.0.0](https://github.com/ably/ably-python/tree/v2.0.0) + +**New ably-python realtime client**: This new release features our first ever python realtime client! Currently the realtime client only supports realtime message subscription. Check out the README for usage examples. There have been some minor breaking changes from the 1.2 version, please consult the [migration guide](https://github.com/ably/ably-python/blob/main/UPDATING.md) for instructions on how to upgrade to 2.0. + +[Full Changelog](https://github.com/ably/ably-python/compare/v1.2.2...v2.0.0) + +- refactor!: add mandatory version param to `Rest.request` [\#500](https://github.com/ably/ably-python/issues/500) +- bump api_version to 2.0, add DeviceDetails.deviceSecret [\#507](https://github.com/ably/ably-python/issues/507) +- Include cause in AblyException.__str__ result [\#508](https://github.com/ably/ably-python/issues/508) +- feat!: use api v3 and untyped stats [\#505](https://github.com/ably/ably-python/issues/505) +- Implement `add_request_ids` client option [\#399](https://github.com/ably/ably-python/issues/399) +- Improve logger output upon disconnection [\#492](https://github.com/ably/ably-python/issues/492) +- Fix an issue where in some cases the client was unable to recover after loss of connectivity [\#493](https://github.com/ably/ably-python/issues/493) +- Remove soft-deprecated APIs [\#482](https://github.com/ably/ably-python/issues/482) +- Improve realtime client typings [\#476](https://github.com/ably/ably-python/issues/476) +- Improve REST client typings [\#477](https://github.com/ably/ably-python/issues/477) +- Stop raising `KeyError` when releasing a channel which doesn't exist [\#474](https://github.com/ably/ably-python/issues/474) +- Allow token auth methods for realtime constructor [\#425](https://github.com/ably/ably-python/issues/425) +- Send `AUTH` protocol message when `Auth.authorize` called on realtime client [\#427](https://github.com/ably/ably-python/issues/427) +- Reauth upon inbound `AUTH` protocol message [\#428](https://github.com/ably/ably-python/issues/428) +- Handle connection request failure due to token error [\#445](https://github.com/ably/ably-python/issues/445) +- Handle token `ERROR` response to a resume request [\#444](https://github.com/ably/ably-python/issues/444) +- Handle `DISCONNECTED` messages containing token errors [\#443](https://github.com/ably/ably-python/issues/443) +- Pass `clientId` as query string param when opening a new connection [\#449](https://github.com/ably/ably-python/issues/449) +- Validate `clientId` in `ClientOptions` [\#448](https://github.com/ably/ably-python/issues/448) +- Apply `Auth#clientId` only after a realtime connection has been established [\#409](https://github.com/ably/ably-python/issues/409) +- Channels should transition to `INITIALIZED` if `Connection.connect` called from terminal state [\#411](https://github.com/ably/ably-python/issues/411) +- Calling connect while `CLOSING` should start connect on a new transport [\#410](https://github.com/ably/ably-python/issues/410) +- Handle realtime channel errors [\#455](https://github.com/ably/ably-python/issues/455) +- Resend protocol messages for pending channels upon resume [\#347](https://github.com/ably/ably-python/issues/347) +- Attempt to resume connection when disconnected unexpectedly [\#346](https://github.com/ably/ably-python/issues/346) +- Handle `CONNECTED` messages once connected [\#345](https://github.com/ably/ably-python/issues/345) +- Implement `maxIdleInterval` [\#344](https://github.com/ably/ably-python/issues/344) +- Implement realtime connectivity check [\#343](https://github.com/ably/ably-python/issues/343) +- Use fallback realtime hosts when encountering an appropriate error [\#342](https://github.com/ably/ably-python/issues/342) +- Add `fallbackHosts` client option for realtime clients [\#341](https://github.com/ably/ably-python/issues/341) +- Implement `connectionStateTtl` [\#340](https://github.com/ably/ably-python/issues/340) +- Implement `disconnectedRetryTimeout` [\#339](https://github.com/ably/ably-python/issues/339) +- Handle recoverable connection opening errors [\#338](https://github.com/ably/ably-python/issues/338) +- Implement `channelRetryTimeout` [\#442](https://github.com/ably/ably-python/issues/436) +- Queue protocol messages when connection state is `CONNECTING` or `DISCONNECTED` [\#418](https://github.com/ably/ably-python/issues/418) +- Propagate connection interruptions to realtime channels [\#417](https://github.com/ably/ably-python/issues/417) +- Spec compliance: `Realtime.connect` should be sync [\#413](https://github.com/ably/ably-python/issues/413) +- Emit `update` event on additional `ATTACHED` message [\#386](https://github.com/ably/ably-python/issues/386) +- Set the `ATTACH_RESUME` flag on unclean attach [\#385](https://github.com/ably/ably-python/issues/385) +- Handle fatal resume error [\#384](https://github.com/ably/ably-python/issues/384) +- Handle invalid resume response [\#383](https://github.com/ably/ably-python/issues/383) +- Handle clean resume response [\#382](https://github.com/ably/ably-python/issues/382) +- Send resume query param when reconnecting within `connectionStateTtl` [\#381](https://github.com/ably/ably-python/issues/381) +- Immediately reattempt connection when unexpectedly disconnected [\#380](https://github.com/ably/ably-python/issues/380) +- Clear connection state when `connectionStateTtl` elapsed [\#379](https://github.com/ably/ably-python/issues/379) +- Refactor websocket async tasks into WebSocketTransport class [\#373](https://github.com/ably/ably-python/issues/373) +- Send version transport param [\#368](https://github.com/ably/ably-python/issues/368) +- Clear `Connection.error_reason` when `Connection.connect` is called [\#367](https://github.com/ably/ably-python/issues/367) +- Fix a bug with realtime_host configuration [\#358](https://github.com/ably/ably-python/pull/358) +- Create Basic Api Key connection [\#311](https://github.com/ably/ably-python/pull/311) +- Send Ably-Agent header in realtime connection [\#314](https://github.com/ably/ably-python/pull/314) +- Close client service [\#315](https://github.com/ably/ably-python/pull/315) +- Implement EventEmitter interface on Connection [\#316](https://github.com/ably/ably-python/pull/316) +- Finish tasks gracefully on failed connection [\#317](https://github.com/ably/ably-python/pull/317) +- Implement realtime ping [\#318](https://github.com/ably/ably-python/pull/318) +- Realtime channel attach/detach [\#319](https://github.com/ably/ably-python/pull/319) +- Add `auto_connect` implementation and client option [\#325](https://github.com/ably/ably-python/pull/325) +- RealtimeChannel subscribe/unsubscribe [\#326](https://github.com/ably/ably-python/pull/326) +- ConnectionStateChange [\#327](https://github.com/ably/ably-python/pull/327) +- Improve realtime logging [\#330](https://github.com/ably/ably-python/pull/330) +- Update readme with realtime documentation [\#334](334](https://github.com/ably/ably-python/pull/334) +- Use string-based enums [\#351](https://github.com/ably/ably-python/pull/351) +- Add environment client option for realtime [\#335](https://github.com/ably/ably-python/pull/335) +- EventEmitter: allow signatures with no event arg [\#350](https://github.com/ably/ably-python/pull/350) + ## [v2.0.0-beta.6](https://github.com/ably/ably-python/tree/v2.0.0-beta.6) [Full Changelog](https://github.com/ably/ably-python/compare/v2.0.0-beta.5...v2.0.0-beta.6) From e78acae1bcf8add6d94bc00d91a4b1e4320236a0 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 29 Jun 2023 17:18:48 +0100 Subject: [PATCH 480/481] chore: bump version for 2.0 release --- ably/__init__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ably/__init__.py b/ably/__init__.py index c708a2f8..fde9e044 100644 --- a/ably/__init__.py +++ b/ably/__init__.py @@ -15,4 +15,4 @@ logger.addHandler(logging.NullHandler()) api_version = '3' -lib_version = '2.0.0-beta.6' +lib_version = '2.0.0' diff --git a/pyproject.toml b/pyproject.toml index ac5e6c7b..75e2414d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "ably" -version = "2.0.0-beta.6" +version = "2.0.0" description = "Python REST and Realtime client library SDK for Ably realtime messaging service" license = "Apache-2.0" authors = ["Ably <support@ably.com>"] From aa2ec090a60f8a631893b98b92f04383bb0acb91 Mon Sep 17 00:00:00 2001 From: Owen Pearson <owen.pearson@ably.com> Date: Thu, 29 Jun 2023 18:00:27 +0100 Subject: [PATCH 481/481] docs: update `capabilities.yaml` --- .ably/capabilities.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.ably/capabilities.yaml b/.ably/capabilities.yaml index 9f124310..807e2f55 100644 --- a/.ably/capabilities.yaml +++ b/.ably/capabilities.yaml @@ -18,8 +18,11 @@ compliance: JSON: MessagePack: Realtime: + Authentication: + Get Confirmed Client Identifier: Channel: Attach: + Retry Timeout: State Events: Subscribe: Connection: @@ -65,6 +68,7 @@ compliance: Remove: Save: Publish: + Request Identifiers: Request Timeout: Service: Get Time: