diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5a602cc5..88fe8479 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,6 +14,12 @@ Change Log Unreleased ---------- +[9.3.0] - 2024-01-17 +-------------------- +Added +~~~~~ +* Adds utility function to reset application state similar to setup/teardown in Django request/response cycle. + [9.2.0] - 2023-11-16 -------------------- Added diff --git a/docs/how-tos/add-new-event-bus-concrete-implementation.rst b/docs/how-tos/add-new-event-bus-concrete-implementation.rst new file mode 100644 index 00000000..a4d21eac --- /dev/null +++ b/docs/how-tos/add-new-event-bus-concrete-implementation.rst @@ -0,0 +1,35 @@ +How to add a new concrete implementation of the event bus +========================================================= + +Context +------- + +Here is a list of the existing concrete implementations of the event bus: + +- `Kafka `_ +- `Redis Streams `_ + +This how-to is to help you add a new concrete implementation, for example using Pulsar or some other technology. + +Producing +--------- + +There should be a producer class that inherits from `EventBusProducer `_ in openedx-events. + +The defined ``send`` method is meant to be called from within a signal receiver in the producing service. + +Consuming +--------- + +At a high level, the consumer should be a process that takes the signals and events from the broker and emits the signal with the event. There should be a consumer class that inherits from `EventBusConsumer `_ in openedx-events. + +The consumer class then needs to implement ``consume_indefinitely`` loop, which will stay running and listen to events as they come in. + +We have included an utility function called `prepare_for_new_work_cycle <../../openedx_events/tooling.py#L323>`_ in openedx-events which needs to be called before processing any signal. Currently, it reconnects the db connection if required as well as clears RequestCache and there may be later, more comprehensive changes. These steps mimic some setup/teardown that is normally performed by Django in its request/response based architecture. + +Checkout `consumer.py `_ in event bus redis implementation. + +Abstraction tickets +------------------- + +The known remaining work for a fully abstracted event bus is captured in the `Abstraction tickets `_ diff --git a/docs/how-tos/index.rst b/docs/how-tos/index.rst index f8908b33..3c210b0c 100644 --- a/docs/how-tos/index.rst +++ b/docs/how-tos/index.rst @@ -9,3 +9,4 @@ How-tos adding-events-to-a-service adding-events-to-event-bus using-events + add-new-event-bus-concrete-implementation diff --git a/openedx_events/__init__.py b/openedx_events/__init__.py index c1facb6f..bb45b9cd 100644 --- a/openedx_events/__init__.py +++ b/openedx_events/__init__.py @@ -5,4 +5,4 @@ more information about the project. """ -__version__ = "9.2.0" +__version__ = "9.3.0" diff --git a/openedx_events/tooling.py b/openedx_events/tooling.py index bee23a9a..4994c833 100644 --- a/openedx_events/tooling.py +++ b/openedx_events/tooling.py @@ -7,7 +7,9 @@ from logging import getLogger from django.conf import settings +from django.db import connection from django.dispatch import Signal +from edx_django_utils.cache import RequestCache from openedx_events.data import EventsMetadata from openedx_events.exceptions import SenderValidationError @@ -292,3 +294,45 @@ def load_all_signals(): Loads all non-test signals.py modules. """ _process_all_signals_modules(import_module) + + +def _reconnect_to_db_if_needed(): # pragma: no cover + """ + Reconnects the db connection if needed. + + This is important because Django only does connection validity/age checks as part of + its request/response cycle, which isn't in effect for the consume-loop. If we don't + force these checks, a broken connection will remain broken indefinitely. For most + consumers, this will cause event processing to fail. + """ + has_connection = bool(connection.connection) + requires_reconnect = has_connection and not connection.is_usable() + if requires_reconnect: + connection.connect() + + +def _clear_request_cache(): # pragma: no cover + """ + Clear the RequestCache so that each event consumption starts fresh. + + Signal handlers may be written with the assumption that they are called in the context + of a web request, so we clear the request cache just in case. + """ + RequestCache.clear_all_namespaces() + + +def prepare_for_new_work_cycle(): # pragma: no cover + """ + Ensure that the application state is appropriate for performing a new unit of work. + + This mimics some setup/teardown that is normally performed by Django in its + request/response based architecture and that is needed for ensuring a clean and + usable state in this worker-based application. + + See https://github.com/openedx/openedx-events/issues/236 for details. + """ + # Ensure that the database connection is active and usable. + _reconnect_to_db_if_needed() + + # Clear the request cache, in case anything in the signal handlers rely on it. + _clear_request_cache() diff --git a/requirements/base.in b/requirements/base.in index 7c12f67c..5afd59f7 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,6 +1,7 @@ # Core requirements for using this application -c constraints.txt +edx_django_utils django attrs fastavro diff --git a/requirements/base.txt b/requirements/base.txt index b949b005..b0828833 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -8,24 +8,47 @@ asgiref==3.7.2 # via django attrs==23.1.0 # via -r requirements/base.in -django==3.2.21 +cffi==1.16.0 + # via pynacl +click==8.1.7 + # via edx-django-utils +django==3.2.22 # via # -c requirements/common_constraints.txt # -r requirements/base.in + # django-crum + # django-waffle + # edx-django-utils +django-crum==0.7.9 + # via edx-django-utils +django-waffle==4.0.0 + # via edx-django-utils +edx-django-utils==5.7.0 + # via -r requirements/base.in edx-opaque-keys[django]==2.5.1 # via -r requirements/base.in -fastavro==1.8.3 +fastavro==1.8.4 # via -r requirements/base.in +newrelic==9.1.0 + # via edx-django-utils pbr==5.11.1 # via stevedore +psutil==5.9.5 + # via edx-django-utils +pycparser==2.21 + # via cffi pymongo==3.13.0 # via edx-opaque-keys +pynacl==1.5.0 + # via edx-django-utils pytz==2023.3.post1 # via django sqlparse==0.4.4 # via django stevedore==5.1.0 - # via edx-opaque-keys + # via + # edx-django-utils + # edx-opaque-keys typing-extensions==4.8.0 # via # asgiref diff --git a/requirements/ci.txt b/requirements/ci.txt index a8caf1c7..d611fd0c 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -12,7 +12,7 @@ filelock==3.12.4 # virtualenv packaging==23.2 # via tox -platformdirs==3.10.0 +platformdirs==3.11.0 # via virtualenv pluggy==1.3.0 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 9b1295a5..aa611e20 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -27,6 +27,7 @@ cffi==1.16.0 # via # -r requirements/quality.txt # cryptography + # pynacl chardet==5.2.0 # via diff-cover charset-normalizer==3.3.0 @@ -39,6 +40,7 @@ click==8.1.7 # -r requirements/quality.txt # click-log # code-annotations + # edx-django-utils # edx-lint # pip-tools click-log==0.4.0 @@ -49,7 +51,7 @@ code-annotations==1.5.0 # via # -r requirements/quality.txt # edx-lint -coverage[toml]==7.3.1 +coverage[toml]==7.3.2 # via # -r requirements/quality.txt # pytest-cov @@ -69,14 +71,27 @@ distlib==0.3.7 # via # -r requirements/ci.txt # virtualenv -django==3.2.21 +django==3.2.22 # via # -c requirements/common_constraints.txt # -r requirements/quality.txt + # django-crum + # django-waffle + # edx-django-utils +django-crum==0.7.9 + # via + # -r requirements/quality.txt + # edx-django-utils +django-waffle==4.0.0 + # via + # -r requirements/quality.txt + # edx-django-utils docutils==0.20.1 # via # -r requirements/quality.txt # readme-renderer +edx-django-utils==5.7.0 + # via -r requirements/quality.txt edx-lint==5.3.4 # via -r requirements/quality.txt edx-opaque-keys[django]==2.5.1 @@ -85,7 +100,7 @@ exceptiongroup==1.1.3 # via # -r requirements/quality.txt # pytest -fastavro==1.8.3 +fastavro==1.8.4 # via -r requirements/quality.txt filelock==3.12.4 # via @@ -157,6 +172,10 @@ more-itertools==10.1.0 # via # -r requirements/quality.txt # jaraco-classes +newrelic==9.1.0 + # via + # -r requirements/quality.txt + # edx-django-utils nh3==0.2.14 # via # -r requirements/quality.txt @@ -179,7 +198,7 @@ pkginfo==1.9.6 # via # -r requirements/quality.txt # twine -platformdirs==3.10.0 +platformdirs==3.11.0 # via # -r requirements/ci.txt # -r requirements/quality.txt @@ -192,11 +211,15 @@ pluggy==1.3.0 # diff-cover # pytest # tox +psutil==5.9.5 + # via + # -r requirements/quality.txt + # edx-django-utils py==1.11.0 # via # -r requirements/ci.txt # tox -pycodestyle==2.11.0 +pycodestyle==2.11.1 # via -r requirements/quality.txt pycparser==2.21 # via @@ -234,6 +257,10 @@ pymongo==3.13.0 # via # -r requirements/quality.txt # edx-opaque-keys +pynacl==1.5.0 + # via + # -r requirements/quality.txt + # edx-django-utils pyproject-hooks==1.0.0 # via # -r requirements/pip-tools.txt @@ -302,6 +329,7 @@ stevedore==5.1.0 # via # -r requirements/quality.txt # code-annotations + # edx-django-utils # edx-opaque-keys text-unidecode==1.3 # via @@ -337,7 +365,7 @@ typing-extensions==4.8.0 # edx-opaque-keys # pylint # rich -urllib3==2.0.5 +urllib3==2.0.6 # via # -r requirements/quality.txt # requests diff --git a/requirements/doc.txt b/requirements/doc.txt index 3b246582..e62ccf2e 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -12,7 +12,7 @@ asgiref==3.7.2 # django attrs==23.1.0 # via -r requirements/test.txt -babel==2.12.1 +babel==2.13.0 # via sphinx beautifulsoup4==4.12.2 # via pydata-sphinx-theme @@ -21,18 +21,22 @@ build==1.0.3 certifi==2023.7.22 # via requests cffi==1.16.0 - # via cryptography + # via + # -r requirements/test.txt + # cryptography + # pynacl charset-normalizer==3.3.0 # via requests click==8.1.7 # via # -r requirements/test.txt # code-annotations + # edx-django-utils code-annotations==1.5.0 # via -r requirements/test.txt colorama==0.4.6 # via sphinx-autobuild -coverage[toml]==7.3.1 +coverage[toml]==7.3.2 # via # -r requirements/test.txt # pytest-cov @@ -40,10 +44,21 @@ cryptography==41.0.4 # via secretstorage ddt==1.6.0 # via -r requirements/test.txt -django==3.2.21 +django==3.2.22 # via # -c requirements/common_constraints.txt # -r requirements/test.txt + # django-crum + # django-waffle + # edx-django-utils +django-crum==0.7.9 + # via + # -r requirements/test.txt + # edx-django-utils +django-waffle==4.0.0 + # via + # -r requirements/test.txt + # edx-django-utils doc8==1.1.1 # via -r requirements/doc.in docutils==0.19 @@ -53,13 +68,15 @@ docutils==0.19 # readme-renderer # restructuredtext-lint # sphinx +edx-django-utils==5.7.0 + # via -r requirements/test.txt edx-opaque-keys[django]==2.5.1 # via -r requirements/test.txt exceptiongroup==1.1.3 # via # -r requirements/test.txt # pytest -fastavro==1.8.3 +fastavro==1.8.4 # via -r requirements/test.txt idna==3.4 # via requests @@ -102,6 +119,10 @@ mdurl==0.1.2 # via markdown-it-py more-itertools==10.1.0 # via jaraco-classes +newrelic==9.1.0 + # via + # -r requirements/test.txt + # edx-django-utils nh3==0.2.14 # via readme-renderer packaging==23.2 @@ -121,8 +142,14 @@ pluggy==1.3.0 # via # -r requirements/test.txt # pytest +psutil==5.9.5 + # via + # -r requirements/test.txt + # edx-django-utils pycparser==2.21 - # via cffi + # via + # -r requirements/test.txt + # cffi pydata-sphinx-theme==0.12.0 # via sphinx-book-theme pygments==2.16.1 @@ -136,6 +163,10 @@ pymongo==3.13.0 # via # -r requirements/test.txt # edx-opaque-keys +pynacl==1.5.0 + # via + # -r requirements/test.txt + # edx-django-utils pyproject-hooks==1.0.0 # via build pytest==7.4.2 @@ -225,6 +256,7 @@ stevedore==5.1.0 # -r requirements/test.txt # code-annotations # doc8 + # edx-django-utils # edx-opaque-keys text-unidecode==1.3 # via @@ -248,7 +280,7 @@ typing-extensions==4.8.0 # asgiref # edx-opaque-keys # rich -urllib3==2.0.5 +urllib3==2.0.6 # via # requests # twine diff --git a/requirements/quality.txt b/requirements/quality.txt index 8d675672..1a3ab1ee 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -17,7 +17,10 @@ attrs==23.1.0 certifi==2023.7.22 # via requests cffi==1.16.0 - # via cryptography + # via + # -r requirements/test.txt + # cryptography + # pynacl charset-normalizer==3.3.0 # via requests click==8.1.7 @@ -25,6 +28,7 @@ click==8.1.7 # -r requirements/test.txt # click-log # code-annotations + # edx-django-utils # edx-lint click-log==0.4.0 # via edx-lint @@ -32,7 +36,7 @@ code-annotations==1.5.0 # via # -r requirements/test.txt # edx-lint -coverage[toml]==7.3.1 +coverage[toml]==7.3.2 # via # -r requirements/test.txt # pytest-cov @@ -42,12 +46,25 @@ ddt==1.6.0 # via -r requirements/test.txt dill==0.3.7 # via pylint -django==3.2.21 +django==3.2.22 # via # -c requirements/common_constraints.txt # -r requirements/test.txt + # django-crum + # django-waffle + # edx-django-utils +django-crum==0.7.9 + # via + # -r requirements/test.txt + # edx-django-utils +django-waffle==4.0.0 + # via + # -r requirements/test.txt + # edx-django-utils docutils==0.20.1 # via readme-renderer +edx-django-utils==5.7.0 + # via -r requirements/test.txt edx-lint==5.3.4 # via -r requirements/quality.in edx-opaque-keys[django]==2.5.1 @@ -56,7 +73,7 @@ exceptiongroup==1.1.3 # via # -r requirements/test.txt # pytest -fastavro==1.8.3 +fastavro==1.8.4 # via -r requirements/test.txt idna==3.4 # via requests @@ -100,6 +117,10 @@ mdurl==0.1.2 # via markdown-it-py more-itertools==10.1.0 # via jaraco-classes +newrelic==9.1.0 + # via + # -r requirements/test.txt + # edx-django-utils nh3==0.2.14 # via readme-renderer packaging==23.2 @@ -112,16 +133,22 @@ pbr==5.11.1 # stevedore pkginfo==1.9.6 # via twine -platformdirs==3.10.0 +platformdirs==3.11.0 # via pylint pluggy==1.3.0 # via # -r requirements/test.txt # pytest -pycodestyle==2.11.0 +psutil==5.9.5 + # via + # -r requirements/test.txt + # edx-django-utils +pycodestyle==2.11.1 # via -r requirements/quality.in pycparser==2.21 - # via cffi + # via + # -r requirements/test.txt + # cffi pydocstyle==6.3.0 # via -r requirements/quality.in pygments==2.16.1 @@ -146,6 +173,10 @@ pymongo==3.13.0 # via # -r requirements/test.txt # edx-opaque-keys +pynacl==1.5.0 + # via + # -r requirements/test.txt + # edx-django-utils pytest==7.4.2 # via # -r requirements/test.txt @@ -193,6 +224,7 @@ stevedore==5.1.0 # via # -r requirements/test.txt # code-annotations + # edx-django-utils # edx-opaque-keys text-unidecode==1.3 # via @@ -216,7 +248,7 @@ typing-extensions==4.8.0 # edx-opaque-keys # pylint # rich -urllib3==2.0.5 +urllib3==2.0.6 # via # requests # twine diff --git a/requirements/test.txt b/requirements/test.txt index 6eecb2c7..3d8f84de 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -10,23 +10,43 @@ asgiref==3.7.2 # django attrs==23.1.0 # via -r requirements/base.txt +cffi==1.16.0 + # via + # -r requirements/base.txt + # pynacl click==8.1.7 - # via code-annotations + # via + # -r requirements/base.txt + # code-annotations + # edx-django-utils code-annotations==1.5.0 # via -r requirements/test.in -coverage[toml]==7.3.1 +coverage[toml]==7.3.2 # via pytest-cov ddt==1.6.0 # via -r requirements/test.in -django==3.2.21 +django==3.2.22 # via # -c requirements/common_constraints.txt # -r requirements/base.txt + # django-crum + # django-waffle + # edx-django-utils +django-crum==0.7.9 + # via + # -r requirements/base.txt + # edx-django-utils +django-waffle==4.0.0 + # via + # -r requirements/base.txt + # edx-django-utils +edx-django-utils==5.7.0 + # via -r requirements/base.txt edx-opaque-keys[django]==2.5.1 # via -r requirements/base.txt exceptiongroup==1.1.3 # via pytest -fastavro==1.8.3 +fastavro==1.8.4 # via -r requirements/base.txt iniconfig==2.0.0 # via pytest @@ -34,6 +54,10 @@ jinja2==3.1.2 # via code-annotations markupsafe==2.1.3 # via jinja2 +newrelic==9.1.0 + # via + # -r requirements/base.txt + # edx-django-utils packaging==23.2 # via pytest pbr==5.11.1 @@ -42,10 +66,22 @@ pbr==5.11.1 # stevedore pluggy==1.3.0 # via pytest +psutil==5.9.5 + # via + # -r requirements/base.txt + # edx-django-utils +pycparser==2.21 + # via + # -r requirements/base.txt + # cffi pymongo==3.13.0 # via # -r requirements/base.txt # edx-opaque-keys +pynacl==1.5.0 + # via + # -r requirements/base.txt + # edx-django-utils pytest==7.4.2 # via # pytest-cov @@ -70,6 +106,7 @@ stevedore==5.1.0 # via # -r requirements/base.txt # code-annotations + # edx-django-utils # edx-opaque-keys text-unidecode==1.3 # via python-slugify