From ac97086602e55a46da8da559d64d7cb89d45b011 Mon Sep 17 00:00:00 2001 From: catileptic Date: Thu, 10 Aug 2023 17:21:14 +0300 Subject: [PATCH 01/13] Make servicelayer cross-compatible with SQLAlchemy 1.4 and 2+ (#100) * Make servicelayer cross-compatible with SQLAlchemy 1.4 and 2+ * Set minimum version for SQLAlchemy to be the same as for dataset * Increase minimum version of SQLAlchemy to 1.4.49 --- servicelayer/tags.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/servicelayer/tags.py b/servicelayer/tags.py index cf8f92d..f863894 100644 --- a/servicelayer/tags.py +++ b/servicelayer/tags.py @@ -18,7 +18,7 @@ class Tags(object): def __init__(self, name, uri=settings.TAGS_DATABASE_URI, **config): self.name = name - self.engine = create_engine(uri, **config) + self.engine = create_engine(uri, future=True, **config) self.is_postgres = self.engine.dialect.name == "postgresql" self.table = Table( name, diff --git a/setup.py b/setup.py index c79c07c..a1d342d 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ "banal >= 1.0.6, < 2.0.0", "normality >= 2.4.0, < 3.0.0", "fakeredis >=2.11.2, < 3.0.0", - "sqlalchemy >= 2.0.4, < 3.0.0", + "sqlalchemy >= 1.4.49, < 3.0.0", "structlog >= 20.2.0, < 24.0.0", "colorama >= 0.4.4, < 1.0.0", "pika >= 1.3.1, < 2.0.0", From 7fd565ae2b31292a1d7b410a1f31f266d24d7aba Mon Sep 17 00:00:00 2001 From: Alex Stefanescu Date: Thu, 10 Aug 2023 17:33:07 +0300 Subject: [PATCH 02/13] =?UTF-8?q?Bump=20version:=201.21.0=20=E2=86=92=201.?= =?UTF-8?q?21.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- servicelayer/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 84d492c..ecbeb32 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.21.0 +current_version = 1.21.1 commit = True tag = True diff --git a/servicelayer/__init__.py b/servicelayer/__init__.py index 5d48fef..b8b6e1d 100644 --- a/servicelayer/__init__.py +++ b/servicelayer/__init__.py @@ -1,6 +1,6 @@ import logging -__version__ = "1.21.0" +__version__ = "1.21.1" logging.getLogger("boto3").setLevel(logging.WARNING) diff --git a/setup.py b/setup.py index a1d342d..25c2c02 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name="servicelayer", - version="1.21.0", + version="1.21.1", description="Basic remote service functions for alephdata components", classifiers=[ "Development Status :: 3 - Alpha", From ae76c103a70af6d3d06695f9c8636b908841d40a Mon Sep 17 00:00:00 2001 From: catileptic Date: Thu, 24 Aug 2023 17:06:28 +0300 Subject: [PATCH 03/13] Fix SQLAlchemy calls (#105) * Fix SQLAlchemy calls so that all cursor operations happen in the context of an open DB connection * Cap redis at previous version --- servicelayer/tags.py | 4 ++-- setup.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/servicelayer/tags.py b/servicelayer/tags.py index f863894..3e5b3a3 100644 --- a/servicelayer/tags.py +++ b/servicelayer/tags.py @@ -50,7 +50,7 @@ def get(self, key, since=None): stmt = stmt.where(self.table.c.timestamp >= since) with self.engine.connect() as conn: rp = conn.execute(stmt) - row = rp.fetchone() + row = rp.fetchone() if row is not None: return row.value @@ -61,7 +61,7 @@ def exists(self, key, since=None): stmt = stmt.where(self.table.c.timestamp >= since) with self.engine.connect() as conn: rp = conn.execute(stmt) - count = rp.scalar() + count = rp.scalar() return count > 0 def _store_values(self, conn, row): diff --git a/setup.py b/setup.py index 25c2c02..af2adef 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,8 @@ install_requires=[ "banal >= 1.0.6, < 2.0.0", "normality >= 2.4.0, < 3.0.0", - "fakeredis >=2.11.2, < 3.0.0", + "fakeredis >= 2.11.2, < 3.0.0", + "redis <= 4.6.0", "sqlalchemy >= 1.4.49, < 3.0.0", "structlog >= 20.2.0, < 24.0.0", "colorama >= 0.4.4, < 1.0.0", From b657e3b4039e13158d80d653bdeb394a47be8ae1 Mon Sep 17 00:00:00 2001 From: Alex Stefanescu Date: Thu, 24 Aug 2023 17:25:35 +0300 Subject: [PATCH 04/13] =?UTF-8?q?Bump=20version:=201.21.1=20=E2=86=92=201.?= =?UTF-8?q?21.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- servicelayer/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ecbeb32..bc0f507 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.21.1 +current_version = 1.21.2 commit = True tag = True diff --git a/servicelayer/__init__.py b/servicelayer/__init__.py index b8b6e1d..feb72ab 100644 --- a/servicelayer/__init__.py +++ b/servicelayer/__init__.py @@ -1,6 +1,6 @@ import logging -__version__ = "1.21.1" +__version__ = "1.21.2" logging.getLogger("boto3").setLevel(logging.WARNING) diff --git a/setup.py b/setup.py index af2adef..f93ea4d 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name="servicelayer", - version="1.21.1", + version="1.21.2", description="Basic remote service functions for alephdata components", classifiers=[ "Development Status :: 3 - Alpha", From e1c102a71fd81278c15c6957b1f1de048f638d43 Mon Sep 17 00:00:00 2001 From: Christian Stefanescu <217554+stchris@users.noreply.github.com> Date: Mon, 9 Oct 2023 12:00:47 +0200 Subject: [PATCH 05/13] Log worker retry count and retry count exhaustion (#113) * More specific log messages for retry and giving up * Convert to pytest style. Add a test for failing retries * formatting * linter * Add retry count to structlog output --- servicelayer/logs.py | 2 + servicelayer/worker.py | 11 ++++- tests/test_worker.py | 96 ++++++++++++++++++++++++++---------------- 3 files changed, 71 insertions(+), 38 deletions(-) diff --git a/servicelayer/logs.py b/servicelayer/logs.py index 33e0fb3..08ca9a0 100644 --- a/servicelayer/logs.py +++ b/servicelayer/logs.py @@ -7,6 +7,7 @@ from structlog.contextvars import clear_contextvars, bind_contextvars from servicelayer import settings +from servicelayer.util import unpack_int LOG_FORMAT_TEXT = "TEXT" LOG_FORMAT_JSON = "JSON" @@ -84,6 +85,7 @@ def apply_task_context(task, **kwargs): dataset=task.job.dataset.name, start_time=time.time(), trace_id=str(uuid.uuid4()), + retry=unpack_int(task.context.get("retries")), **kwargs ) diff --git a/servicelayer/worker.py b/servicelayer/worker.py index fca5580..e5983a6 100644 --- a/servicelayer/worker.py +++ b/servicelayer/worker.py @@ -75,9 +75,16 @@ def init_internal(self): def retry(self, task): retries = unpack_int(task.context.get("retries")) if retries < settings.WORKER_RETRY: - log.warning("Queue failed task for re-try...") - task.context["retries"] = retries + 1 + retry_count = retries + 1 + log.warning( + f"Queueing failed task for retry #{retry_count}/{settings.WORKER_RETRY}..." # noqa + ) + task.context["retries"] = retry_count task.stage.queue(task.payload, task.context) + else: + log.warning( + f"Failed task, exhausted retry count of {settings.WORKER_RETRY}" + ) def process(self, blocking=True, interval=INTERVAL): retries = 0 diff --git a/tests/test_worker.py b/tests/test_worker.py index 496f526..c4a9a90 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -1,4 +1,4 @@ -from unittest import TestCase +import logging import pytest from servicelayer.cache import get_fakeredis @@ -17,39 +17,63 @@ def handle(self, task): self.test_done += 1 -class WorkerTest(TestCase): - def test_run(self): - conn = get_fakeredis() - operation = "lala" - worker = CountingWorker(conn=conn, stages=[operation]) - worker.sync() - assert worker.test_done == 0, worker.test_done - job = Job.create(conn, "test") - stage = job.get_stage(operation) - task = stage.queue({}, {}) - assert not job.is_done() - assert worker.test_done == 0, worker.test_done - worker.sync() - assert worker.test_done == 1, worker.test_done - assert job.is_done() - worker.retry(task) - assert not job.is_done() - worker.sync() - assert job.is_done() - assert worker.exit_code == 0, worker.exit_code - assert worker.test_done == 1, worker.test_done - worker.retry(task) - worker.run(blocking=False) - assert job.is_done() - assert worker.exit_code == 0, worker.exit_code - worker.num_threads = None +class NoOpWorker(worker.Worker): + def handle(self, task): + pass + + +def test_run(): + conn = get_fakeredis() + operation = "lala" + worker = CountingWorker(conn=conn, stages=[operation]) + worker.sync() + assert worker.test_done == 0, worker.test_done + job = Job.create(conn, "test") + stage = job.get_stage(operation) + task = stage.queue({}, {}) + assert not job.is_done() + assert worker.test_done == 0, worker.test_done + worker.sync() + assert worker.test_done == 1, worker.test_done + assert job.is_done() + worker.retry(task) + assert not job.is_done() + worker.sync() + assert job.is_done() + assert worker.exit_code == 0, worker.exit_code + assert worker.test_done == 1, worker.test_done + worker.retry(task) + worker.run(blocking=False) + assert job.is_done() + assert worker.exit_code == 0, worker.exit_code + worker.num_threads = None + worker.retry(task) + worker.run(blocking=False) + assert job.is_done() + assert worker.exit_code == 0, worker.exit_code + try: + worker._handle_signal(5, None) + except SystemExit as exc: + assert exc.code == 5, exc.code + with pytest.raises(SystemExit) as exc: # noqa + worker._handle_signal(5, None) + + +def test_fails(caplog): + caplog.set_level(logging.DEBUG) + conn = get_fakeredis() + operation = "fails" + job = Job.create(conn, "test") + stage = job.get_stage(operation) + task = stage.queue({}, {}) + + worker = NoOpWorker(conn=conn, stages=[operation]) + worker.sync() + for _ in range(4): worker.retry(task) - worker.run(blocking=False) - assert job.is_done() - assert worker.exit_code == 0, worker.exit_code - try: - worker._handle_signal(5, None) - except SystemExit as exc: - assert exc.code == 5, exc.code - with pytest.raises(SystemExit) as exc: # noqa - worker._handle_signal(5, None) + + log_messages = [r.msg for r in caplog.records] + assert "Queueing failed task for retry #1/3..." in log_messages + assert "Queueing failed task for retry #2/3..." in log_messages + assert "Queueing failed task for retry #3/3..." in log_messages + assert "Failed task, exhausted retry count of 3" in log_messages From 61805907e0250782f82aadf5567c2a41208498d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Oct 2023 11:11:19 +0200 Subject: [PATCH 06/13] Bump pytest-env from 0.8.1 to 1.0.1 (#110) Bumps [pytest-env](https://github.com/pytest-dev/pytest-env) from 0.8.1 to 1.0.1. - [Release notes](https://github.com/pytest-dev/pytest-env/releases) - [Commits](https://github.com/pytest-dev/pytest-env/compare/0.8.1...1.0.1) --- updated-dependencies: - dependency-name: pytest-env dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 719f83b..0406699 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ black==23.3.0 ruff==0.0.270 pytest==7.3.1 -pytest-env==0.8.1 +pytest-env==1.0.1 pytest-cov==4.1.0 pytest-mock==3.10.0 wheel==0.40.0 From 32a3ebf9c909697cc9e28bc30ec23ffbd19c4d5d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Oct 2023 11:11:35 +0200 Subject: [PATCH 07/13] Bump wheel from 0.40.0 to 0.41.2 (#108) Bumps [wheel](https://github.com/pypa/wheel) from 0.40.0 to 0.41.2. - [Changelog](https://github.com/pypa/wheel/blob/main/docs/news.rst) - [Commits](https://github.com/pypa/wheel/compare/0.40.0...0.41.2) --- updated-dependencies: - dependency-name: wheel dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0406699..13203c9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,4 +4,4 @@ pytest==7.3.1 pytest-env==1.0.1 pytest-cov==4.1.0 pytest-mock==3.10.0 -wheel==0.40.0 +wheel==0.41.2 From f9d1305fb9c8fdabf6edc2384940efc7a44fc124 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Oct 2023 11:58:29 +0200 Subject: [PATCH 08/13] Bump ruff from 0.0.270 to 0.0.292 (#119) Bumps [ruff](https://github.com/astral-sh/ruff) from 0.0.270 to 0.0.292. - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/BREAKING_CHANGES.md) - [Commits](https://github.com/astral-sh/ruff/compare/v0.0.270...v0.0.292) --- updated-dependencies: - dependency-name: ruff dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 13203c9..9cd85d0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ black==23.3.0 -ruff==0.0.270 +ruff==0.0.292 pytest==7.3.1 pytest-env==1.0.1 pytest-cov==4.1.0 From 6fca6a7c3e67ba064804ac56320bfa6fab0f8bd3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Oct 2023 12:21:28 +0200 Subject: [PATCH 09/13] Bump fakeredis from 2.13.0 to 2.19.0 (#118) Bumps [fakeredis](https://github.com/cunla/fakeredis-py) from 2.13.0 to 2.19.0. - [Release notes](https://github.com/cunla/fakeredis-py/releases) - [Commits](https://github.com/cunla/fakeredis-py/compare/v2.13.0...v2.19.0) --- updated-dependencies: - dependency-name: fakeredis dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 37b60b1..e849fa7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ banal==1.0.6 normality==2.4.0 -fakeredis==2.13.0 +fakeredis==2.19.0 sqlalchemy==2.0.4 structlog==23.1.0 colorama==0.4.6 From 43b821f780908ce738b9b87b9944de94ef241799 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Oct 2023 13:01:48 +0200 Subject: [PATCH 10/13] Bump black from 23.3.0 to 23.9.1 (#117) Bumps [black](https://github.com/psf/black) from 23.3.0 to 23.9.1. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/23.3.0...23.9.1) --- updated-dependencies: - dependency-name: black dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 9cd85d0..daca6e3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,4 @@ -black==23.3.0 +black==23.9.1 ruff==0.0.292 pytest==7.3.1 pytest-env==1.0.1 From 5453b3a42ae6730798b3e39d3fcb056766aede3d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Oct 2023 13:19:51 +0200 Subject: [PATCH 11/13] Bump structlog from 23.1.0 to 23.2.0 (#116) Bumps [structlog](https://github.com/hynek/structlog) from 23.1.0 to 23.2.0. - [Release notes](https://github.com/hynek/structlog/releases) - [Changelog](https://github.com/hynek/structlog/blob/main/CHANGELOG.md) - [Commits](https://github.com/hynek/structlog/compare/23.1.0...23.2.0) --- updated-dependencies: - dependency-name: structlog dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e849fa7..4197dfa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,6 @@ banal==1.0.6 normality==2.4.0 fakeredis==2.19.0 sqlalchemy==2.0.4 -structlog==23.1.0 +structlog==23.2.0 colorama==0.4.6 pika==1.3.2 From 291d692311274b6f0bd79a80896ffe9f97425538 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Oct 2023 14:02:41 +0200 Subject: [PATCH 12/13] Bump pytest from 7.3.1 to 7.4.2 (#115) Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.3.1 to 7.4.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.3.1...7.4.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index daca6e3..48b697d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,6 @@ black==23.9.1 ruff==0.0.292 -pytest==7.3.1 +pytest==7.4.2 pytest-env==1.0.1 pytest-cov==4.1.0 pytest-mock==3.10.0 From 6241a1011d6016b004c16048b3f95a4ac4a8a22c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Oct 2023 14:21:48 +0200 Subject: [PATCH 13/13] Bump normality from 2.4.0 to 2.5.0 (#114) Bumps [normality](https://github.com/pudo/normality) from 2.4.0 to 2.5.0. - [Commits](https://github.com/pudo/normality/compare/2.4.0...2.5.0) --- updated-dependencies: - dependency-name: normality dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4197dfa..0e5c73f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ banal==1.0.6 -normality==2.4.0 +normality==2.5.0 fakeredis==2.19.0 sqlalchemy==2.0.4 structlog==23.2.0