Skip to content

Commit

Permalink
bugfix: remove the retry count redis key on permanent failure
Browse files Browse the repository at this point in the history
  • Loading branch information
stchris committed Nov 6, 2024
1 parent fe9bff9 commit 2d3bbeb
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 2 deletions.
4 changes: 4 additions & 0 deletions servicelayer/taskqueue.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@ def increment_retry_count(self, conn):
conn.incr(self.retry_key)
conn.expire(self.retry_key, settings.REDIS_EXPIRE)

def clear_retry_count(self, conn):
conn.delete(self.retry_key)

def get_dataset(self, conn):
dataset = Dataset(
conn=conn, name=dataset_from_collection_id(self.collection_id)
Expand Down Expand Up @@ -592,6 +595,7 @@ def handle(self, task: Task, channel) -> Tuple[bool, bool]:
log.exception(
f"Task {task.task_id} permanently failed and will be discarded."
)
task.clear_retry_count(self.conn)
success = False
retry = False
except Exception:
Expand Down
5 changes: 3 additions & 2 deletions tests/test_taskqueue.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,9 @@ def test_task_queue(self):
delivery_tag=1, multiple=False, requeue=False
)
assert "Max retries reached for task test-task. Aborting." in ctx.output[0]
# Assert that retry count stays the same
assert task.get_retry_count(conn) == 1
# Assert that the retry count is 0 and the key is gone
assert task.get_retry_count(conn) == 0
assert conn.get("tq:qdj:2:taskretry:test-task") is None

worker.ack_message(worker.test_task, channel)
status = dataset.get_status()
Expand Down

0 comments on commit 2d3bbeb

Please sign in to comment.