Added docker cleanup auto-fixture to improve tests stability (#396)

* Added docker cleanup auto-fixture to improve tests stability

* Use docker API instead of subprocess

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

tests/conftest.py

@@ -1,5 +1,6 @@
 import os
 
+import docker
 import pytest
 from celery import Celery
 
@@ -45,3 +46,54 @@ def default_worker_app(default_worker_app: Celery) -> Celery:
     if app.conf.broker_url and app.conf.broker_url.startswith("sqs"):
         app.conf.broker_transport_options["region"] = LOCALSTACK_CREDS["AWS_DEFAULT_REGION"]
     return app
+
+
+@pytest.fixture(scope="module", autouse=True)
+def auto_clean_docker_resources():
+    """Clean up Docker resources after each test module."""
+    # Used for debugging
+    verbose = False
+
+    def log(message):
+        if verbose:
+            print(message)
+
+    def cleanup_docker_resources():
+        """Function to clean up Docker containers, networks, and volumes based
+        on labels."""
+        docker_client = docker.from_env()
+
+        try:
+            # Clean up containers with the label 'creator=pytest-docker-tools'
+            containers = docker_client.containers.list(all=True, filters={"label": "creator=pytest-docker-tools"})
+            for con in containers:
+                con.reload()  # Ensure we have the latest status
+                if con.status != "running":  # Only remove non-running containers
+                    log(f"Removing container {con.name}")
+                    con.remove(force=True)
+                else:
+                    log(f"Skipping running container {con.name}")
+
+            # Clean up networks with names starting with 'pytest-'
+            networks = docker_client.networks.list(names=["pytest-*"])
+            for network in networks:
+                if not network.containers:  # Check if the network is in use
+                    log(f"Removing network {network.name}")
+                    network.remove()
+                else:
+                    log(f"Skipping network {network.name}, still in use")
+
+            # Clean up volumes with names starting with 'pytest-*'
+            volumes = docker_client.volumes.list(filters={"name": "pytest-*"})
+            for volume in volumes:
+                if not volume.attrs.get("UsageData", {}).get("RefCount", 0):  # Check if volume is not in use
+                    log(f"Removing volume {volume.name}")
+                    volume.remove()
+                else:
+                    log(f"Skipping volume {volume.name}, still in use")
+
+        except Exception as e:
+            log(f"Error occurred while cleaning up Docker resources: {e}")
+
+    log("--- Running Docker resource cleanup ---")
+    cleanup_docker_resources()

tests/unit/conftest.py

@@ -55,3 +55,8 @@ def default_redis_broker() -> RedisContainer:
 @pytest.fixture
 def default_worker_container() -> CeleryWorkerContainer:
     return mocked_container(CeleryWorkerContainer)
+
+
+@pytest.fixture(scope="module", autouse=True)
+def auto_clean_docker_resources():
+    """Skip cleanup in the unit tests."""