add health check for containers

ohcnetwork · Dec 28, 2023 · 9741379 · 9741379
1 parent d8cfae1
commit 9741379
Show file tree

Hide file tree

Showing 16 changed files with 83 additions and 18 deletions.
diff --git a/.github/workflows/test-base.yml b/.github/workflows/test-base.yml
@@ -16,7 +16,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ hashFiles('Pipfile.lock', 'docker/prod.Dockerfile') }}
+          key: ${{ runner.os }}-buildx-${{ hashFiles('Pipfile.lock', 'docker/dev.Dockerfile') }}
           restore-keys: |
             ${{ runner.os }}-buildx-
 
@@ -30,7 +30,7 @@ jobs:
           files: docker-compose.yaml,docker-compose.local.yaml
 
       - name: Start services
-        run: docker compose -f docker-compose.yaml -f docker-compose.local.yaml up -d --no-build
+        run: docker compose -f docker-compose.yaml -f docker-compose.local.yaml up -d --wait --no-build
 
       - name: Check migrations
         run: make checkmigration

diff --git a/Makefile b/Makefile
@@ -20,7 +20,7 @@ build:
 	docker compose -f docker-compose.yaml -f $(docker_config_file) build
 
 up:
-	docker compose -f docker-compose.yaml -f $(docker_config_file) up -d
+	docker compose -f docker-compose.yaml -f $(docker_config_file) up -d --wait
 
 down:
 	docker compose -f docker-compose.yaml -f $(docker_config_file) down
@@ -34,16 +34,16 @@ list:
 logs:
 	docker compose -f docker-compose.yaml -f $(docker_config_file) logs
 
-checkmigration: up
+checkmigration:
 	docker compose exec backend bash -c "python manage.py makemigrations --check --dry-run"
 
-makemigrations: up
+makemigrations:
 	docker compose exec backend bash -c "python manage.py makemigrations"
 
-test: up
+test:
 	docker compose exec backend bash -c "python manage.py test --keepdb --parallel"
 
-test-coverage: up
+test-coverage:
 	docker compose exec backend bash -c "coverage run manage.py test --settings=config.settings.test --keepdb --parallel"
 	docker compose exec backend bash -c "coverage combine || true; coverage xml"
 	docker compose cp backend:/app/coverage.xml coverage.xml
diff --git a/docker-compose.local.yaml b/docker-compose.local.yaml
@@ -25,7 +25,6 @@ services:
     entrypoint: [ "bash", "scripts/celery-dev.sh" ]
     depends_on:
       - db
-      - backend
       - redis
     volumes:
       - .:/app
diff --git a/docker-compose.pre-built.yaml b/docker-compose.pre-built.yaml
@@ -1,7 +1,6 @@
 version: '3.4'
 
 services:
-
   backend:
     image: "ghcr.io/coronasafe/care:latest"
     env_file:
@@ -20,7 +19,6 @@ services:
     entrypoint: [ "bash", "celery_worker-ecs.sh" ]
     depends_on:
       - db
-      - backend
       - redis
 
   celery-beat:
@@ -30,5 +28,5 @@ services:
     entrypoint: [ "bash", "celery_beat-ecs.sh" ]
     depends_on:
       - db
-      - backend
       - redis
+      - celery-beat
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -26,8 +26,8 @@ services:
       - EDGE_PORT=4566
       - SERVICES=s3
     volumes:
-      - "${TEMPDIR:-/tmp/localstack}:/tmp/localstack"
-      - "./docker/awslocal:/docker-entrypoint-initaws.d"
+      - "${TEMPDIR:-./care/media/localstack}:/var/lib/localstack"
+      - "./docker/awslocal:/etc/localstack/init/ready.d/"
 
   fidelius:
     image: khavinshankar/fidelius:v1.0

diff --git a/docker/dev.Dockerfile b/docker/dev.Dockerfile
@@ -8,7 +8,7 @@ ENV PATH /venv/bin:$PATH
 
 RUN apt-get update && apt-get install --no-install-recommends -y \
   build-essential libjpeg-dev zlib1g-dev \
-  libpq-dev gettext wget gnupg chromium \
+  libpq-dev gettext wget curl gnupg chromium \
   && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
   && rm -rf /var/lib/apt/lists/*
 
@@ -21,4 +21,11 @@ RUN pipenv install --system --categories "packages dev-packages"
 
 COPY . /app
 
+HEALTHCHECK \
+  --interval=10s \
+  --timeout=5s \
+  --start-period=10s \
+  --retries=12 \
+  CMD ["/app/scripts/healthcheck.sh"]
+
 WORKDIR /app
diff --git a/docker/prod.Dockerfile b/docker/prod.Dockerfile
@@ -41,7 +41,7 @@ ENV PATH /venv/bin:$PATH
 WORKDIR ${APP_HOME}
 
 RUN apt-get update && apt-get install --no-install-recommends -y \
-  libpq-dev gettext wget gnupg chromium \
+  libpq-dev gettext wget curl gnupg chromium \
   && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
   && rm -rf /var/lib/apt/lists/*
 
@@ -50,6 +50,13 @@ COPY --from=builder /venv /venv
 
 COPY --chmod=0755 ./scripts/*.sh ./
 
+HEALTHCHECK \
+  --interval=30s \
+  --timeout=5s \
+  --start-period=10s \
+  --retries=6 \
+  CMD ["/app/healthcheck.sh"]
+
 COPY . ${APP_HOME}
 
 EXPOSE 9000
diff --git a/scripts/celery-dev.sh b/scripts/celery-dev.sh
@@ -1,5 +1,37 @@
 #!/bin/bash
 
+printf "celery" >> /tmp/container-role
+
+if [ -z "${DATABASE_URL}" ]; then
+    export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"
+fi
+
+postgres_ready() {
+python << END
+import sys
+
+import psycopg
+
+try:
+    psycopg.connect(conninfo="${DATABASE_URL}")
+except psycopg.OperationalError as e:
+    print(e)
+    sys.exit(-1)
+sys.exit(0)
+
+END
+}
+
+until postgres_ready; do
+  >&2 echo 'Waiting for PostgreSQL to become available...'
+  sleep 1
+done
+>&2 echo 'PostgreSQL is available'
+
+python manage.py migrate --noinput
+python manage.py load_redis_index
+
+
 watchmedo \
     auto-restart --directory=./ --pattern=*.py --recursive -- \
     celery --workdir="/app" -A config.celery_app worker -B --loglevel=INFO
diff --git a/scripts/celery_beat-ecs.sh b/scripts/celery_beat-ecs.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+printf "celery-beat" >> /tmp/container-role
 
 if [ -z "${DATABASE_URL}" ]; then
     export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"
@@ -29,4 +30,5 @@ done
 python manage.py migrate --noinput
 python manage.py load_redis_index
 
+
 celery --app=config.celery_app beat --loglevel=info
diff --git a/scripts/celery_beat.sh b/scripts/celery_beat.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+printf "celery-beat" >> /tmp/container-role
 
 if [ -z "${DATABASE_URL}" ]; then
     export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"
@@ -29,5 +30,6 @@ done
 python manage.py migrate --noinput
 python manage.py load_redis_index
 
+
 export NEW_RELIC_CONFIG_FILE=/etc/newrelic.ini
 newrelic-admin run-program celery --app=config.celery_app beat --loglevel=info
diff --git a/scripts/celery_worker-ecs.sh b/scripts/celery_worker-ecs.sh
@@ -1,6 +1,9 @@
 #!/bin/bash
+printf "celery-worker" >> /tmp/container-role
 
 if [ -z "${DATABASE_URL}" ]; then
     export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"
 fi
+
+
 celery --app=config.celery_app worker --max-tasks-per-child=6 --loglevel=info
diff --git a/scripts/celery_worker.sh b/scripts/celery_worker.sh
@@ -1,7 +1,10 @@
 #!/bin/bash
+printf "celery-worker" >> /tmp/container-role
 
 if [ -z "${DATABASE_URL}" ]; then
     export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"
 fi
+
+
 export NEW_RELIC_CONFIG_FILE=/etc/newrelic.ini
 newrelic-admin run-program celery --app=config.celery_app worker --max-tasks-per-child=6 --loglevel=info
diff --git a/scripts/healthcheck.sh b/scripts/healthcheck.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+CONTAINER_ROLE=$(cat /tmp/container-role)
+if [[ "$CONTAINER_ROLE" = "api" ]]; then
+    curl -fsS http://localhost:9000/health/ || exit 1
+elif [[ "$CONTAINER_ROLE" == celery* ]]; then
+    celery -A config.celery_app inspect ping -d celery@$HOSTNAME || exit 1
+fi
diff --git a/scripts/start-dev.sh b/scripts/start-dev.sh
@@ -1,16 +1,16 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-cd /app
+printf "api" >> /tmp/container-role
 
-echo "running migrations..."
-python manage.py migrate
+cd /app
 
 echo "running collectstatic..."
 python manage.py collectstatic --noinput
 
 echo "starting server..."
 if [[ "${DJANGO_DEBUG,,}" == "true" ]]; then
+  echo "waiting for debugger..."
   python -m debugpy --wait-for-client --listen 0.0.0.0:9876 manage.py runserver_plus 0.0.0.0:9000
 else
   python manage.py runserver 0.0.0.0:9000

diff --git a/scripts/start-ecs.sh b/scripts/start-ecs.sh
@@ -4,6 +4,8 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
+printf "api" >> /tmp/container-role
+
 if [ -z "${DATABASE_URL}" ]; then
     export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"
 fi

diff --git a/scripts/start.sh b/scripts/start.sh
@@ -4,6 +4,8 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
+printf "api" >> /tmp/container-role
+
 if [ -z "${DATABASE_URL}" ]; then
     export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"
 fi