add health check for containers

ohcnetwork · Dec 28, 2023 · 2be3bcd · 2be3bcd
1 parent 0cbf284
commit 2be3bcd
Show file tree

Hide file tree

Showing 16 changed files with 94 additions and 13 deletions.
diff --git a/.github/workflows/test-base.yml b/.github/workflows/test-base.yml
@@ -16,7 +16,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ hashFiles('Pipfile.lock', 'docker/prod.Dockerfile') }}
+          key: ${{ runner.os }}-buildx-${{ hashFiles('Pipfile.lock', 'docker/dev.Dockerfile') }}
           restore-keys: |
             ${{ runner.os }}-buildx-
 
@@ -30,7 +30,7 @@ jobs:
           files: docker-compose.yaml,docker-compose.local.yaml
 
       - name: Start services
-        run: docker compose -f docker-compose.yaml -f docker-compose.local.yaml up -d --no-build
+        run: docker compose -f docker-compose.yaml -f docker-compose.local.yaml up -d --wait --no-build
 
       - name: Check migrations
         run: make checkmigration

diff --git a/Makefile b/Makefile
@@ -20,7 +20,7 @@ build:
 	docker compose -f docker-compose.yaml -f $(docker_config_file) build
 
 up:
-	docker compose -f docker-compose.yaml -f $(docker_config_file) up -d
+	docker compose -f docker-compose.yaml -f $(docker_config_file) up -d --wait
 
 down:
 	docker compose -f docker-compose.yaml -f $(docker_config_file) down

diff --git a/docker-compose.local.yaml b/docker-compose.local.yaml
@@ -17,6 +17,7 @@ services:
     depends_on:
       - db
       - redis
+      - celery
 
   celery:
     image: care_local
@@ -25,7 +26,6 @@ services:
     entrypoint: [ "bash", "scripts/celery-dev.sh" ]
     depends_on:
       - db
-      - backend
       - redis
     volumes:
       - .:/app
diff --git a/docker-compose.pre-built.yaml b/docker-compose.pre-built.yaml
@@ -1,7 +1,6 @@
 version: '3.4'
 
 services:
-
   backend:
     image: "ghcr.io/coronasafe/care:latest"
     env_file:
@@ -10,6 +9,7 @@ services:
     depends_on:
       - db
       - redis
+      - celery-beat
     ports:
       - "9000:9000"
 
@@ -20,7 +20,6 @@ services:
     entrypoint: [ "bash", "celery_worker-ecs.sh" ]
     depends_on:
       - db
-      - backend
       - redis
 
   celery-beat:
@@ -30,5 +29,5 @@ services:
     entrypoint: [ "bash", "celery_beat-ecs.sh" ]
     depends_on:
       - db
-      - backend
       - redis
+      - celery-beat
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -26,8 +26,8 @@ services:
       - EDGE_PORT=4566
       - SERVICES=s3
     volumes:
-      - "${TEMPDIR:-/tmp/localstack}:/tmp/localstack"
-      - "./docker/awslocal:/docker-entrypoint-initaws.d"
+      - "${TEMPDIR:-./care/media/localstack}:/var/lib/localstack"
+      - "./docker/awslocal:/etc/localstack/init/ready.d/"
 
   fidelius:
     image: khavinshankar/fidelius:v1.0

diff --git a/docker/dev.Dockerfile b/docker/dev.Dockerfile
@@ -21,4 +21,11 @@ RUN pipenv install --system --categories "packages dev-packages"
 
 COPY . /app
 
+HEALTHCHECK \
+  --interval=5s \
+  --timeout=5s \
+  --start-period=5s \
+  --retries=24 \
+  CMD ["/app/scripts/healthcheck.sh"]
+
 WORKDIR /app
diff --git a/docker/prod.Dockerfile b/docker/prod.Dockerfile
@@ -41,7 +41,7 @@ ENV PATH /venv/bin:$PATH
 WORKDIR ${APP_HOME}
 
 RUN apt-get update && apt-get install --no-install-recommends -y \
-  libpq-dev gettext wget gnupg chromium \
+  libpq-dev gettext wget curl gnupg chromium \
   && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
   && rm -rf /var/lib/apt/lists/*
 
@@ -50,6 +50,13 @@ COPY --from=builder /venv /venv
 
 COPY --chmod=0755 ./scripts/*.sh ./
 
+HEALTHCHECK \
+  --interval=5s \
+  --timeout=5s \
+  --start-period=5s \
+  --retries=24 \
+  CMD ["/app/healthcheck.sh"]
+
 COPY . ${APP_HOME}
 
 EXPOSE 9000
diff --git a/scripts/celery-dev.sh b/scripts/celery-dev.sh
@@ -1,5 +1,39 @@
 #!/bin/bash
 
+printf "celery" >> /tmp/container-role
+
+if [ -z "${DATABASE_URL}" ]; then
+    export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"
+fi
+
+postgres_ready() {
+python << END
+import sys
+
+import psycopg
+
+try:
+    psycopg.connect(conninfo="${DATABASE_URL}")
+except psycopg.OperationalError as e:
+    print(e)
+    sys.exit(-1)
+sys.exit(0)
+
+END
+}
+
+until postgres_ready; do
+  >&2 echo 'Waiting for PostgreSQL to become available...'
+  sleep 1
+done
+>&2 echo 'PostgreSQL is available'
+
+python manage.py migrate --noinput
+python manage.py load_redis_index
+
+
+touch /tmp/healthy
+
 watchmedo \
     auto-restart --directory=./ --pattern=*.py --recursive -- \
     celery --workdir="/app" -A config.celery_app worker -B --loglevel=INFO
diff --git a/scripts/celery_beat-ecs.sh b/scripts/celery_beat-ecs.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+printf "celery-beat" >> /tmp/container-role
 
 if [ -z "${DATABASE_URL}" ]; then
     export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"
@@ -29,4 +30,7 @@ done
 python manage.py migrate --noinput
 python manage.py load_redis_index
 
+
+touch /tmp/healthy
+
 celery --app=config.celery_app beat --loglevel=info
diff --git a/scripts/celery_beat.sh b/scripts/celery_beat.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+printf "celery-beat" >> /tmp/container-role
 
 if [ -z "${DATABASE_URL}" ]; then
     export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"
@@ -29,5 +30,8 @@ done
 python manage.py migrate --noinput
 python manage.py load_redis_index
 
+
+touch /tmp/healthy
+
 export NEW_RELIC_CONFIG_FILE=/etc/newrelic.ini
 newrelic-admin run-program celery --app=config.celery_app beat --loglevel=info
diff --git a/scripts/celery_worker-ecs.sh b/scripts/celery_worker-ecs.sh
@@ -1,6 +1,11 @@
 #!/bin/bash
+printf "celery-worker" >> /tmp/container-role
 
 if [ -z "${DATABASE_URL}" ]; then
     export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"
 fi
+
+
+touch /tmp/healthy
+
 celery --app=config.celery_app worker --max-tasks-per-child=6 --loglevel=info
diff --git a/scripts/celery_worker.sh b/scripts/celery_worker.sh
@@ -1,7 +1,12 @@
 #!/bin/bash
+printf "celery-worker" >> /tmp/container-role
 
 if [ -z "${DATABASE_URL}" ]; then
     export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"
 fi
+
+
+touch /tmp/healthy
+
 export NEW_RELIC_CONFIG_FILE=/etc/newrelic.ini
 newrelic-admin run-program celery --app=config.celery_app worker --max-tasks-per-child=6 --loglevel=info
diff --git a/scripts/healthcheck.sh b/scripts/healthcheck.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+CONTAINER_ROLE=$(cat /tmp/container-role)
+if [[ "$CONTAINER_ROLE" = "api" ]]; then
+    curl -f http://localhost:9000/health/ || exit 1
+else
+    # Health check for Worker/beat
+    if [[ -f /tmp/healthy ]]; then
+        exit 0
+    else
+        exit 1
+    fi
+fi
diff --git a/scripts/start-dev.sh b/scripts/start-dev.sh
@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-cd /app
+printf "api" >> /tmp/container-role
 
-echo "running migrations..."
-python manage.py migrate
+cd /app
 
 echo "running collectstatic..."
 python manage.py collectstatic --noinput

diff --git a/scripts/start-ecs.sh b/scripts/start-ecs.sh
@@ -4,6 +4,8 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
+printf "api" >> /tmp/container-role
+
 if [ -z "${DATABASE_URL}" ]; then
     export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"
 fi

diff --git a/scripts/start.sh b/scripts/start.sh
@@ -4,6 +4,8 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
+printf "api" >> /tmp/container-role
+
 if [ -z "${DATABASE_URL}" ]; then
     export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"
 fi