Skip to content

Commit

Permalink
feat(alert): add wrapper on cron command to send alert on mattermost …
Browse files Browse the repository at this point in the history
…in case of failure
  • Loading branch information
hlecuyer committed Aug 29, 2024
1 parent 78a7747 commit 87f9466
Show file tree
Hide file tree
Showing 3 changed files with 27 additions and 4 deletions.
5 changes: 4 additions & 1 deletion api/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,13 @@ RUN useradd --no-log-init -g gunicorn gunicorn
# Copy venv with compiled dependencies
COPY --chown=gunicorn:gunicorn --from=compile-image /srv/venv /srv/venv

COPY --chown=gunicorn:gunicorn ["docker-entrypoint.sh", "pyproject.toml", "alembic.ini", "/srv/"]
COPY --chown=gunicorn:gunicorn ["run_management_command.sh", "docker-entrypoint.sh", "pyproject.toml", "alembic.ini", "/srv/"]
COPY --chown=gunicorn:gunicorn src /srv/src
COPY --chown=gunicorn:gunicorn tests /srv/tests


RUN chmod +x docker-entrypoint.sh
RUN chmod +x run_management_command.sh

USER gunicorn
EXPOSE 8000
Expand Down
6 changes: 3 additions & 3 deletions api/cron.json
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
{
"jobs": [
{
"command": "30 * * * * TQDM_DISABLE=1 test $ENV = 'prod' && data-inclusion-api load_inclusion_data",
"command": "30 * * * * TQDM_DISABLE=1 test $ENV = 'prod' && sh run_management_command.sh data-inclusion-api load_inclusion_data",
"size": "XL"
},
{
"command": "30 * * * * TQDM_DISABLE=1 test $ENV != 'prod' && data-inclusion-api load_inclusion_data",
"command": "30 * * * * TQDM_DISABLE=1 test $ENV != 'prod' && sh run_management_command.sh data-inclusion-api load_inclusion_data",
"size": "S"
},
{
"command": "0 4 * * * vacuumdb --full --analyze --verbose --table api__structures --table api__services $DATABASE_URL",
"command": "0 4 * * * sh run_management_command.sh vacuumdb --full --analyze --verbose --table api__structures --table api__services $DATABASE_URL",
"size": "S"
}
]
Expand Down
20 changes: 20 additions & 0 deletions api/run_management_command.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#!/bin/sh
set -u

# Execute the command
"$@" &

wait $command_pid

exit_code=$? # Capture the exit code of the previous command

# Check if the exit code indicates an error (not equal to 0)
if [ $exit_code -ne 0 ] && [ "$ENV" = "prod" ]; then
# Execute the curl command in case of an error
escaped_command="in command '$@'"
echo "Error with exit code $exit_code $escaped_command, sending message to Mattermost"
curl -i -X POST -H "Content-Type: application/json" -d "{\"text\": \"Error with exit code $exit_code $escaped_command: \nLogs: scalingo --region osc-secnum-fr1 --app data-inclusion-api-prod logs --lines 1000 -F $CONTAINER -f\"}" $MATTERMOST_HOOK
fi

# Return the exit code of the initial command
exit $exit_code

0 comments on commit 87f9466

Please sign in to comment.