From 89b479539d4b9574ffe848bad37afdb50a30a565 Mon Sep 17 00:00:00 2001 From: Valentin Matton Date: Mon, 23 Oct 2023 12:26:51 +0200 Subject: [PATCH] fix(scalingo): remove multiline commands --- pipeline/Procfile | 24 +++---------- pipeline/entrypoint.sh | 78 +++++++++++++++++++++++++++++------------- 2 files changed, 60 insertions(+), 42 deletions(-) diff --git a/pipeline/Procfile b/pipeline/Procfile index 7c98b5b9..1b75af74 100644 --- a/pipeline/Procfile +++ b/pipeline/Procfile @@ -1,19 +1,5 @@ -# The `DATABASE_URL` env var is automatically set by Scalingo and uses the depreciated -# scheme `postgres://`. Therefore it is replaced. -web: env \ - AIRFLOW__API__AUTH_BACKENDS=airflow.api.auth.backend.basic_auth \ - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN="${DATABASE_URL/postgres\:\/\//postgresql\:\/\/}" \ - airflow webserver --port "${PORT}" -postdeploy: env \ - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN="${DATABASE_URL/postgres\:\/\//postgresql\:\/\/}" \ - airflow db migrate -scheduler: env \ - AIRFLOW_HOME=./airflow \ - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN="${DATABASE_URL/postgres\:\/\//postgresql\:\/\/}" \ - AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=False \ - AIRFLOW__CORE__LOAD_EXAMPLES=False \ - AIRFLOW__CORE__EXECUTOR=LocalExecutor \ - AIRFLOW__CORE__DEFAULT_TIMEZONE=Europe/Paris \ - AIRFLOW__CORE__FERNET_KEY="${SECRET_KEY}" \ - AIRFLOW__CORE__DAGS_FOLDER=./dags \ - ./entrypoint.sh +web: ./entrypoint.sh webserver + +postdeploy: ./entrypoint.sh migrate + +scheduler: ./entrypoint.sh scheduler diff --git a/pipeline/entrypoint.sh b/pipeline/entrypoint.sh index 1c23933a..40a6adff 100755 --- a/pipeline/entrypoint.sh +++ b/pipeline/entrypoint.sh @@ -6,26 +6,58 @@ set -e # Trace execution [[ "${DEBUG}" ]] && set -x -# Create additional virtualenvs for isolated task executions -VIRTUAL_ENV="${AIRFLOW_HOME}/venvs/python/venv" -python -m venv "${VIRTUAL_ENV}" -"${VIRTUAL_ENV}/bin/python" -m pip install -U pip setuptools wheel -"${VIRTUAL_ENV}/bin/python" -m pip install -r requirements/tasks/python/requirements.txt -"${VIRTUAL_ENV}/bin/python" -m pip install . - -# Create additional virtualenvs for isolated task executions -VIRTUAL_ENV="${AIRFLOW_HOME}/venvs/pipx/venv" -python -m venv "${VIRTUAL_ENV}" -"${VIRTUAL_ENV}/bin/python" -m pip install -U pip setuptools wheel -"${VIRTUAL_ENV}/bin/python" -m pip install -r requirements/tasks/pipx/requirements.txt - -# Create additional virtualenvs for isolated task executions -VIRTUAL_ENV="${AIRFLOW_HOME}/venvs/dbt/venv" -python -m venv "${VIRTUAL_ENV}" -"${VIRTUAL_ENV}/bin/python" -m pip install -U pip setuptools wheel -"${VIRTUAL_ENV}/bin/python" -m pip install -r requirements/tasks/dbt/requirements.txt - -# Install dbt packages (not python packages) -"${VIRTUAL_ENV}/bin/dbt" deps --project-dir "${AIRFLOW_VAR_DBT_PROJECT_DIR}" - -airflow scheduler +if [[ $# -eq 0 ]]; then + echo "No service parameter provided."; + exit 1; +fi + +COMMAND=$1 + +# The `DATABASE_URL` env var is automatically set by Scalingo and uses the depreciated +# scheme `postgres://`. Therefore it is replaced. +export AIRFLOW__DATABASE__SQL_ALCHEMY_CONN="${DATABASE_URL/postgres\:\/\//postgresql\:\/\/}" + +if [[ "${COMMAND}" = "webserver" ]]; then + export AIRFLOW__API__AUTH_BACKENDS=airflow.api.auth.backend.basic_auth + + airflow webserver --port "${PORT}" +fi + +if [[ "${COMMAND}" = "migrate" ]]; then + airflow db migrate +fi + +if [[ "${COMMAND}" = "scheduler" ]]; then + # Create additional virtualenvs for isolated task executions + VIRTUAL_ENV="${AIRFLOW_HOME}/venvs/python/venv" + python -m venv "${VIRTUAL_ENV}" + "${VIRTUAL_ENV}/bin/python" -m pip install -U pip setuptools wheel + "${VIRTUAL_ENV}/bin/python" -m pip install -r requirements/tasks/python/requirements.txt + "${VIRTUAL_ENV}/bin/python" -m pip install . + + # Create additional virtualenvs for isolated task executions + VIRTUAL_ENV="${AIRFLOW_HOME}/venvs/pipx/venv" + python -m venv "${VIRTUAL_ENV}" + "${VIRTUAL_ENV}/bin/python" -m pip install -U pip setuptools wheel + "${VIRTUAL_ENV}/bin/python" -m pip install -r requirements/tasks/pipx/requirements.txt + + # Create additional virtualenvs for isolated task executions + VIRTUAL_ENV="${AIRFLOW_HOME}/venvs/dbt/venv" + python -m venv "${VIRTUAL_ENV}" + "${VIRTUAL_ENV}/bin/python" -m pip install -U pip setuptools wheel + "${VIRTUAL_ENV}/bin/python" -m pip install -r requirements/tasks/dbt/requirements.txt + + # Install dbt packages (not python packages) + "${VIRTUAL_ENV}/bin/dbt" deps --project-dir "${AIRFLOW_VAR_DBT_PROJECT_DIR}" + + export AIRFLOW_HOME=./airflow + export AIRFLOW__DATABASE__SQL_ALCHEMY_CONN="${DATABASE_URL/postgres\:\/\//postgresql\:\/\/}" + export AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=False + export AIRFLOW__CORE__LOAD_EXAMPLES=False + export AIRFLOW__CORE__EXECUTOR=LocalExecutor + export AIRFLOW__CORE__DEFAULT_TIMEZONE=Europe/Paris + export AIRFLOW__CORE__FERNET_KEY="${SECRET_KEY}" + export AIRFLOW__CORE__DAGS_FOLDER=./dags + + airflow scheduler +fi