diff --git a/docker-compose.yml b/docker-compose.yml index 8340683a..1546df21 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,12 @@ x-airflow-common: AIRFLOW_CONN_S3_SOURCES: ${AIRFLOW_CONN_S3_SOURCES} AIRFLOW_CONN_PG: postgresql://data-inclusion:data-inclusion@target-db:5432/data-inclusion + AIRFLOW_CONN_S3_LOGS: aws://@/data-inclusion-lake?endpoint_url=http%3A%2F%2Fminio%3A9000&aws_access_key_id=minioadmin&aws_secret_access_key=minioadmin + AIRFLOW__LOGGING__REMOTE_LOGGING: True + AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER: s3://data-inclusion-lake/logs + AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: s3_logs + AIRFLOW__LOGGING__DELETE_LOCAL_LOGS: True + # Variables AIRFLOW_VAR_AGEFIPH_SERVICES_API_URL: ${AGEFIPH_SERVICES_API_URL} AIRFLOW_VAR_AGEFIPH_STRUCTURES_FILE_URL: ${AGEFIPH_STRUCTURES_FILE_URL} diff --git a/pipeline/entrypoint.sh b/pipeline/entrypoint.sh index b4b3c5fa..17e01fd8 100755 --- a/pipeline/entrypoint.sh +++ b/pipeline/entrypoint.sh @@ -26,6 +26,12 @@ export AIRFLOW__CORE__DEFAULT_TIMEZONE=Europe/Paris export AIRFLOW__CORE__FERNET_KEY="${SECRET_KEY}" export AIRFLOW__CORE__DAGS_FOLDER=./dags +export AIRFLOW_CONN_S3_LOGS="${AIRFLOW_CONN_S3_LOGS}" +export AIRFLOW__LOGGING__REMOTE_LOGGING=True +export AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER=s3://data-inclusion-lake/logs +export AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID=s3_logs +export AIRFLOW__LOGGING__DELETE_LOCAL_LOGS=True + if [[ "${COMMAND}" = "webserver" ]]; then airflow webserver --port "${PORT}" fi