chore(airflow): enable remote logging

gip-inclusion · Oct 23, 2023 · 10623f7 · 10623f7
1 parent 127f8f6
commit 10623f7
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 0 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -21,6 +21,12 @@ x-airflow-common:
     AIRFLOW_CONN_S3_SOURCES: ${AIRFLOW_CONN_S3_SOURCES}
     AIRFLOW_CONN_PG: postgresql://data-inclusion:data-inclusion@target-db:5432/data-inclusion
 
+    AIRFLOW_CONN_S3_LOGS: aws://@/data-inclusion-lake?endpoint_url=http%3A%2F%2Fminio%3A9000&aws_access_key_id=minioadmin&aws_secret_access_key=minioadmin
+    AIRFLOW__LOGGING__REMOTE_LOGGING: True
+    AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER: s3://data-inclusion-lake/logs
+    AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: s3_logs
+    AIRFLOW__LOGGING__DELETE_LOCAL_LOGS: True
+
     # Variables
     AIRFLOW_VAR_AGEFIPH_SERVICES_API_URL: ${AGEFIPH_SERVICES_API_URL}
     AIRFLOW_VAR_AGEFIPH_STRUCTURES_FILE_URL: ${AGEFIPH_STRUCTURES_FILE_URL}

diff --git a/pipeline/entrypoint.sh b/pipeline/entrypoint.sh
@@ -26,6 +26,12 @@ export AIRFLOW__CORE__DEFAULT_TIMEZONE=Europe/Paris
 export AIRFLOW__CORE__FERNET_KEY="${SECRET_KEY}"
 export AIRFLOW__CORE__DAGS_FOLDER=./dags
 
+export AIRFLOW_CONN_S3_LOGS="${AIRFLOW_CONN_S3_LOGS}"
+export AIRFLOW__LOGGING__REMOTE_LOGGING=True
+export AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER=s3://data-inclusion-lake/logs
+export AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID=s3_logs
+export AIRFLOW__LOGGING__DELETE_LOCAL_LOGS=True
+
 if [[ "${COMMAND}" = "webserver" ]]; then
     airflow webserver --port "${PORT}"
 fi