[WIP] Airflow version 2.0.0

nxtlytics · Dec 30, 2020 · 81f16f1 · 81f16f1
1 parent dcab7e7
commit 81f16f1
Show file tree

Hide file tree

Showing 11 changed files with 1,595 additions and 1,832 deletions.
diff --git a/apache-airflow/Dockerfile b/apache-airflow/Dockerfile
@@ -1,10 +1,7 @@
-FROM nxtlytics/python37:v0.0.3
-
-# Never prompt the user for choices on installation/configuration of packages
-ENV DEBIAN_FRONTEND noninteractive
+FROM nxtlytics/python38:v0.0.3
 
 # Airflow
-ARG AIRFLOW_VERSION=1.10.10
+ENV DEBIAN_FRONTEND=noninteractive
 ENV AIRFLOW_USER_HOME=/usr/local/airflow
 ENV AIRFLOW_HOME=${AIRFLOW_USER_HOME}
 WORKDIR ${AIRFLOW_USER_HOME}
@@ -28,6 +25,7 @@ RUN set -ex \
         libssl-dev \
         libffi-dev \
         libpq-dev \
+        libmysqlclient-dev \
     ' \
     && apt update -yqq \
     && apt dist-upgrade -yqq \
@@ -63,8 +61,6 @@ RUN set -ex \
         /usr/share/doc-base
 
 COPY *.sh /usr/local/bin/
-COPY airflow.cfg ${AIRFLOW_USER_HOME}/airflow.cfg
-COPY config ${AIRFLOW_USER_HOME}/config/
 
 EXPOSE 8080 5555 8793
 

diff --git a/apache-airflow/Pipfile b/apache-airflow/Pipfile
@@ -1,11 +1,11 @@
 [scripts]
-lint = "python -m black --target-version py37 --line-length 120 --check ."
-lint-fix = "python -m black --target-version py37 --line-length 120 ."
+lint = "python -m black --target-version py38 --line-length 120 --check ."
+lint-fix = "python -m black --target-version py38 --line-length 120 ."
 test = "python -m pytest -v --junitxml=tmp/test-output/pytest.xml"
 airflow = "start-airflow.sh"
 
 [packages]
-apache-airflow = {extras = ["amazon", "cassandra", "celery", "crypto", "datadog", "docker", "flask_oauth", "google", "google_auth", "grpc", "hive", "jdbc", "kubernetes", "postgres", "presto", "redis", "sentry", "slack", "spark", "ssh", "statsd", "virtualenv"], version = "==1.10.10"}
+apache-airflow = {extras = ["amazon", "cassandra", "celery", "crypto", "datadog", "docker", "flask_oauth", "google", "google_auth", "grpc", "hive", "jdbc", "kubernetes", "mysql", "postgres", "presto", "redis", "sentry", "slack", "spark", "ssh", "statsd", "virtualenv"], version = "==2.0.0"}
 pytz = "*"
 pyopenssl = "*"
 ndg-httpsclient = "*"
@@ -15,7 +15,7 @@ protobuf = "*"
 
 [dev-packages]
 pytest = "*"
-black = "==19.3b0"
+black = "==20.8b1"
 
 [requires]
-python_version = "3.7"
+python_version = "3.8"
diff --git a/apache-airflow/Pipfile.lock b/apache-airflow/Pipfile.lock
diff --git a/apache-airflow/README.md b/apache-airflow/README.md
@@ -29,8 +29,7 @@ Container image for running Apache Airflow with a `CeleryExecutor`.
 |         `GOOGLE_DOMAINS` | -                                                                                      |                                                                                                    Comma Separated G Suite Domains to allow login |
 |             `SENTRY_DSN` | -                                                                                      |                                                                                                                     Sentry Data Source Name (DSN) |
 |     `SENTRY_ENVIRONMENT` | `develop`                                                                              |                                                                                             Sentry environment for this project to send errors as |
-|   `LOGGING_CONFIG_CLASS` | `airflow.config_templates.`</br>`airflow_local_settings.`</br>`DEFAULT_LOGGING_CONFIG` |                          Logging class Specify the class that will specify the logging configuration This class has to be on the python classpath |
-|      `HOSTNAME_CALLABLE` | `socket:getfqdn`                                                                       |                                     Hostname by providing a path to a callable, which will resolve the hostname. The format is `package:function` |
+|      `HOSTNAME_CALLABLE` | `socket.getfqdn`                                                                       |                                     Hostname by providing a path to a callable, which will resolve the hostname. The format is `package:function` |
 |                   `REPO` | -                                                                                      |                                                                                                            git repository `git-sync.sh` will pull |
 |                 `BRANCH` | -                                                                                      |                                                                                                     git repository branch `git-sync.sh` will pull |
 |                    `DIR` | -                                                                                      |                                                                                    local directory where `git-sync.sh` will clone a repository to |
@@ -68,7 +67,7 @@ AIRFLOW__CORE__DB_NAME=airflow
 REDIS_HOST=redis
 REDIS_DBNUM=1
 LOAD_EX=n
-FERNET_KEY= # the result of $(python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY)")
+FERNET_KEY= # the result of $(openssl rand -base64 32)
 [email protected]:your-org/repository-where-dags-live.git
 BRANCH=some-branch-with-dags
 DIR=/usr/local/airflow/dags