Remove airflow dependency (#1044)

* removed airflow dependency

* removed unnecessary webserver config

* removed unused env var

* linting

* typo

* add PyYAML 6.0.2 to requirements

* removed cattrs and cloudpcikle from the dependencies

* removed user override

.github/dependabot.yml

@@ -6,9 +6,6 @@ updates:
     interval: daily
   open-pull-requests-limit: 10
   ignore:
-  - dependency-name: apache-airflow[celery,crypto,jdbc,postgres,ssh]
-    versions:
-    - ">= 2.a, < 3"
   - dependency-name: cattrs
     versions:
     - ">= 1.1.a, < 1.2"

Dockerfile

@@ -1,29 +1,25 @@
-FROM apache/airflow:2.7.1-python3.9
+FROM python:3.9-slim
 ARG install_dev=n
 
 USER root
 
 RUN apt-get update \
-  && apt-get install sudo gcc -yqq \
+  && apt-get install gcc -yqq \
   && rm -rf /var/lib/apt/lists/*
 
-RUN usermod -aG sudo airflow
-RUN echo "airflow ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+ENV PIP_NO_CACHE_DIR=1
 
-USER airflow
+WORKDIR /ejp_xml_pipeline
 
 COPY requirements.build.txt ./
 RUN pip install --disable-pip-version-check -r requirements.build.txt --user
 
 COPY requirements.txt ./
 RUN pip install --disable-pip-version-check -r requirements.txt --user
 
-USER airflow
 COPY requirements.dev.txt ./
 RUN if [ "${install_dev}" = "y" ]; then pip install --disable-pip-version-check --user -r requirements.dev.txt; fi
 
-ENV PATH /home/airflow/.local/bin:$PATH
-
 COPY ejp_xml_pipeline ./ejp_xml_pipeline
 COPY setup.py ./setup.py
 RUN pip install -e . --user --no-dependencies
@@ -33,8 +29,3 @@ COPY .flake8 ./.flake8
 COPY tests ./tests
 COPY mypy.ini ./
 COPY run_test.sh ./
-
-RUN mkdir -p $AIRFLOW_HOME/serve
-RUN ln -s $AIRFLOW_HOME/logs $AIRFLOW_HOME/serve/log
-
-ENTRYPOINT []

Makefile

@@ -46,11 +46,6 @@ dev-lint: dev-flake8 dev-pylint dev-mypy
 dev-unittest:
 	$(PYTHON) -m pytest -p no:cacheprovider $(ARGS) tests/unit_test
 
-
-dev-integration-test: dev-install
-	$(VENV)/bin/airflow upgradedb
-	$(PYTHON) -m pytest -p no:cacheprovider $(ARGS) tests/integration_test
-
 dev-watch:
 	$(PYTHON) -m pytest_watch -- -p no:cacheprovider $(ARGS) $(PYTEST_WATCH_MODULES)
 

docker-compose.ci.override.yml

@@ -3,6 +3,6 @@ version: '3.4'
 services:
     test-client:
         volumes:
-            - ./sample_data_config/ci-ejp-xml-data-pipeline.config.yaml:/home/airflow/app-config/ejp-xml/ejp-xml-data-pipeline.config.yaml
-            - ./credentials.json:/tmp/credentials.json
-            - ~/.aws/credentials:/home/airflow/.aws/credentials
+            - ./sample_data_config/ci-ejp-xml-data-pipeline.config.yaml:/ejp_xml_pipeline/app-config/ejp-xml/ejp-xml-data-pipeline.config.yaml
+            - ./credentials.json:/root/.config/gcloud/credentials.json
+            - ~/.aws/credentials:/root/.aws/credentials

docker-compose.dev.override.yml

@@ -3,21 +3,19 @@ version: '3.4'
 services:
     data-hub-pipelines-dev:
         volumes:
-            - ./tests:/opt/airflow/tests
-            - ./ejp_xml_pipeline:/opt/airflow/ejp_xml_pipeline
+            - ./tests:/ejp_xml_pipeline/tests
+            - ./ejp_xml_pipeline:/ejp_xml_pipeline/ejp_xml_pipeline
 
     test-client:
         volumes:
-            - ~/.config/gcloud/application_default_credentials.json:/tmp/credentials.json
-            - ./sample_data_config/ci-ejp-xml-data-pipeline.config.yaml:/home/airflow/app-config/ejp-xml/ejp-xml-data-pipeline.config.yaml
-            - ~/.aws/credentials:/home/airflow/.aws/credentials
+            - ~/.config/gcloud/application_default_credentials.json:/root/.config/gcloud/credentials.json
+            - ./sample_data_config/ci-ejp-xml-data-pipeline.config.yaml:/ejp_xml_pipeline/app-config/ejp-xml/ejp-xml-data-pipeline.config.yaml
+            - ~/.aws/credentials:/root/.aws/credentials
 
     data-hub-pipelines:
         volumes:
-            - ~/.config/gcloud/application_default_credentials.json:/tmp/credentials.json
-            - ./sample_data_config/ejp-xml-data-pipeline.config.yaml:/home/airflow/app-config/ejp-xml/ejp-xml-data-pipeline.config.yaml
+            - ~/.config/gcloud/application_default_credentials.json:/root/.config/gcloud/credentials.json
+            - ./sample_data_config/ejp-xml-data-pipeline.config.yaml:/ejp_xml_pipeline/app-config/ejp-xml/ejp-xml-data-pipeline.config.yaml
             - ~/.aws/credentials:/root/.aws/credentials
         environment:
-            - PYTHONPATH=/home/airflow/.local/lib/python3.9/site-packages
-            - GOOGLE_APPLICATION_CREDENTIALS=/tmp/credentials.json
-        user: 0:0
+            - GOOGLE_APPLICATION_CREDENTIALS=/root/.config/gcloud/credentials.json

docker-compose.yml

@@ -3,9 +3,9 @@ version: '3.4'
 x-pipeline-env:
   &pipeline-env
   - LOAD_EX=n
-  - GOOGLE_APPLICATION_CREDENTIALS=/home/airflow/.config/gcloud/credentials.json
+  - GOOGLE_APPLICATION_CREDENTIALS=/root/.config/gcloud/credentials.json
   - DEPLOYMENT_ENV=ci
-  - EJP_XML_CONFIG_FILE_PATH=/home/airflow/app-config/ejp-xml/ejp-xml-data-pipeline.config.yaml
+  - EJP_XML_CONFIG_FILE_PATH=/ejp_xml_pipeline/app-config/ejp-xml/ejp-xml-data-pipeline.config.yaml
 
 
 services:
@@ -30,6 +30,5 @@ services:
         image:  ${IMAGE_REPO}-dev:${IMAGE_TAG}
         environment: *pipeline-env
         command: >
-            bash -c "sudo install -D /tmp/credentials.json -m 644 -t  /home/airflow/.config/gcloud
-            && ./run_test.sh with-end-to-end"
+            bash -c "./run_test.sh with-end-to-end"
 

ejp_xml_pipeline/cli.py

@@ -38,8 +38,6 @@
 DEPLOYMENT_ENV_ENV_NAME = "DEPLOYMENT_ENV"
 DEFAULT_DEPLOYMENT_ENV_VALUE = "ci"
 
-NOTIFICATION_EMAILS_ENV_NAME = "AIRFLOW_NOTIFICATION_EMAIL_XML_LIST"
-
 
 def get_default_initial_s3_last_modified_date():
     return os.getenv(

requirements.txt

@@ -1,13 +1,10 @@
-apache-airflow[crypto,celery,postgres,jdbc,ssh]==2.7.1
-apache-airflow-providers-amazon==8.3.1
 lxml==5.2.2
 python-dateutil==2.9.0.post0
 pytz==2024.1
 boto3==1.24.59
 botocore==1.27.59
 bigquery-schema-generator==1.6.1
-cattrs>=22.1.0
-cloudpickle==3.0.0
 google-cloud-bigquery==3.25.0
 six==1.16.0
 urllib3>=1.25.4, <2.3
+PyYAML==6.0.2

run_test.sh

@@ -7,19 +7,20 @@ export PYTHONOPTIMIZE=
 
 echo "running pylint"
 PYLINTHOME=/tmp/datahub-dags-pylint \
- pylint tests/ ejp_xml_pipeline/
+ python -m pylint tests/ ejp_xml_pipeline/
 
 echo "running flake8"
-flake8 tests/ ejp_xml_pipeline/
+python -m flake8 tests/ ejp_xml_pipeline/
 
 echo "running mypy"
-mypy tests/ ejp_xml_pipeline/
+python -m mypy tests/ ejp_xml_pipeline/
 
-pytest tests/unit_test/ -p no:cacheprovider -s --disable-warnings
+echo "running unit tests"s
+python -m pytest tests/unit_test/ -p no:cacheprovider -s --disable-warnings
 
 if [[ $1  &&  $1 == "with-end-to-end" ]]; then
     echo "running end to end tests"
-    pytest tests/end2end_test/ -p no:cacheprovider -s
+    python -m pytest tests/end2end_test/ -p no:cacheprovider -s
 fi
 
 echo "done"