feat(pipeline): Test DAGs

TODO: Would be really nice to run SQLFluff, Black, Isort... in the CI as
well. But SQLFluff requires a database connection and deps to work.

TODO: Now that we test the loading of the DAGs, we should test running ours
with a mock behind it.

.github/workflows/pipeline.yml

@@ -18,12 +18,54 @@ on:
 
 jobs:
   tests:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-23.10
+    env:
+      CI: 1
+      PYTHONPATH: .
+      PGPASSWORD: password
+      PGHOST: localhost
+      PGUSER: postgres
+      PGPORT: 5432
+      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql://postgres:password@localhost:5432/airflow
+
+    services:
+      postgres:
+        image: postgres:14
+        env:
+          POSTGRES_PASSWORD: password
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
 
     steps:
+      - name: Bootstrap PostgreSQL
+        run: |
+          psql <<SQL
+            ALTER SYSTEM SET fsync=off;
+            ALTER SYSTEM SET synchronous_commit=off;
+            ALTER SYSTEM SET full_page_writes=off;
+            CREATE DATABASE data-inclusion;
+            CREATE DATABASE airflow;
+          SQL
+          docker kill --signal=SIGHUP ${{ job.services.postgres.id }}
+
+      - name: Setup airflow database
+        run: |
+          airflow db reset -y
+          airflow db upgrade
+
       - uses: actions/checkout@v3
+
       - uses: actions/setup-python@v4
         with:
           python-version: '3.11'
+
       - run: pip install tox
-      - run: tox -- -vv
+
+      - name: Run tests
+        run: tox -- -vv

pipeline/requirements/dev/requirements.in

@@ -4,5 +4,6 @@
 black
 pre-commit
 pytest
+pytest-subtests
 tox
 requests-mock

pipeline/requirements/dev/requirements.txt

@@ -71,6 +71,7 @@ attrs==23.1.0
     #   cattrs
     #   fiona
     #   jsonschema
+    #   pytest-subtests
     #   referencing
 babel==2.13.1
     # via flask-babel
@@ -472,6 +473,10 @@ pyproj==3.6.1
 pyproject-api==1.6.1
     # via tox
 pytest==7.4.3
+    # via
+    #   -r requirements/dev/requirements.in
+    #   pytest-subtests
+pytest-subtests==0.11.0
     # via -r requirements/dev/requirements.in
 python-daemon==3.0.1
     # via apache-airflow

pipeline/tests/unit/test_dags.py

@@ -0,0 +1,11 @@
+from airflow.models import DagBag
+
+
+def test_dags_generic(subtests):
+    dagbag = DagBag()
+    for dag_id in dagbag.dag_ids:
+        with subtests.test(msg="DAG generic checks", dag_id=dag_id):
+            dag = dagbag.get_dag(dag_id=dag_id)
+            assert dagbag.import_errors == {}
+            assert dag is not None
+            assert len(dag.tasks) >= 1

pipeline/tox.ini

@@ -2,3 +2,10 @@
 deps = -r requirements/dev/requirements.txt
 usedevelop = true
 commands = pytest {posargs}
+passenv = AIRFLOW_VAR_*
+setenv =
+    AIRFLOW__CORE__DAGS_FOLDER=./dags
+    AIRFLOW__CORE__DEFAULT_TIMEZONE=Europe/Paris
+    AIRFLOW__CORE__EXECUTOR=LocalExecutor
+    AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=false
+    AIRFLOW__CORE__LOAD_EXAMPLES=false