From 9b889a48a137113c0725e5d149f6ade535ce82ff Mon Sep 17 00:00:00 2001
From: Pankaj Koti <pankajkoti699@gmail.com>
Date: Thu, 29 Aug 2024 20:43:03 +0530
Subject: [PATCH] Resolve CI failures for Apache Airflow 2.7 jobs (#1182)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Description

This is an effort to fix the currently broken CI jobs. The CI pipeline
was running smoothly until the Cosmos 1.6.0 release (#1103). However,
after the recent release of Apache Airflow providers, we’ve been
encountering deep resolution errors specifically with Apache Airflow 2.7
jobs.

To address this:

1. For Airflow 2.7 tests, we’re pre-installing the providers using
Airflow 2.7 constraints.
2. We’re also upgrading certain jobs, like type-checks and dbt 1.5.4
tests, from Airflow 2.7 to Airflow 2.8, as it seems beneficial to move
from 2.7 to 2.8.

Despite pre-installing the providers, we’re noticing that on Python 3.8
and 3.9 with Apache Airflow 2.7, Cosmos still attempts to sync
dependencies and continues to encounter deep resolution errors.
Therefore, we’re specifically excluding those combinations from running
in Unit and Integration test jobs. However, we have Python 3.10 and 3.11
jobs enabled and running fine for Airflow 2.7 in our CI.

## Related Issue(s)
related: #1180
---
 .github/workflows/test.yml          | 20 ++++++++++++++++----
 scripts/test/pre-install-airflow.sh | 11 +++++++++++
 2 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index be33c90f6..846872bff 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -32,7 +32,7 @@ jobs:
           architecture: "x64"
 
       - run: pip3 install hatch
-      - run: hatch run tests.py3.9-2.7:type-check
+      - run: hatch run tests.py3.9-2.8:type-check
 
   Run-Unit-Tests:
     runs-on: ubuntu-latest
@@ -59,6 +59,12 @@ jobs:
             airflow-version: "2.7"
           - python-version: "3.12"
             airflow-version: "2.8"
+          # It's observed that the dependencies resolution for Apache Airflow versions 2.7 are error-ring out with deep
+          # resolutions. This is a temporary exclusion until the issue is resolved.
+          - python-version: "3.8"
+            airflow-version: "2.7"
+          - python-version: "3.9"
+            airflow-version: "2.7"
     steps:
       - uses: actions/checkout@v3
         with:
@@ -104,6 +110,12 @@ jobs:
             airflow-version: "2.4"
           - python-version: "3.11"
             airflow-version: "2.5"
+          # It's observed that the dependencies resolution for Apache Airflow versions 2.7 are error-ring out with deep
+          # resolutions. This is a temporary exclusion until the issue is resolved.
+          - python-version: "3.8"
+            airflow-version: "2.7"
+          - python-version: "3.9"
+            airflow-version: "2.7"
     services:
       postgres:
         image: postgres
@@ -254,7 +266,7 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.11"]
-        airflow-version: ["2.7"]
+        airflow-version: ["2.8"]
 
     steps:
       - uses: actions/checkout@v3
@@ -319,7 +331,7 @@ jobs:
     strategy:
       matrix:
         python-version: [ "3.11" ]
-        airflow-version: [ "2.7" ]
+        airflow-version: [ "2.8" ]
     services:
       postgres:
         image: postgres
@@ -395,7 +407,7 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.11"]
-        airflow-version: ["2.7"]
+        airflow-version: ["2.8"]
         num-models: [1, 10, 50, 100]
     services:
       postgres:
diff --git a/scripts/test/pre-install-airflow.sh b/scripts/test/pre-install-airflow.sh
index 08dcf042d..29ec01038 100755
--- a/scripts/test/pre-install-airflow.sh
+++ b/scripts/test/pre-install-airflow.sh
@@ -22,4 +22,15 @@ mv /tmp/constraint.txt.tmp /tmp/constraint.txt
 pip install uv
 uv pip install "apache-airflow==$AIRFLOW_VERSION" --constraint /tmp/constraint.txt
 uv pip install pydantic --constraint /tmp/constraint.txt
+
+
+if [ "$AIRFLOW_VERSION" = "2.7" ]; then
+  uv pip install "apache-airflow-providers-amazon[s3fs]" --constraint /tmp/constraint.txt
+  uv pip install "apache-airflow-providers-cncf-kubernetes" --constraint /tmp/constraint.txt
+  uv pip install "apache-airflow-providers-docker" --constraint /tmp/constraint.txt
+  uv pip install "apache-airflow-providers-google" --constraint /tmp/constraint.txt
+  uv pip install "apache-airflow-providers-microsoft-azure" --constraint /tmp/constraint.txt
+  uv pip install "apache-airflow-providers-postgres" --constraint /tmp/constraint.txt
+fi
+
 rm /tmp/constraint.txt