Pin dbt-databricks version to < 1.9 (astronomer#1376)

Our CI is broken due to a `dbt-databricks` release with conflicting
dependencies with `Airflow>=2.8`.

The latest release of`dbt-databricks` has pinned the Pydantic to
`"pydantic>=1.10.0, <2"`
https://github.com/databricks/dbt-databricks/blob/main/pyproject.toml#L33-L33

This change was added in PR:
databricks/dbt-databricks#843 and released on 9
December as part of `dbt-databricks==1.9.0`
(https://pypi.org/project/dbt-databricks/1.9.0/).

The `debt-data bricks` conflicts with the neAirflow version, which that
requires Pydantic 2.0.

For Airflow < 2.8, things work as expected:
https://github.com/astronomer/astronomer-cosmos/actions/runs/12259335118/job/34201293598.

In this PR, I'm pinning dbt-databricks version to < 1.9 to make CI green
again.

Follow-up PR:
astronomer#1379

pyproject.toml

@@ -45,7 +45,8 @@ dbt-all = [
     "dbt-athena",
     "dbt-bigquery",
     "dbt-clickhouse",
-    "dbt-databricks",
+    # TODO: https://github.com/astronomer/astronomer-cosmos/issues/1379
+    "dbt-databricks<1.9",
     "dbt-exasol",
     "dbt-postgres",
     "dbt-redshift",
@@ -144,7 +145,7 @@ dependencies = [
     "types-requests",
     "types-python-dateutil",
     "Werkzeug<3.0.0",
-    "dbt-core"
+    "dbt-core",
 ]
 pre-install-commands = ["sh scripts/test/pre-install-airflow.sh {matrix:airflow} {matrix:python}"]
 
@@ -189,7 +190,6 @@ dependencies = [
     "apache-airflow-providers-microsoft-azure>=8.5.0",
     "msgpack",
     "openlineage-airflow",
-    "pydantic>=1.10.0",
     "pydata-sphinx-theme",
     "sphinx",
     "sphinx-autoapi",

scripts/test/integration-setup.sh

@@ -11,4 +11,4 @@ rm -rf airflow.*
 pip freeze | grep airflow
 airflow db reset -y
 airflow db init
-pip install 'dbt-databricks' 'dbt-bigquery' 'dbt-postgres' 'dbt-vertica' 'openlineage-airflow'
+pip install 'dbt-databricks<1.9' 'dbt-bigquery' 'dbt-postgres' 'dbt-vertica' 'openlineage-airflow'