diff --git a/cosmos/cache.py b/cosmos/cache.py index 3c2086c7a..cb2345f92 100644 --- a/cosmos/cache.py +++ b/cosmos/cache.py @@ -3,6 +3,7 @@ import shutil from pathlib import Path +import msgpack from airflow.models.dag import DAG from airflow.utils.task_group import TaskGroup @@ -121,4 +122,18 @@ def _copy_partial_parse_to_project(partial_parse_filepath: Path, project_path: P source_manifest_filepath = partial_parse_filepath.parent / DBT_MANIFEST_FILE_NAME target_manifest_filepath = target_partial_parse_file.parent / DBT_MANIFEST_FILE_NAME shutil.copy(str(partial_parse_filepath), str(target_partial_parse_file)) + + # Update root_path in partial parse file to point to the target project directory. This is necessary because in some + # earlier versions of dbt (e.g. 1.5.4), the root_path was hardcoded to a stale directory and is not updated to the + # needed target directory. This seems to have been resolved in later versions of dbt, but we still need to handle + # this for compatibility with older versions. + with target_partial_parse_file.open("rb") as f: + data = msgpack.unpack(f) + for node in data["nodes"].values(): + if node.get("root_path"): + node["root_path"] = str(project_path) + with target_partial_parse_file.open("wb") as f: + packed = msgpack.packb(data) + f.write(packed) + shutil.copy(str(source_manifest_filepath), str(target_manifest_filepath)) diff --git a/pyproject.toml b/pyproject.toml index 5f0e5ee0e..4d5a220c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -141,16 +141,17 @@ matrix.airflow.dependencies = [ [tool.hatch.envs.tests.scripts] freeze = "pip freeze" -type-check = "mypy cosmos" test = 'sh scripts/test/unit.sh' test-cov = 'sh scripts/test/unit-cov.sh' -test-integration-setup = 'sh scripts/test/integration-setup.sh' test-integration = 'sh scripts/test/integration.sh' +test-integration-dbt-1-5-4 = 'sh scripts/test/integration-dbt-1-5-4.sh' test-integration-expensive = 'sh scripts/test/integration-expensive.sh' -test-integration-sqlite-setup = 'sh scripts/test/integration-sqlite-setup.sh' +test-integration-setup = 'sh scripts/test/integration-setup.sh' test-integration-sqlite = 'sh scripts/test/integration-sqlite.sh' -test-performance-setup = 'sh scripts/test/performance-setup.sh' +test-integration-sqlite-setup = 'sh scripts/test/integration-sqlite-setup.sh' test-performance = 'sh scripts/test/performance.sh' +test-performance-setup = 'sh scripts/test/performance-setup.sh' +type-check = "mypy cosmos" [tool.pytest.ini_options] filterwarnings = ["ignore::DeprecationWarning"] diff --git a/scripts/test/integration-dbt-1-5-4.sh b/scripts/test/integration-dbt-1-5-4.sh new file mode 100644 index 000000000..04597bd18 --- /dev/null +++ b/scripts/test/integration-dbt-1-5-4.sh @@ -0,0 +1,10 @@ +pip uninstall dbt-adapters dbt-common dbt-core dbt-extractor dbt-postgres dbt-semantic-interfaces -y +pip install dbt-postgres==1.5.4 +pytest -vv \ + --cov=cosmos \ + --cov-report=term-missing \ + --cov-report=xml \ + --durations=0 \ + -m integration \ + --ignore=tests/perf \ + -k 'basic_cosmos_task_group'