diff --git a/.changes/unreleased/Fixes-20231006-134551.yaml b/.changes/unreleased/Fixes-20231006-134551.yaml new file mode 100644 index 00000000000..9dea4f6e194 --- /dev/null +++ b/.changes/unreleased/Fixes-20231006-134551.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Enable seeds to be handled from stored manifest data +time: 2023-10-06T13:45:51.925546-04:00 +custom: + Author: michelleark + Issue: "6875" diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index 996d5027c58..0863b465b53 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -862,8 +862,18 @@ def try_or_compiler_error( def load_agate_table(self) -> agate.Table: if not isinstance(self.model, SeedNode): raise LoadAgateTableNotSeedError(self.model.resource_type, node=self.model) - assert self.model.root_path - path = os.path.join(self.model.root_path, self.model.original_file_path) + + # include package_path for seeds defined in packages + package_path = ( + os.path.join(self.config.packages_install_path, self.model.package_name) + if self.model.package_name != self.config.project_name + else "." + ) + path = os.path.join(self.config.project_root, package_path, self.model.original_file_path) + if not os.path.exists(path): + assert self.model.root_path + path = os.path.join(self.model.root_path, self.model.original_file_path) + column_types = self.model.config.column_types delimiter = self.model.config.delimiter try: diff --git a/core/dbt/tests/util.py b/core/dbt/tests/util.py index 7c22a4df94f..21a23b58010 100644 --- a/core/dbt/tests/util.py +++ b/core/dbt/tests/util.py @@ -224,6 +224,10 @@ def rm_dir(directory_path): raise FileNotFoundError(f"{directory_path} does not exist.") +def rename_dir(src_directory_path, dest_directory_path): + os.rename(src_directory_path, dest_directory_path) + + # Get an artifact (usually from the target directory) such as # manifest.json or catalog.json to use in a test def get_artifact(*paths): diff --git a/tests/functional/partial_parsing/test_partial_parsing.py b/tests/functional/partial_parsing/test_partial_parsing.py index cd89fd0d014..4efc3577be9 100644 --- a/tests/functional/partial_parsing/test_partial_parsing.py +++ b/tests/functional/partial_parsing/test_partial_parsing.py @@ -1,7 +1,17 @@ +from argparse import Namespace import pytest from unittest import mock -from dbt.tests.util import run_dbt, get_manifest, write_file, rm_file, run_dbt_and_capture +import dbt.flags as flags +from dbt.tests.util import ( + run_dbt, + get_manifest, + write_file, + rm_file, + run_dbt_and_capture, + rename_dir, +) +from tests.functional.utils import up_one from dbt.tests.fixtures.project import write_project_files from tests.functional.partial_parsing.fixtures import ( model_one_sql, @@ -846,3 +856,65 @@ def test_pp_external_models( run_dbt(["--partial-parse", "parse"]) assert len(manifest.nodes) == 6 assert len(manifest.external_node_unique_ids) == 4 + + +class TestPortablePartialParsing: + @pytest.fixture(scope="class") + def models(self): + return { + "model_one.sql": model_one_sql, + } + + @pytest.fixture(scope="class") + def packages(self): + return {"packages": [{"local": "local_dependency"}]} + + @pytest.fixture(scope="class") + def local_dependency_files(self): + return { + "dbt_project.yml": local_dependency__dbt_project_yml, + "models": { + "schema.yml": local_dependency__models__schema_yml, + "model_to_import.sql": local_dependency__models__model_to_import_sql, + }, + "macros": {"dep_macro.sql": local_dependency__macros__dep_macro_sql}, + "seeds": {"seed.csv": local_dependency__seeds__seed_csv}, + } + + def rename_project_root(self, project, new_project_root): + with up_one(new_project_root): + rename_dir(project.project_root, new_project_root) + project.project_root = new_project_root + # flags.project_dir is set during the project test fixture, and is persisted across run_dbt calls, + # so it needs to be reset between invocations + flags.set_from_args(Namespace(PROJECT_DIR=new_project_root), None) + + @pytest.fixture(scope="class", autouse=True) + def initial_run_and_rename_project_dir(self, project, local_dependency_files): + initial_project_root = project.project_root + renamed_project_root = os.path.join(project.project_root.dirname, "renamed_project_dir") + + write_project_files(project.project_root, "local_dependency", local_dependency_files) + + # initial run + run_dbt(["deps"]) + assert len(run_dbt(["seed"])) == 1 + assert len(run_dbt(["run"])) == 2 + + self.rename_project_root(project, renamed_project_root) + yield + self.rename_project_root(project, initial_project_root) + + def test_pp_renamed_project_dir_unchanged_project_contents(self, project): + # partial parse same project in new absolute dir location, using partial_parse.msgpack created in previous dir + run_dbt(["deps"]) + assert len(run_dbt(["--partial-parse", "seed"])) == 1 + assert len(run_dbt(["--partial-parse", "run"])) == 2 + + def test_pp_renamed_project_dir_changed_project_contents(self, project): + write_file(model_two_sql, project.project_root, "models", "model_two.sql") + + # partial parse changed project in new absolute dir location, using partial_parse.msgpack created in previous dir + run_dbt(["deps"]) + len(run_dbt(["--partial-parse", "seed"])) == 1 + len(run_dbt(["--partial-parse", "run"])) == 3 diff --git a/tests/functional/utils.py b/tests/functional/utils.py index 26d31c752ac..ddfe367856b 100644 --- a/tests/functional/utils.py +++ b/tests/functional/utils.py @@ -1,13 +1,14 @@ import os from contextlib import contextmanager +from typing import Optional from pathlib import Path @contextmanager -def up_one(): +def up_one(return_path: Optional[Path] = None): current_path = Path.cwd() os.chdir("../") try: yield finally: - os.chdir(current_path) + os.chdir(return_path or current_path)