Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Fix] respect project root when loading seeds #8762

Merged
merged 9 commits into from
Oct 10, 2023
6 changes: 6 additions & 0 deletions .changes/unreleased/Fixes-20231006-134551.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Fixes
body: Enable seeds to be handled from stored manifest data
time: 2023-10-06T13:45:51.925546-04:00
custom:
Author: michelleark
Issue: "6875"
14 changes: 12 additions & 2 deletions core/dbt/context/providers.py
Original file line number Diff line number Diff line change
Expand Up @@ -862,8 +862,18 @@
def load_agate_table(self) -> agate.Table:
if not isinstance(self.model, SeedNode):
raise LoadAgateTableNotSeedError(self.model.resource_type, node=self.model)
assert self.model.root_path
path = os.path.join(self.model.root_path, self.model.original_file_path)

# include package_path for seeds defined in packages
package_path = (
os.path.join(self.config.packages_install_path, self.model.package_name)
if self.model.package_name != self.config.project_name
else "."
)
path = os.path.join(self.config.project_root, package_path, self.model.original_file_path)
if not os.path.exists(path):
assert self.model.root_path
path = os.path.join(self.model.root_path, self.model.original_file_path)

Check warning on line 875 in core/dbt/context/providers.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/context/providers.py#L874-L875

Added lines #L874 - L875 were not covered by tests

column_types = self.model.config.column_types
delimiter = self.model.config.delimiter
try:
Expand Down
4 changes: 4 additions & 0 deletions core/dbt/tests/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,10 @@ def rm_dir(directory_path):
raise FileNotFoundError(f"{directory_path} does not exist.")


def rename_dir(src_directory_path, dest_directory_path):
os.rename(src_directory_path, dest_directory_path)


# Get an artifact (usually from the target directory) such as
# manifest.json or catalog.json to use in a test
def get_artifact(*paths):
Expand Down
74 changes: 73 additions & 1 deletion tests/functional/partial_parsing/test_partial_parsing.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,17 @@
from argparse import Namespace
import pytest
from unittest import mock

from dbt.tests.util import run_dbt, get_manifest, write_file, rm_file, run_dbt_and_capture
import dbt.flags as flags
from dbt.tests.util import (
run_dbt,
get_manifest,
write_file,
rm_file,
run_dbt_and_capture,
rename_dir,
)
from tests.functional.utils import up_one
from dbt.tests.fixtures.project import write_project_files
from tests.functional.partial_parsing.fixtures import (
model_one_sql,
Expand Down Expand Up @@ -846,3 +856,65 @@ def test_pp_external_models(
run_dbt(["--partial-parse", "parse"])
assert len(manifest.nodes) == 6
assert len(manifest.external_node_unique_ids) == 4


class TestPortablePartialParsing:
@pytest.fixture(scope="class")
def models(self):
return {
"model_one.sql": model_one_sql,
}

@pytest.fixture(scope="class")
def packages(self):
return {"packages": [{"local": "local_dependency"}]}

@pytest.fixture(scope="class")
def local_dependency_files(self):
return {
"dbt_project.yml": local_dependency__dbt_project_yml,
"models": {
"schema.yml": local_dependency__models__schema_yml,
"model_to_import.sql": local_dependency__models__model_to_import_sql,
},
"macros": {"dep_macro.sql": local_dependency__macros__dep_macro_sql},
"seeds": {"seed.csv": local_dependency__seeds__seed_csv},
}

def rename_project_root(self, project, new_project_root):
with up_one(new_project_root):
rename_dir(project.project_root, new_project_root)
project.project_root = new_project_root
# flags.project_dir is set during the project test fixture, and is persisted across run_dbt calls,
# so it needs to be reset between invocations
flags.set_from_args(Namespace(PROJECT_DIR=new_project_root), None)

@pytest.fixture(scope="class", autouse=True)
def initial_run_and_rename_project_dir(self, project, local_dependency_files):
initial_project_root = project.project_root
renamed_project_root = os.path.join(project.project_root.dirname, "renamed_project_dir")

write_project_files(project.project_root, "local_dependency", local_dependency_files)

# initial run
run_dbt(["deps"])
assert len(run_dbt(["seed"])) == 1
assert len(run_dbt(["run"])) == 2

self.rename_project_root(project, renamed_project_root)
yield
self.rename_project_root(project, initial_project_root)

def test_pp_renamed_project_dir_unchanged_project_contents(self, project):
# partial parse same project in new absolute dir location, using partial_parse.msgpack created in previous dir
run_dbt(["deps"])
assert len(run_dbt(["--partial-parse", "seed"])) == 1
assert len(run_dbt(["--partial-parse", "run"])) == 2

def test_pp_renamed_project_dir_changed_project_contents(self, project):
write_file(model_two_sql, project.project_root, "models", "model_two.sql")

# partial parse changed project in new absolute dir location, using partial_parse.msgpack created in previous dir
run_dbt(["deps"])
len(run_dbt(["--partial-parse", "seed"])) == 1
len(run_dbt(["--partial-parse", "run"])) == 3
5 changes: 3 additions & 2 deletions tests/functional/utils.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
import os
from contextlib import contextmanager
from typing import Optional
from pathlib import Path


@contextmanager
def up_one():
def up_one(return_path: Optional[Path] = None):
current_path = Path.cwd()
os.chdir("../")
try:
yield
finally:
os.chdir(current_path)
os.chdir(return_path or current_path)
Loading