-
Notifications
You must be signed in to change notification settings - Fork 1.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Fix] respect project root when loading seeds (#8762)
(cherry picked from commit 964e0e4)
- Loading branch information
1 parent
7f0f239
commit 6e2cddd
Showing
6 changed files
with
188 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
kind: Fixes | ||
body: Enable seeds to be handled from stored manifest data | ||
time: 2023-10-06T13:45:51.925546-04:00 | ||
custom: | ||
Author: michelleark | ||
Issue: "6875" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,66 @@ | ||
local_dependency__dbt_project_yml = """ | ||
name: 'local_dep' | ||
version: '1.0' | ||
config-version: 2 | ||
profile: 'default' | ||
model-paths: ["models"] | ||
analysis-paths: ["analyses"] | ||
test-paths: ["tests"] | ||
seed-paths: ["seeds"] | ||
macro-paths: ["macros"] | ||
require-dbt-version: '>=0.1.0' | ||
target-path: "target" # directory which will store compiled SQL files | ||
clean-targets: # directories to be removed by `dbt clean` | ||
- "target" | ||
- "dbt_packages" | ||
seeds: | ||
quote_columns: False | ||
""" | ||
|
||
local_dependency__models__schema_yml = """ | ||
version: 2 | ||
sources: | ||
- name: seed_source | ||
schema: "{{ var('schema_override', target.schema) }}" | ||
tables: | ||
- name: "seed" | ||
columns: | ||
- name: id | ||
tests: | ||
- unique | ||
""" | ||
|
||
local_dependency__models__model_to_import_sql = """ | ||
select * from {{ ref('seed') }} | ||
""" | ||
|
||
local_dependency__macros__dep_macro_sql = """ | ||
{% macro some_overridden_macro() -%} | ||
100 | ||
{%- endmacro %} | ||
""" | ||
|
||
local_dependency__seeds__seed_csv = """id | ||
1 | ||
""" | ||
|
||
model_one_sql = """ | ||
select 1 as fun | ||
""" | ||
|
||
model_two_sql = """ | ||
select 1 as notfun | ||
""" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,86 @@ | ||
from argparse import Namespace | ||
import pytest | ||
|
||
import dbt.flags as flags | ||
from dbt.tests.util import ( | ||
run_dbt, | ||
write_file, | ||
rename_dir, | ||
) | ||
from tests.functional.utils import up_one | ||
from dbt.tests.fixtures.project import write_project_files | ||
from tests.functional.partial_parsing.fixtures import ( | ||
model_one_sql, | ||
model_two_sql, | ||
local_dependency__dbt_project_yml, | ||
local_dependency__models__schema_yml, | ||
local_dependency__models__model_to_import_sql, | ||
local_dependency__macros__dep_macro_sql, | ||
local_dependency__seeds__seed_csv, | ||
) | ||
|
||
import os | ||
|
||
os.environ["DBT_PP_TEST"] = "true" | ||
|
||
|
||
class TestPortablePartialParsing: | ||
@pytest.fixture(scope="class") | ||
def models(self): | ||
return { | ||
"model_one.sql": model_one_sql, | ||
} | ||
|
||
@pytest.fixture(scope="class") | ||
def packages(self): | ||
return {"packages": [{"local": "local_dependency"}]} | ||
|
||
@pytest.fixture(scope="class") | ||
def local_dependency_files(self): | ||
return { | ||
"dbt_project.yml": local_dependency__dbt_project_yml, | ||
"models": { | ||
"schema.yml": local_dependency__models__schema_yml, | ||
"model_to_import.sql": local_dependency__models__model_to_import_sql, | ||
}, | ||
"macros": {"dep_macro.sql": local_dependency__macros__dep_macro_sql}, | ||
"seeds": {"seed.csv": local_dependency__seeds__seed_csv}, | ||
} | ||
|
||
def rename_project_root(self, project, new_project_root): | ||
with up_one(new_project_root): | ||
rename_dir(project.project_root, new_project_root) | ||
project.project_root = new_project_root | ||
# flags.project_dir is set during the project test fixture, and is persisted across run_dbt calls, | ||
# so it needs to be reset between invocations | ||
flags.set_from_args(Namespace(PROJECT_DIR=new_project_root), None) | ||
|
||
@pytest.fixture(scope="class", autouse=True) | ||
def initial_run_and_rename_project_dir(self, project, local_dependency_files): | ||
initial_project_root = project.project_root | ||
renamed_project_root = os.path.join(project.project_root.dirname, "renamed_project_dir") | ||
|
||
write_project_files(project.project_root, "local_dependency", local_dependency_files) | ||
|
||
# initial run | ||
run_dbt(["deps"]) | ||
assert len(run_dbt(["seed"])) == 1 | ||
assert len(run_dbt(["run"])) == 2 | ||
|
||
self.rename_project_root(project, renamed_project_root) | ||
yield | ||
self.rename_project_root(project, initial_project_root) | ||
|
||
def test_pp_renamed_project_dir_unchanged_project_contents(self, project): | ||
# partial parse same project in new absolute dir location, using partial_parse.msgpack created in previous dir | ||
run_dbt(["deps"]) | ||
assert len(run_dbt(["--partial-parse", "seed"])) == 1 | ||
assert len(run_dbt(["--partial-parse", "run"])) == 2 | ||
|
||
def test_pp_renamed_project_dir_changed_project_contents(self, project): | ||
write_file(model_two_sql, project.project_root, "models", "model_two.sql") | ||
|
||
# partial parse changed project in new absolute dir location, using partial_parse.msgpack created in previous dir | ||
run_dbt(["deps"]) | ||
len(run_dbt(["--partial-parse", "seed"])) == 1 | ||
len(run_dbt(["--partial-parse", "run"])) == 3 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
import os | ||
from contextlib import contextmanager | ||
from typing import Optional | ||
from pathlib import Path | ||
|
||
|
||
@contextmanager | ||
def up_one(return_path: Optional[Path] = None): | ||
current_path = Path.cwd() | ||
os.chdir("../") | ||
try: | ||
yield | ||
finally: | ||
os.chdir(return_path or current_path) |