diff --git a/tests/functional/context_methods/first_dependency.py b/tests/functional/context_methods/first_dependency.py deleted file mode 100644 index 8e1365be..00000000 --- a/tests/functional/context_methods/first_dependency.py +++ /dev/null @@ -1,95 +0,0 @@ -from dbt.tests.fixtures.project import write_project_files -import pytest - - -first_dependency__dbt_project_yml = """ -name: 'first_dep' -version: '1.0' -config-version: 2 - -profile: 'default' - -model-paths: ["models"] -analysis-paths: ["analyses"] -test-paths: ["tests"] -seed-paths: ["seeds"] -macro-paths: ["macros"] - -require-dbt-version: '>=0.1.0' - -target-path: "target" # directory which will store compiled SQL files -clean-targets: # directories to be removed by `dbt clean` - - "target" - - "dbt_packages" - -vars: - first_dep: - first_dep_global: 'first_dep_global_value_overridden' - test_config_root_override: 'configured_from_dependency' - test_config_package: 'configured_from_dependency' - -seeds: - quote_columns: True - -""" - -first_dependency__models__nested__first_dep_model_sql = """ -select - '{{ var("first_dep_global") }}' as first_dep_global, - '{{ var("from_root_to_first") }}' as from_root -""" - -first_dependency__seeds__first_dep_expected_csv = """first_dep_global,from_root -first_dep_global_value_overridden,root_first_value -""" - -first_dependency__models__nested__first_dep_model_var_expected_csv = """test_config_root_override,test_config_package -configured_from_root,configured_from_dependency -""" - -first_dependency__models__nested__first_dep_model_var_sql = """ -select - '{{ config.get("test_config_root_override") }}' as test_config_root_override, - '{{ config.get("test_config_package") }}' as test_config_package -""" - -first_dependency__model_var_in_config_schema = """ -models: -- name: first_dep_model - config: - test_config_root_override: "{{ var('test_config_root_override') }}" - test_config_package: "{{ var('test_config_package') }}" -""" - - -class FirstDependencyProject: - @pytest.fixture(scope="class") - def first_dependency(self, project): - first_dependency_files = { - "dbt_project.yml": first_dependency__dbt_project_yml, - "models": { - "nested": { - "first_dep_model.sql": first_dependency__models__nested__first_dep_model_sql - } - }, - "seeds": {"first_dep_expected.csv": first_dependency__seeds__first_dep_expected_csv}, - } - write_project_files(project.project_root, "first_dependency", first_dependency_files) - - -class FirstDependencyConfigProject: - @pytest.fixture(scope="class") - def first_dependency(self, project): - first_dependency_files = { - "dbt_project.yml": first_dependency__dbt_project_yml, - "models": { - "nested": { - "first_dep_model.sql": first_dependency__models__nested__first_dep_model_var_sql, - "schema.yml": first_dependency__model_var_in_config_schema, - } - }, - "seeds": { - "first_dep_expected.csv": first_dependency__models__nested__first_dep_model_var_expected_csv - }, - } - write_project_files(project.project_root, "first_dependency", first_dependency_files) diff --git a/tests/functional/context_methods/test_builtin_functions.py b/tests/functional/context_methods/test_builtin_functions.py deleted file mode 100644 index b8a47b34..00000000 --- a/tests/functional/context_methods/test_builtin_functions.py +++ /dev/null @@ -1,143 +0,0 @@ -import json - -from dbt.tests.util import write_file -from dbt_common.exceptions import CompilationError -import pytest - -from tests.functional.utils import run_dbt, run_dbt_and_capture - - -macros__validate_set_sql = """ -{% macro validate_set() %} - {% set set_result = set([1, 2, 2, 3, 'foo', False]) %} - {{ log("set_result: " ~ set_result) }} - {% set set_strict_result = set_strict([1, 2, 2, 3, 'foo', False]) %} - {{ log("set_strict_result: " ~ set_strict_result) }} -{% endmacro %} -""" - -macros__validate_zip_sql = """ -{% macro validate_zip() %} - {% set list_a = [1, 2] %} - {% set list_b = ['foo', 'bar'] %} - {% set zip_result = zip(list_a, list_b) | list %} - {{ log("zip_result: " ~ zip_result) }} - {% set zip_strict_result = zip_strict(list_a, list_b) | list %} - {{ log("zip_strict_result: " ~ zip_strict_result) }} -{% endmacro %} -""" - -macros__validate_invocation_sql = """ -{% macro validate_invocation(my_variable) %} - -- check a specific value - {{ log("use_colors: "~ invocation_args_dict['use_colors']) }} - -- whole dictionary (as string) - {{ log("invocation_result: "~ invocation_args_dict) }} -{% endmacro %} -""" - -macros__validate_dbt_metadata_envs_sql = """ -{% macro validate_dbt_metadata_envs() %} - {{ log("dbt_metadata_envs_result:"~ dbt_metadata_envs) }} -{% endmacro %} -""" - -models__set_exception_sql = """ -{% set set_strict_result = set_strict(1) %} -""" - -models__zip_exception_sql = """ -{% set zip_strict_result = zip_strict(1) %} -""" - - -def parse_json_logs(json_log_output): - parsed_logs = [] - for line in json_log_output.split("\n"): - try: - log = json.loads(line) - except ValueError: - continue - - parsed_logs.append(log) - - return parsed_logs - - -def find_result_in_parsed_logs(parsed_logs, result_name): - return next( - ( - item["data"]["msg"] - for item in parsed_logs - if result_name in item["data"].get("msg", "msg") - ), - False, - ) - - -class TestContextBuiltins: - @pytest.fixture(scope="class") - def macros(self): - return { - "validate_set.sql": macros__validate_set_sql, - "validate_zip.sql": macros__validate_zip_sql, - "validate_invocation.sql": macros__validate_invocation_sql, - "validate_dbt_metadata_envs.sql": macros__validate_dbt_metadata_envs_sql, - } - - def test_builtin_set_function(self, project): - _, log_output = run_dbt_and_capture(["--debug", "run-operation", "validate_set"]) - - # The order of the set isn't guaranteed so we can't check for the actual set in the logs - assert "set_result: " in log_output - assert "False" in log_output - assert "set_strict_result: " in log_output - - def test_builtin_zip_function(self, project): - _, log_output = run_dbt_and_capture(["--debug", "run-operation", "validate_zip"]) - - expected_zip = [(1, "foo"), (2, "bar")] - assert f"zip_result: {expected_zip}" in log_output - assert f"zip_strict_result: {expected_zip}" in log_output - - def test_builtin_invocation_args_dict_function(self, project): - _, log_output = run_dbt_and_capture( - [ - "--debug", - "--log-format=json", - "run-operation", - "validate_invocation", - "--args", - "{my_variable: test_variable}", - ] - ) - - parsed_logs = parse_json_logs(log_output) - use_colors = result = find_result_in_parsed_logs(parsed_logs, "use_colors") - assert use_colors == "use_colors: True" - invocation_dict = find_result_in_parsed_logs(parsed_logs, "invocation_result") - assert result - # The result should include a dictionary of all flags with values that aren't None - expected = ( - "'send_anonymous_usage_stats': False", - "'quiet': False", - "'print': True", - "'cache_selected_only': False", - "'macro': 'validate_invocation'", - "'args': {'my_variable': 'test_variable'}", - "'which': 'run-operation'", - "'indirect_selection': 'eager'", - ) - assert all(element in invocation_dict for element in expected) - - -class TestContextBuiltinExceptions: - # Assert compilation errors are raised with _strict equivalents - def test_builtin_function_exception(self, project): - write_file(models__set_exception_sql, project.project_root, "models", "raise.sql") - with pytest.raises(CompilationError): - run_dbt(["compile"]) - - write_file(models__zip_exception_sql, project.project_root, "models", "raise.sql") - with pytest.raises(CompilationError): - run_dbt(["compile"]) diff --git a/tests/functional/context_methods/test_cli_var_override.py b/tests/functional/context_methods/test_cli_var_override.py deleted file mode 100644 index 757ab521..00000000 --- a/tests/functional/context_methods/test_cli_var_override.py +++ /dev/null @@ -1,66 +0,0 @@ -from dbt.tests.util import run_dbt -import pytest - - -models_override__schema_yml = """ -version: 2 -models: -- name: test_vars - columns: - - name: field - data_tests: - - accepted_values: - values: - - override -""" - -models_override__test_vars_sql = """ -select '{{ var("required") }}'::varchar as field -""" - - -# Tests that cli vars override vars set in the project config -class TestCLIVarOverride: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_override__schema_yml, - "test_vars.sql": models_override__test_vars_sql, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "vars": { - "required": "present", - }, - } - - def test__override_vars_global(self, project): - run_dbt(["run", "--vars", "{required: override}"]) - run_dbt(["test"]) - - -# This one switches to setting a var in 'test' -class TestCLIVarOverridePorject: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_override__schema_yml, - "test_vars.sql": models_override__test_vars_sql, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "vars": { - "test": { - "required": "present", - }, - }, - } - - def test__override_vars_project_level(self, project): - # This should be "override" - run_dbt(["run", "--vars", "{required: override}"]) - run_dbt(["test"]) diff --git a/tests/functional/context_methods/test_cli_vars.py b/tests/functional/context_methods/test_cli_vars.py deleted file mode 100644 index 8f6d6e8d..00000000 --- a/tests/functional/context_methods/test_cli_vars.py +++ /dev/null @@ -1,205 +0,0 @@ -from dbt.tests.fixtures.project import write_project_files -from dbt.tests.util import get_artifact, run_dbt, write_config_file -from dbt_common.exceptions import CompilationError, DbtRuntimeError -import pytest -import yaml - - -models_complex__schema_yml = """ -version: 2 -models: -- name: complex_model - columns: - - name: var_1 - data_tests: - - accepted_values: - values: - - abc - - name: var_2 - data_tests: - - accepted_values: - values: - - def - - name: var_3 - data_tests: - - accepted_values: - values: - - jkl -""" - -models_complex__complex_model_sql = """ -select - '{{ var("variable_1") }}'::varchar as var_1, - '{{ var("variable_2")[0] }}'::varchar as var_2, - '{{ var("variable_3")["value"] }}'::varchar as var_3 -""" - -models_simple__schema_yml = """ -version: 2 -models: -- name: simple_model - columns: - - name: simple - data_tests: - - accepted_values: - values: - - abc -""" - -models_simple__simple_model_sql = """ -select - '{{ var("simple") }}'::varchar as simple -""" - -really_simple_model_sql = """ -select 'abc' as simple -""" - - -class TestCLIVars: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_complex__schema_yml, - "complex_model.sql": models_complex__complex_model_sql, - } - - def test__cli_vars_longform(self, project): - cli_vars = { - "variable_1": "abc", - "variable_2": ["def", "ghi"], - "variable_3": {"value": "jkl"}, - } - results = run_dbt(["run", "--vars", yaml.dump(cli_vars)]) - assert len(results) == 1 - results = run_dbt(["test", "--vars", yaml.dump(cli_vars)]) - assert len(results) == 3 - - -class TestCLIVarsSimple: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_simple__schema_yml, - "simple_model.sql": models_simple__simple_model_sql, - } - - def test__cli_vars_shorthand(self, project): - results = run_dbt(["run", "--vars", "simple: abc"]) - assert len(results) == 1 - results = run_dbt(["test", "--vars", "simple: abc"]) - assert len(results) == 1 - - def test__cli_vars_longer(self, project): - results = run_dbt(["run", "--vars", "{simple: abc, unused: def}"]) - assert len(results) == 1 - results = run_dbt(["test", "--vars", "{simple: abc, unused: def}"]) - assert len(results) == 1 - run_results = get_artifact(project.project_root, "target", "run_results.json") - assert run_results["args"]["vars"] == {"simple": "abc", "unused": "def"} - - -class TestCLIVarsProfile: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_simple__schema_yml, - "simple_model.sql": really_simple_model_sql, - } - - def test_cli_vars_in_profile(self, project, dbt_profile_data): - profile = dbt_profile_data - profile["test"]["outputs"]["default"]["host"] = "{{ var('db_host') }}" - write_config_file(profile, project.profiles_dir, "profiles.yml") - with pytest.raises(DbtRuntimeError): - results = run_dbt(["run"]) - results = run_dbt(["run", "--vars", "db_host: localhost"]) - assert len(results) == 1 - - -class TestCLIVarsPackages: - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project_root, dbt_integration_project): # noqa: F811 - write_project_files(project_root, "dbt_integration_project", dbt_integration_project) - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_simple__schema_yml, - "simple_model.sql": really_simple_model_sql, - } - - @pytest.fixture(scope="class") - def packages_config(self): - return {"packages": [{"local": "dbt_integration_project"}]} - - def test_cli_vars_in_packages(self, project, packages_config): - # Run working deps and run commands - run_dbt(["deps"]) - results = run_dbt(["run"]) - assert len(results) == 1 - - # Change packages.yml to contain a var - packages = packages_config - packages["packages"][0]["local"] = "{{ var('path_to_project') }}" - write_config_file(packages, project.project_root, "packages.yml") - - # Without vars args deps fails - with pytest.raises(DbtRuntimeError): - run_dbt(["deps"]) - - # With vars arg deps succeeds - results = run_dbt(["deps", "--vars", "path_to_project: dbt_integration_project"]) - assert results is None - - -initial_selectors_yml = """ -selectors: - - name: dev_defer_snapshots - default: "{{ target.name == 'dev' | as_bool }}" - definition: - method: fqn - value: '*' - exclude: - - method: config.materialized - value: snapshot -""" - -var_selectors_yml = """ -selectors: - - name: dev_defer_snapshots - default: "{{ var('snapshot_target') == 'dev' | as_bool }}" - definition: - method: fqn - value: '*' - exclude: - - method: config.materialized - value: snapshot -""" - - -class TestCLIVarsSelectors: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_simple__schema_yml, - "simple_model.sql": really_simple_model_sql, - } - - @pytest.fixture(scope="class") - def selectors(self): - return initial_selectors_yml - - def test_vars_in_selectors(self, project): - # initially runs ok - results = run_dbt(["run"]) - assert len(results) == 1 - - # Update the selectors.yml file to have a var - write_config_file(var_selectors_yml, project.project_root, "selectors.yml") - with pytest.raises(CompilationError): - run_dbt(["run"]) - - # Var in cli_vars works - results = run_dbt(["run", "--vars", "snapshot_target: dev"]) - assert len(results) == 1 diff --git a/tests/functional/context_methods/test_custom_env_vars.py b/tests/functional/context_methods/test_custom_env_vars.py deleted file mode 100644 index 50a9b00c..00000000 --- a/tests/functional/context_methods/test_custom_env_vars.py +++ /dev/null @@ -1,36 +0,0 @@ -import json -import os - -import pytest - -from tests.functional.utils import run_dbt_and_capture - - -def parse_json_logs(json_log_output): - parsed_logs = [] - for line in json_log_output.split("\n"): - try: - log = json.loads(line) - except ValueError: - continue - - parsed_logs.append(log) - - return parsed_logs - - -class TestCustomVarInLogs: - @pytest.fixture(scope="class", autouse=True) - def setup(self): - # on windows, python uppercases env var names because windows is case insensitive - os.environ["DBT_ENV_CUSTOM_ENV_SOME_VAR"] = "value" - yield - del os.environ["DBT_ENV_CUSTOM_ENV_SOME_VAR"] - - def test_extra_filled(self, project): - _, log_output = run_dbt_and_capture( - ["--log-format=json", "deps"], - ) - logs = parse_json_logs(log_output) - for log in logs: - assert log["info"].get("extra") == {"SOME_VAR": "value"} diff --git a/tests/functional/context_methods/test_env_vars.py b/tests/functional/context_methods/test_env_vars.py deleted file mode 100644 index 0bfbd01c..00000000 --- a/tests/functional/context_methods/test_env_vars.py +++ /dev/null @@ -1,195 +0,0 @@ -import os - -from dbt.constants import DEFAULT_ENV_PLACEHOLDER, SECRET_ENV_PREFIX -from dbt.tests.util import get_manifest -import pytest - -from tests.functional.utils import run_dbt, run_dbt_and_capture - - -context_sql = """ - -{{ - config( - materialized='table' - ) -}} - -select - - -- compile-time variables - '{{ this }}' as "this", - '{{ this.name }}' as "this.name", - '{{ this.schema }}' as "this.schema", - '{{ this.table }}' as "this.table", - - '{{ target.dbname }}' as "target.dbname", - '{{ target.host }}' as "target.host", - '{{ target.name }}' as "target.name", - '{{ target.schema }}' as "target.schema", - '{{ target.type }}' as "target.type", - '{{ target.user }}' as "target.user", - '{{ target.get("pass", "") }}' as "target.pass", -- not actually included, here to test that it is _not_ present! - {{ target.port }} as "target.port", - {{ target.threads }} as "target.threads", - - -- runtime variables - '{{ run_started_at }}' as run_started_at, - '{{ invocation_id }}' as invocation_id, - '{{ thread_id }}' as thread_id, - - '{{ env_var("DBT_TEST_ENV_VAR") }}' as env_var, - '{{ env_var("DBT_TEST_IGNORE_DEFAULT", "ignored_default_val") }}' as env_var_ignore_default, - '{{ env_var("DBT_TEST_USE_DEFAULT", "use_my_default_val") }}' as env_var_use_default, - 'secret_variable' as env_var_secret, -- make sure the value itself is scrubbed from the logs - '{{ env_var("DBT_TEST_NOT_SECRET") }}' as env_var_not_secret - -""" - - -class TestEnvVars: - @pytest.fixture(scope="class") - def models(self): - return {"context.sql": context_sql} - - @pytest.fixture(scope="class", autouse=True) - def setup(self): - os.environ["DBT_TEST_ENV_VAR"] = "1" - os.environ["DBT_TEST_USER"] = "root" - os.environ["DBT_TEST_PASS"] = "password" - os.environ[SECRET_ENV_PREFIX + "SECRET"] = "secret_variable" - os.environ["DBT_TEST_NOT_SECRET"] = "regular_variable" - os.environ["DBT_TEST_IGNORE_DEFAULT"] = "ignored_default" - yield - del os.environ["DBT_TEST_ENV_VAR"] - del os.environ["DBT_TEST_USER"] - del os.environ[SECRET_ENV_PREFIX + "SECRET"] - del os.environ["DBT_TEST_NOT_SECRET"] - del os.environ["DBT_TEST_IGNORE_DEFAULT"] - - @pytest.fixture(scope="class") - def profiles_config_update(self, unique_schema): - return { - "test": { - "outputs": { - # don't use env_var's here so the integration tests can run - # seed sql statements and the like. default target is used - "dev": { - "type": "postgres", - "threads": 1, - "host": "localhost", - "port": 5432, - "user": "root", - "pass": "password", - "dbname": "dbt", - "schema": unique_schema, - }, - "prod": { - "type": "postgres", - "threads": 1, - "host": "localhost", - "port": 5432, - # root/password - "user": "{{ env_var('DBT_TEST_USER') }}", - "pass": "{{ env_var('DBT_TEST_PASS') }}", - "dbname": "dbt", - "schema": unique_schema, - }, - }, - "target": "dev", - } - } - - def get_ctx_vars(self, project): - fields = [ - "this", - "this.name", - "this.schema", - "this.table", - "target.dbname", - "target.host", - "target.name", - "target.port", - "target.schema", - "target.threads", - "target.type", - "target.user", - "target.pass", - "run_started_at", - "invocation_id", - "thread_id", - "env_var", - ] - field_list = ", ".join(['"{}"'.format(f) for f in fields]) - query = "select {field_list} from {schema}.context".format( - field_list=field_list, schema=project.test_schema - ) - vals = project.run_sql(query, fetch="all") - ctx = dict([(k, v) for (k, v) in zip(fields, vals[0])]) - return ctx - - def test_env_vars_dev( - self, - project, - ): - results = run_dbt(["run"]) - assert len(results) == 1 - ctx = self.get_ctx_vars(project) - - manifest = get_manifest(project.project_root) - expected = { - "DBT_TEST_ENV_VAR": "1", - "DBT_TEST_NOT_SECRET": "regular_variable", - "DBT_TEST_IGNORE_DEFAULT": "ignored_default", - "DBT_TEST_USE_DEFAULT": DEFAULT_ENV_PLACEHOLDER, - } - assert manifest.env_vars == expected - - this = '"{}"."{}"."context"'.format(project.database, project.test_schema) - assert ctx["this"] == this - - assert ctx["this.name"] == "context" - assert ctx["this.schema"] == project.test_schema - assert ctx["this.table"] == "context" - - assert ctx["target.dbname"] == "dbt" - assert ctx["target.host"] == "localhost" - assert ctx["target.name"] == "dev" - assert ctx["target.port"] == 5432 - assert ctx["target.schema"] == project.test_schema - assert ctx["target.threads"] == 1 - assert ctx["target.type"] == "postgres" - assert ctx["target.user"] == "root" - assert ctx["target.pass"] == "" - - assert ctx["env_var"] == "1" - - def test_env_vars_prod(self, project): - results = run_dbt(["run", "--target", "prod"]) - assert len(results) == 1 - ctx = self.get_ctx_vars(project) - - this = '"{}"."{}"."context"'.format(project.database, project.test_schema) - assert ctx["this"] == this - - assert ctx["this.name"] == "context" - assert ctx["this.schema"] == project.test_schema - assert ctx["this.table"] == "context" - - assert ctx["target.dbname"] == "dbt" - assert ctx["target.host"] == "localhost" - assert ctx["target.name"] == "prod" - assert ctx["target.port"] == 5432 - assert ctx["target.schema"] == project.test_schema - assert ctx["target.threads"] == 1 - assert ctx["target.type"] == "postgres" - assert ctx["target.user"] == "root" - assert ctx["target.pass"] == "" - assert ctx["env_var"] == "1" - - def test_env_vars_secrets(self, project): - os.environ["DBT_DEBUG"] = "True" - _, log_output = run_dbt_and_capture(["run", "--target", "prod"]) - - assert not ("secret_variable" in log_output) - assert "regular_variable" in log_output diff --git a/tests/functional/context_methods/test_secret_env_vars.py b/tests/functional/context_methods/test_secret_env_vars.py deleted file mode 100644 index a6a5537a..00000000 --- a/tests/functional/context_methods/test_secret_env_vars.py +++ /dev/null @@ -1,185 +0,0 @@ -import os - -from dbt.constants import SECRET_ENV_PREFIX -from dbt.exceptions import ParsingError -from dbt.tests.util import read_file -from dbt_common.exceptions import DbtInternalError -import pytest - -from tests.functional.context_methods.first_dependency import FirstDependencyProject -from tests.functional.utils import run_dbt, run_dbt_and_capture - - -secret_bad__context_sql = """ - -{{ - config( - materialized='table' - ) -}} - -select - - '{{ env_var("DBT_TEST_ENV_VAR") }}' as env_var, - '{{ env_var("DBT_ENV_SECRET_SECRET") }}' as env_var_secret, -- this should raise an error! - '{{ env_var("DBT_TEST_NOT_SECRET") }}' as env_var_not_secret - -""" - - -class TestDisallowSecretModel: - @pytest.fixture(scope="class") - def models(self): - return {"context.sql": secret_bad__context_sql} - - def test_disallow_secret(self, project): - with pytest.raises(ParsingError): - run_dbt(["compile"]) - - -models__context_sql = """ -{{ - config( - materialized='table' - ) -}} - -select - - -- compile-time variables - '{{ this }}' as "this", - '{{ this.name }}' as "this.name", - '{{ this.schema }}' as "this.schema", - '{{ this.table }}' as "this.table", - - '{{ target.dbname }}' as "target.dbname", - '{{ target.host }}' as "target.host", - '{{ target.name }}' as "target.name", - '{{ target.schema }}' as "target.schema", - '{{ target.type }}' as "target.type", - '{{ target.user }}' as "target.user", - '{{ target.get("pass", "") }}' as "target.pass", -- not actually included, here to test that it is _not_ present! - {{ target.port }} as "target.port", - {{ target.threads }} as "target.threads", - - -- runtime variables - '{{ run_started_at }}' as run_started_at, - '{{ invocation_id }}' as invocation_id, - '{{ thread_id }}' as thread_id, - - '{{ env_var("DBT_TEST_ENV_VAR") }}' as env_var, - 'secret_variable' as env_var_secret, -- make sure the value itself is scrubbed from the logs - '{{ env_var("DBT_TEST_NOT_SECRET") }}' as env_var_not_secret -""" - - -class TestAllowSecretProfilePackage(FirstDependencyProject): - @pytest.fixture(scope="class", autouse=True) - def setup(self): - os.environ[SECRET_ENV_PREFIX + "USER"] = "root" - os.environ[SECRET_ENV_PREFIX + "PASS"] = "password" - os.environ[SECRET_ENV_PREFIX + "PACKAGE"] = "first_dependency" - os.environ[SECRET_ENV_PREFIX + "GIT_TOKEN"] = "abc123" - yield - del os.environ[SECRET_ENV_PREFIX + "USER"] - del os.environ[SECRET_ENV_PREFIX + "PASS"] - del os.environ[SECRET_ENV_PREFIX + "PACKAGE"] - del os.environ[SECRET_ENV_PREFIX + "GIT_TOKEN"] - - @pytest.fixture(scope="class") - def models(self): - return {"context.sql": models__context_sql} - - @pytest.fixture(scope="class") - def packages(self): - return { - "packages": [ - { - # the raw value of this secret *will* be written to lock file - "local": "{{ env_var('DBT_ENV_SECRET_PACKAGE') }}" - }, - { - # this secret env var will *not* be written to lock file - "git": "https://{{ env_var('DBT_ENV_SECRET_GIT_TOKEN') }}@github.com/dbt-labs/dbt-external-tables.git" - }, - { - # this secret env var will *not* be written to lock file - "tarball": "https://{{ env_var('DBT_ENV_SECRET_GIT_TOKEN') }}@github.com/dbt-labs/dbt-utils/archive/refs/tags/1.1.1.tar.gz", - "name": "dbt_utils", - }, - ] - } - - @pytest.fixture(scope="class") - def profile_target(self): - return { - "type": "postgres", - "threads": 1, - "host": "localhost", - "port": 5432, - # root/password - "user": "{{ env_var('DBT_ENV_SECRET_USER') }}", - "pass": "{{ env_var('DBT_ENV_SECRET_PASS') }}", - "dbname": "dbt", - } - - def test_allow_secrets(self, project, first_dependency): - _, log_output = run_dbt_and_capture(["deps"]) - lock_file_contents = read_file("package-lock.yml") - - # this will not be written to logs or lock file - assert not ("abc123" in log_output) - assert not ("abc123" in lock_file_contents) - assert "{{ env_var('DBT_ENV_SECRET_GIT_TOKEN') }}" in lock_file_contents - - # this will be scrubbed from logs, but not from the lock file - assert not ("first_dependency" in log_output) - assert "first_dependency" in lock_file_contents - - -class TestCloneFailSecretScrubbed: - @pytest.fixture(scope="class", autouse=True) - def setup(self): - os.environ[SECRET_ENV_PREFIX + "GIT_TOKEN"] = "abc123" - - @pytest.fixture(scope="class") - def models(self): - return {"context.sql": models__context_sql} - - @pytest.fixture(scope="class") - def packages(self): - return { - "packages": [ - { - "git": "https://fakeuser:{{ env_var('DBT_ENV_SECRET_GIT_TOKEN') }}@github.com/dbt-labs/fake-repo.git" - }, - ] - } - - def test_fail_clone_with_scrubbing(self, project): - with pytest.raises(DbtInternalError) as excinfo: - _, log_output = run_dbt_and_capture(["deps"]) - - assert "abc123" not in str(excinfo.value) - - -class TestCloneFailSecretNotRendered(TestCloneFailSecretScrubbed): - # as above, with some Jinja manipulation - @pytest.fixture(scope="class") - def packages(self): - return { - "packages": [ - { - "git": "https://fakeuser:{{ env_var('DBT_ENV_SECRET_GIT_TOKEN') | join(' ') }}@github.com/dbt-labs/fake-repo.git" - }, - ] - } - - def test_fail_clone_with_scrubbing(self, project): - with pytest.raises(DbtInternalError) as excinfo: - _, log_output = run_dbt_and_capture(["deps"]) - - # we should not see any manipulated form of the secret value (abc123) here - # we should see a manipulated form of the placeholder instead - assert "a b c 1 2 3" not in str(excinfo.value) - assert "D B T _ E N V _ S E C R E T _ G I T _ T O K E N" in str(excinfo.value) diff --git a/tests/functional/context_methods/test_var_dependency.py b/tests/functional/context_methods/test_var_dependency.py deleted file mode 100644 index a0c06db7..00000000 --- a/tests/functional/context_methods/test_var_dependency.py +++ /dev/null @@ -1,82 +0,0 @@ -from dbt.tests.util import check_relations_equal, run_dbt -import pytest - -from tests.functional.context_methods.first_dependency import ( - FirstDependencyConfigProject, - FirstDependencyProject, -) - - -dependency_seeds__root_model_expected_csv = """first_dep_global,from_root -dep_never_overridden,root_root_value -""" - -dependency_models__inside__model_sql = """ -select - '{{ var("first_dep_override") }}' as first_dep_global, - '{{ var("from_root_to_root") }}' as from_root - -""" - - -class TestVarDependencyInheritance(FirstDependencyProject): - @pytest.fixture(scope="class") - def seeds(self): - return {"root_model_expected.csv": dependency_seeds__root_model_expected_csv} - - @pytest.fixture(scope="class") - def models(self): - return {"inside": {"model.sql": dependency_models__inside__model_sql}} - - @pytest.fixture(scope="class") - def packages(self): - return { - "packages": [ - {"local": "first_dependency"}, - ] - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "vars": { - "first_dep_override": "dep_never_overridden", - "test": { - "from_root_to_root": "root_root_value", - }, - "first_dep": { - "from_root_to_first": "root_first_value", - }, - }, - } - - def test_var_mutual_overrides_v1_conversion(self, project, first_dependency): - run_dbt(["deps"]) - assert len(run_dbt(["seed"])) == 2 - assert len(run_dbt(["run"])) == 2 - check_relations_equal(project.adapter, ["root_model_expected", "model"]) - check_relations_equal(project.adapter, ["first_dep_expected", "first_dep_model"]) - - -class TestVarConfigDependencyInheritance(FirstDependencyConfigProject): - @pytest.fixture(scope="class") - def packages(self): - return { - "packages": [ - {"local": "first_dependency"}, - ] - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "vars": { - "test_config_root_override": "configured_from_root", - }, - } - - def test_root_var_overrides_package_var(self, project, first_dependency): - run_dbt(["deps"]) - run_dbt(["seed"]) - assert len(run_dbt(["run"])) == 1 - check_relations_equal(project.adapter, ["first_dep_expected", "first_dep_model"]) diff --git a/tests/functional/context_methods/test_var_in_generate_name.py b/tests/functional/context_methods/test_var_in_generate_name.py deleted file mode 100644 index f36bec3a..00000000 --- a/tests/functional/context_methods/test_var_in_generate_name.py +++ /dev/null @@ -1,43 +0,0 @@ -from dbt.tests.util import run_dbt, update_config_file -from dbt_common.exceptions import CompilationError -import pytest - - -model_sql = """ -select 1 as id -""" - -bad_generate_macros__generate_names_sql = """ -{% macro generate_schema_name(custom_schema_name, node) -%} - {% do var('somevar') %} - {% do return(dbt.generate_schema_name(custom_schema_name, node)) %} -{%- endmacro %} - -""" - - -class TestMissingVarGenerateNameMacro: - @pytest.fixture(scope="class") - def macros(self): - return {"generate_names.sql": bad_generate_macros__generate_names_sql} - - @pytest.fixture(scope="class") - def models(self): - return {"model.sql": model_sql} - - def test_generate_schema_name_var(self, project): - # var isn't set, so generate_name macro fails - with pytest.raises(CompilationError) as excinfo: - run_dbt(["compile"]) - - assert "Required var 'somevar' not found in config" in str(excinfo.value) - - # globally scoped -- var is set at top-level - update_config_file({"vars": {"somevar": 1}}, project.project_root, "dbt_project.yml") - run_dbt(["compile"]) - - # locally scoped -- var is set in 'test' scope - update_config_file( - {"vars": {"test": {"somevar": 1}}}, project.project_root, "dbt_project.yml" - ) - run_dbt(["compile"]) diff --git a/tests/functional/context_methods/test_yaml_functions.py b/tests/functional/context_methods/test_yaml_functions.py deleted file mode 100644 index 8996abc9..00000000 --- a/tests/functional/context_methods/test_yaml_functions.py +++ /dev/null @@ -1,49 +0,0 @@ -from dbt.tests.util import run_dbt -import pytest - - -tests__from_yaml_sql = """ -{% set simplest = (fromyaml('a: 1') == {'a': 1}) %} -{% set nested_data %} -a: - b: - - c: 1 - d: 2 - - c: 3 - d: 4 -{% endset %} -{% set nested = (fromyaml(nested_data) == {'a': {'b': [{'c': 1, 'd': 2}, {'c': 3, 'd': 4}]}}) %} - -(select 'simplest' as name {% if simplest %}limit 0{% endif %}) -union all -(select 'nested' as name {% if nested %}limit 0{% endif %}) -""" - -tests__to_yaml_sql = """ -{% set simplest = (toyaml({'a': 1}) == 'a: 1\\n') %} -{% set default_sort = (toyaml({'b': 2, 'a': 1}) == 'b: 2\\na: 1\\n') %} -{% set unsorted = (toyaml({'b': 2, 'a': 1}, sort_keys=False) == 'b: 2\\na: 1\\n') %} -{% set sorted = (toyaml({'b': 2, 'a': 1}, sort_keys=True) == 'a: 1\\nb: 2\\n') %} -{% set default_results = (toyaml({'a': adapter}, 'failed') == 'failed') %} - -(select 'simplest' as name {% if simplest %}limit 0{% endif %}) -union all -(select 'default_sort' as name {% if default_sort %}limit 0{% endif %}) -union all -(select 'unsorted' as name {% if unsorted %}limit 0{% endif %}) -union all -(select 'sorted' as name {% if sorted %}limit 0{% endif %}) -union all -(select 'default_results' as name {% if default_results %}limit 0{% endif %}) -""" - - -class TestContextVars: - # This test has no actual models - - @pytest.fixture(scope="class") - def tests(self): - return {"from_yaml.sql": tests__from_yaml_sql, "to_yaml.sql": tests__to_yaml_sql} - - def test_json_data_tests(self, project): - assert len(run_dbt(["test"])) == 2 diff --git a/tests/functional/defer_state/fixtures.py b/tests/functional/defer_state/fixtures.py deleted file mode 100644 index 8b1d3d35..00000000 --- a/tests/functional/defer_state/fixtures.py +++ /dev/null @@ -1,424 +0,0 @@ -seed_csv = """id,name -1,Alice -2,Bob -""" - -table_model_sql = """ -{{ config(materialized='table') }} -select * from {{ ref('ephemeral_model') }} - --- establish a macro dependency to trigger state:modified.macros --- depends on: {{ my_macro() }} -""" - -table_model_now_view_sql = """ -{{ config(materialized='view') }} -select * from {{ ref('ephemeral_model') }} - --- establish a macro dependency to trigger state:modified.macros --- depends on: {{ my_macro() }} -""" - -table_model_now_incremental_sql = """ -{{ config(materialized='incremental', on_schema_change='append_new_columns') }} -select * from {{ ref('ephemeral_model') }} - --- establish a macro dependency to trigger state:modified.macros --- depends on: {{ my_macro() }} -""" - -changed_table_model_sql = """ -{{ config(materialized='table') }} -select 1 as fun -""" - -view_model_sql = """ -select * from {{ ref('seed') }} - --- establish a macro dependency that trips infinite recursion if not handled --- depends on: {{ my_infinitely_recursive_macro() }} -""" - -view_model_now_table_sql = """ -{{ config(materialized='table') }} -select * from {{ ref('seed') }} - --- establish a macro dependency that trips infinite recursion if not handled --- depends on: {{ my_infinitely_recursive_macro() }} -""" - -changed_view_model_sql = """ -select * from no.such.table -""" - -ephemeral_model_sql = """ -{{ config(materialized='ephemeral') }} -select * from {{ ref('view_model') }} -""" - -changed_ephemeral_model_sql = """ -{{ config(materialized='ephemeral') }} -select * from no.such.table -""" - -schema_yml = """ -version: 2 -models: - - name: view_model - columns: - - name: id - data_tests: - - unique: - severity: error - - not_null - - name: name -""" - -no_contract_schema_yml = """ -version: 2 -models: - - name: table_model - config: {} - columns: - - name: id - data_type: integer - data_tests: - - unique: - severity: error - - not_null - - name: name - data_type: text -""" - -contract_schema_yml = """ -version: 2 -models: - - name: table_model - config: - contract: - enforced: True - columns: - - name: id - data_type: integer - data_tests: - - unique: - severity: error - - not_null - - name: name - data_type: text -""" - -modified_contract_schema_yml = """ -version: 2 -models: - - name: table_model - config: - contract: - enforced: True - columns: - - name: id - data_type: integer - data_tests: - - unique: - severity: error - - not_null - - name: user_name - data_type: text -""" - -disabled_contract_schema_yml = """ -version: 2 -models: - - name: table_model - config: - contract: - enforced: False - columns: - - name: id - data_type: integer - data_tests: - - unique: - severity: error - - not_null - - name: name - data_type: text -""" - -versioned_no_contract_schema_yml = """ -version: 2 -models: - - name: table_model - config: {} - versions: - - v: 1 - columns: - - name: id - data_type: integer - data_tests: - - unique: - severity: error - - not_null - - name: name - data_type: text -""" - -versioned_contract_schema_yml = """ -version: 2 -models: - - name: table_model - config: - contract: - enforced: True - versions: - - v: 1 - columns: - - name: id - data_type: integer - data_tests: - - unique: - severity: error - - not_null - - name: name - data_type: text -""" - -versioned_modified_contract_schema_yml = """ -version: 2 -models: - - name: table_model - config: - contract: - enforced: True - versions: - - v: 1 - columns: - - name: id - data_type: integer - data_tests: - - unique: - severity: error - - not_null - - name: user_name - data_type: text -""" - -versioned_disabled_contract_schema_yml = """ -version: 2 -models: - - name: table_model - config: - contract: - enforced: False - versions: - - v: 1 - columns: - - name: id - data_type: integer - data_tests: - - unique: - severity: error - - not_null - - name: name - data_type: text -""" - -constraint_schema_yml = """ -version: 2 -models: - - name: view_model - columns: - - name: id - data_tests: - - unique: - severity: error - - not_null - - name: name - - name: table_model - config: - contract: - enforced: True - constraints: - - type: primary_key - columns: [id] - columns: - - name: id - constraints: - - type: not_null - data_type: integer - data_tests: - - unique: - severity: error - - not_null - - name: name - data_type: text -""" - -modified_column_constraint_schema_yml = """ -version: 2 -models: - - name: view_model - columns: - - name: id - data_tests: - - unique: - severity: error - - not_null - - name: name - - name: table_model - config: - contract: - enforced: True - constraints: - - type: primary_key - columns: [id] - columns: - - name: id - data_type: integer - data_tests: - - unique: - severity: error - - not_null - - name: name - data_type: text -""" - -modified_model_constraint_schema_yml = """ -version: 2 -models: - - name: view_model - columns: - - name: id - data_tests: - - unique: - severity: error - - not_null - - name: name - - name: table_model - config: - contract: - enforced: True - columns: - - name: id - constraints: - - type: not_null - data_type: integer - data_tests: - - unique: - severity: error - - not_null - - name: name - data_type: text -""" - -exposures_yml = """ -version: 2 -exposures: - - name: my_exposure - type: application - depends_on: - - ref('view_model') - owner: - email: test@example.com -""" - -macros_sql = """ -{% macro my_macro() %} - {% do log('in a macro' ) %} -{% endmacro %} -""" - -infinite_macros_sql = """ -{# trigger infinite recursion if not handled #} - -{% macro my_infinitely_recursive_macro() %} - {{ return(adapter.dispatch('my_infinitely_recursive_macro')()) }} -{% endmacro %} - -{% macro default__my_infinitely_recursive_macro() %} - {% if unmet_condition %} - {{ my_infinitely_recursive_macro() }} - {% else %} - {{ return('') }} - {% endif %} -{% endmacro %} -""" - -snapshot_sql = """ -{% snapshot my_cool_snapshot %} - - {{ - config( - target_database=database, - target_schema=schema, - unique_key='id', - strategy='check', - check_cols=['id'], - ) - }} - select * from {{ ref('view_model') }} - -{% endsnapshot %} -""" - -model_1_sql = """ -select * from {{ ref('seed') }} -""" - -modified_model_1_sql = """ -select * from {{ ref('seed') }} -order by 1 -""" - -model_2_sql = """ -select id from {{ ref('model_1') }} -""" - -modified_model_2_sql = """ -select * from {{ ref('model_1') }} -order by 1 -""" - - -group_schema_yml = """ -groups: - - name: finance - owner: - email: finance@jaffleshop.com - -models: - - name: model_1 - config: - group: finance - - name: model_2 - config: - group: finance -""" - - -group_modified_schema_yml = """ -groups: - - name: accounting - owner: - email: finance@jaffleshop.com -models: - - name: model_1 - config: - group: accounting - - name: model_2 - config: - group: accounting -""" - -group_modified_fail_schema_yml = """ -groups: - - name: finance - owner: - email: finance@jaffleshop.com -models: - - name: model_1 - config: - group: accounting - - name: model_2 - config: - group: finance -""" diff --git a/tests/functional/defer_state/test_defer_state.py b/tests/functional/defer_state/test_defer_state.py deleted file mode 100644 index 45c1d93c..00000000 --- a/tests/functional/defer_state/test_defer_state.py +++ /dev/null @@ -1,329 +0,0 @@ -from copy import deepcopy -import json -import os -import shutil - -from dbt.contracts.results import RunStatus -from dbt.exceptions import DbtRuntimeError -from dbt.tests.util import rm_file, run_dbt, write_file -import pytest - -from tests.functional.defer_state import fixtures - - -class BaseDeferState: - @pytest.fixture(scope="class") - def models(self): - return { - "table_model.sql": fixtures.table_model_sql, - "view_model.sql": fixtures.view_model_sql, - "ephemeral_model.sql": fixtures.ephemeral_model_sql, - "schema.yml": fixtures.schema_yml, - "exposures.yml": fixtures.exposures_yml, - } - - @pytest.fixture(scope="class") - def macros(self): - return { - "macros.sql": fixtures.macros_sql, - "infinite_macros.sql": fixtures.infinite_macros_sql, - } - - @pytest.fixture(scope="class") - def seeds(self): - return { - "seed.csv": fixtures.seed_csv, - } - - @pytest.fixture(scope="class") - def snapshots(self): - return { - "snapshot.sql": fixtures.snapshot_sql, - } - - @pytest.fixture(scope="class") - def other_schema(self, unique_schema): - return unique_schema + "_other" - - @property - def project_config_update(self): - return { - "seeds": { - "test": { - "quote_columns": False, - } - } - } - - @pytest.fixture(scope="class") - def profiles_config_update(self, dbt_profile_target, unique_schema, other_schema): - outputs = {"default": dbt_profile_target, "otherschema": deepcopy(dbt_profile_target)} - outputs["default"]["schema"] = unique_schema - outputs["otherschema"]["schema"] = other_schema - return {"test": {"outputs": outputs, "target": "default"}} - - def copy_state(self, project_root): - state_path = os.path.join(project_root, "state") - if not os.path.exists(state_path): - os.makedirs(state_path) - shutil.copyfile( - f"{project_root}/target/manifest.json", f"{project_root}/state/manifest.json" - ) - - def run_and_save_state(self, project_root, with_snapshot=False): - results = run_dbt(["seed"]) - assert len(results) == 1 - assert not any(r.node.deferred for r in results) - results = run_dbt(["run"]) - assert len(results) == 2 - assert not any(r.node.deferred for r in results) - results = run_dbt(["test"]) - assert len(results) == 2 - - if with_snapshot: - results = run_dbt(["snapshot"]) - assert len(results) == 1 - assert not any(r.node.deferred for r in results) - - # copy files - self.copy_state(project_root) - - -class TestDeferStateUnsupportedCommands(BaseDeferState): - def test_no_state(self, project): - # no "state" files present, snapshot fails - with pytest.raises(DbtRuntimeError): - run_dbt(["snapshot", "--state", "state", "--defer"]) - - -class TestRunCompileState(BaseDeferState): - def test_run_and_compile_defer(self, project): - self.run_and_save_state(project.project_root) - - # defer test, it succeeds - # Change directory to ensure that state directory is underneath - # project directory. - os.chdir(project.profiles_dir) - results = run_dbt(["compile", "--state", "state", "--defer"]) - assert len(results.results) == 6 - assert results.results[0].node.name == "seed" - - -class TestSnapshotState(BaseDeferState): - def test_snapshot_state_defer(self, project): - self.run_and_save_state(project.project_root) - # snapshot succeeds without --defer - run_dbt(["snapshot"]) - # copy files - self.copy_state(project.project_root) - # defer test, it succeeds - run_dbt(["snapshot", "--state", "state", "--defer"]) - # favor_state test, it succeeds - run_dbt(["snapshot", "--state", "state", "--defer", "--favor-state"]) - - -class TestRunDeferState(BaseDeferState): - def test_run_and_defer(self, project, unique_schema, other_schema): - project.create_test_schema(other_schema) - self.run_and_save_state(project.project_root) - - # test tests first, because run will change things - # no state, wrong schema, failure. - run_dbt(["test", "--target", "otherschema"], expect_pass=False) - - # test generate docs - # no state, wrong schema, empty nodes - catalog = run_dbt(["docs", "generate", "--target", "otherschema"]) - assert not catalog.nodes - - # no state, run also fails - run_dbt(["run", "--target", "otherschema"], expect_pass=False) - - # defer test, it succeeds - results = run_dbt( - ["test", "-m", "view_model+", "--state", "state", "--defer", "--target", "otherschema"] - ) - - # defer docs generate with state, catalog refers schema from the happy times - catalog = run_dbt( - [ - "docs", - "generate", - "-m", - "view_model+", - "--state", - "state", - "--defer", - "--target", - "otherschema", - ] - ) - assert "seed.test.seed" not in catalog.nodes - - # with state it should work though - results = run_dbt( - ["run", "-m", "view_model", "--state", "state", "--defer", "--target", "otherschema"] - ) - assert other_schema not in results[0].node.compiled_code - assert unique_schema in results[0].node.compiled_code - - with open("target/manifest.json") as fp: - data = json.load(fp) - assert data["nodes"]["seed.test.seed"]["deferred"] - - assert len(results) == 1 - - -class TestRunDeferStateChangedModel(BaseDeferState): - def test_run_defer_state_changed_model(self, project): - self.run_and_save_state(project.project_root) - - # change "view_model" - write_file(fixtures.changed_view_model_sql, "models", "view_model.sql") - - # the sql here is just wrong, so it should fail - run_dbt( - ["run", "-m", "view_model", "--state", "state", "--defer", "--target", "otherschema"], - expect_pass=False, - ) - # but this should work since we just use the old happy model - run_dbt( - ["run", "-m", "table_model", "--state", "state", "--defer", "--target", "otherschema"], - expect_pass=True, - ) - - # change "ephemeral_model" - write_file(fixtures.changed_ephemeral_model_sql, "models", "ephemeral_model.sql") - # this should fail because the table model refs a broken ephemeral - # model, which it should see - run_dbt( - ["run", "-m", "table_model", "--state", "state", "--defer", "--target", "otherschema"], - expect_pass=False, - ) - - -class TestRunDeferStateIFFNotExists(BaseDeferState): - def test_run_defer_iff_not_exists(self, project, unique_schema, other_schema): - project.create_test_schema(other_schema) - self.run_and_save_state(project.project_root) - - results = run_dbt(["seed", "--target", "otherschema"]) - assert len(results) == 1 - results = run_dbt(["run", "--state", "state", "--defer", "--target", "otherschema"]) - assert len(results) == 2 - - # because the seed now exists in our "other" schema, we should prefer it over the one - # available from state - assert other_schema in results[0].node.compiled_code - - # this time with --favor-state: even though the seed now exists in our "other" schema, - # we should still favor the one available from state - results = run_dbt( - ["run", "--state", "state", "--defer", "--favor-state", "--target", "otherschema"] - ) - assert len(results) == 2 - assert other_schema not in results[0].node.compiled_code - - -class TestDeferStateDeletedUpstream(BaseDeferState): - def test_run_defer_deleted_upstream(self, project, unique_schema, other_schema): - project.create_test_schema(other_schema) - self.run_and_save_state(project.project_root) - - # remove "ephemeral_model" + change "table_model" - rm_file("models", "ephemeral_model.sql") - write_file(fixtures.changed_table_model_sql, "models", "table_model.sql") - - # ephemeral_model is now gone. previously this caused a - # keyerror (dbt#2875), now it should pass - run_dbt( - ["run", "-m", "view_model", "--state", "state", "--defer", "--target", "otherschema"], - expect_pass=True, - ) - - # despite deferral, we should use models just created in our schema - results = run_dbt(["test", "--state", "state", "--defer", "--target", "otherschema"]) - assert other_schema in results[0].node.compiled_code - - # this time with --favor-state: prefer the models in the "other" schema, even though they exist in ours - run_dbt( - [ - "run", - "-m", - "view_model", - "--state", - "state", - "--defer", - "--favor-state", - "--target", - "otherschema", - ], - expect_pass=True, - ) - results = run_dbt(["test", "--state", "state", "--defer", "--favor-state"]) - assert other_schema not in results[0].node.compiled_code - - -class TestDeferStateFlag(BaseDeferState): - def test_defer_state_flag(self, project, unique_schema, other_schema): - project.create_test_schema(other_schema) - - # test that state deferral works correctly - run_dbt(["compile", "--target-path", "target_compile"]) - write_file(fixtures.view_model_now_table_sql, "models", "table_model.sql") - - results = run_dbt(["ls", "--select", "state:modified", "--state", "target_compile"]) - assert results == ["test.table_model"] - - run_dbt(["seed", "--target", "otherschema", "--target-path", "target_otherschema"]) - - # this will fail because we haven't loaded the seed in the default schema - run_dbt( - [ - "run", - "--select", - "state:modified", - "--defer", - "--state", - "target_compile", - "--favor-state", - ], - expect_pass=False, - ) - - # this will fail because we haven't passed in --state - with pytest.raises( - DbtRuntimeError, match="Got a state selector method, but no comparison manifest" - ): - run_dbt( - [ - "run", - "--select", - "state:modified", - "--defer", - "--defer-state", - "target_otherschema", - "--favor-state", - ], - expect_pass=False, - ) - - # this will succeed because we've loaded the seed in other schema and are successfully deferring to it instead - results = run_dbt( - [ - "run", - "--select", - "state:modified", - "--defer", - "--state", - "target_compile", - "--defer-state", - "target_otherschema", - "--favor-state", - ] - ) - - assert len(results.results) == 1 - assert results.results[0].status == RunStatus.Success - assert results.results[0].node.name == "table_model" - assert results.results[0].adapter_response["rows_affected"] == 2 diff --git a/tests/functional/defer_state/test_group_updates.py b/tests/functional/defer_state/test_group_updates.py deleted file mode 100644 index 5f3e8006..00000000 --- a/tests/functional/defer_state/test_group_updates.py +++ /dev/null @@ -1,108 +0,0 @@ -import os - -from dbt.exceptions import ParsingError -from dbt.tests.util import copy_file, run_dbt, write_file -import pytest - -from tests.functional.defer_state import fixtures - - -class GroupSetup: - @pytest.fixture(scope="class") - def models(self): - return { - "model_1.sql": fixtures.model_1_sql, - "model_2.sql": fixtures.model_2_sql, - "schema.yml": fixtures.group_schema_yml, - } - - @pytest.fixture(scope="class") - def seeds(self): - return {"seed.csv": fixtures.seed_csv} - - def group_setup(self): - # save initial state - run_dbt(["seed"]) - results = run_dbt(["compile"]) - - # add sanity checks for first result - assert len(results) == 3 - seed_result = results[0].node - assert seed_result.unique_id == "seed.test.seed" - model_1_result = results[1].node - assert model_1_result.unique_id == "model.test.model_1" - assert model_1_result.group == "finance" - model_2_result = results[2].node - assert model_2_result.unique_id == "model.test.model_2" - assert model_2_result.group == "finance" - - -class TestFullyModifiedGroups(GroupSetup): - def test_changed_groups(self, project): - self.group_setup() - - # copy manifest.json to "state" directory - os.makedirs("state") - target_path = os.path.join(project.project_root, "target") - copy_file(target_path, "manifest.json", project.project_root, ["state", "manifest.json"]) - - # update group name, modify model so it gets picked up - write_file(fixtures.modified_model_1_sql, "models", "model_1.sql") - write_file(fixtures.modified_model_2_sql, "models", "model_2.sql") - write_file(fixtures.group_modified_schema_yml, "models", "schema.yml") - - # this test is flaky if you don't clean first before the build - run_dbt(["clean"]) - # only thing in results should be model_1 - results = run_dbt(["build", "-s", "state:modified", "--defer", "--state", "./state"]) - - assert len(results) == 2 - model_1_result = results[0].node - assert model_1_result.unique_id == "model.test.model_1" - assert model_1_result.group == "accounting" # new group name! - model_2_result = results[1].node - assert model_2_result.unique_id == "model.test.model_2" - assert model_2_result.group == "accounting" # new group name! - - -class TestPartiallyModifiedGroups(GroupSetup): - def test_changed_groups(self, project): - self.group_setup() - - # copy manifest.json to "state" directory - os.makedirs("state") - target_path = os.path.join(project.project_root, "target") - copy_file(target_path, "manifest.json", project.project_root, ["state", "manifest.json"]) - - # update group name, modify model so it gets picked up - write_file(fixtures.modified_model_1_sql, "models", "model_1.sql") - write_file(fixtures.group_modified_schema_yml, "models", "schema.yml") - - # this test is flaky if you don't clean first before the build - run_dbt(["clean"]) - # only thing in results should be model_1 - results = run_dbt(["build", "-s", "state:modified", "--defer", "--state", "./state"]) - - assert len(results) == 1 - model_1_result = results[0].node - assert model_1_result.unique_id == "model.test.model_1" - assert model_1_result.group == "accounting" # new group name! - - -class TestBadGroups(GroupSetup): - def test_changed_groups(self, project): - self.group_setup() - - # copy manifest.json to "state" directory - os.makedirs("state") - target_path = os.path.join(project.project_root, "target") - copy_file(target_path, "manifest.json", project.project_root, ["state", "manifest.json"]) - - # update group with invalid name, modify model so it gets picked up - write_file(fixtures.modified_model_1_sql, "models", "model_1.sql") - write_file(fixtures.group_modified_fail_schema_yml, "models", "schema.yml") - - # this test is flaky if you don't clean first before the build - run_dbt(["clean"]) - with pytest.raises(ParsingError, match="Invalid group 'accounting'"): - run_dbt(["build", "-s", "state:modified", "--defer", "--state", "./state"]) diff --git a/tests/functional/defer_state/test_modified_state.py b/tests/functional/defer_state/test_modified_state.py deleted file mode 100644 index e108fe9f..00000000 --- a/tests/functional/defer_state/test_modified_state.py +++ /dev/null @@ -1,964 +0,0 @@ -import os -import random -import shutil -import string - -from dbt.exceptions import ContractBreakingChangeError -from dbt.tests.util import get_manifest, update_config_file, write_file -from dbt_common.exceptions import CompilationError -import pytest - -from tests.functional.defer_state import fixtures -from tests.functional.utils import run_dbt, run_dbt_and_capture - - -class BaseModifiedState: - @pytest.fixture(scope="class") - def models(self): - return { - "table_model.sql": fixtures.table_model_sql, - "view_model.sql": fixtures.view_model_sql, - "ephemeral_model.sql": fixtures.ephemeral_model_sql, - "schema.yml": fixtures.schema_yml, - "exposures.yml": fixtures.exposures_yml, - } - - @pytest.fixture(scope="class") - def macros(self): - return { - "macros.sql": fixtures.macros_sql, - "infinite_macros.sql": fixtures.infinite_macros_sql, - } - - @pytest.fixture(scope="class") - def seeds(self): - return {"seed.csv": fixtures.seed_csv} - - @property - def project_config_update(self): - return { - "seeds": { - "test": { - "quote_columns": False, - } - } - } - - def copy_state(self): - if not os.path.exists("state"): - os.makedirs("state") - shutil.copyfile("target/manifest.json", "state/manifest.json") - - def run_and_save_state(self): - run_dbt(["seed"]) - run_dbt(["run"]) - self.copy_state() - - -class TestChangedSeedContents(BaseModifiedState): - def test_changed_seed_contents_state(self, project): - self.run_and_save_state() - results = run_dbt( - ["ls", "--resource-type", "seed", "--select", "state:modified", "--state", "./state"], - expect_pass=True, - ) - assert len(results) == 0 - - results = run_dbt( - [ - "ls", - "--resource-type", - "seed", - "--exclude", - "state:unmodified", - "--state", - "./state", - ], - expect_pass=True, - ) - assert len(results) == 0 - - results = run_dbt( - [ - "ls", - "--resource-type", - "seed", - "--select", - "state:unmodified", - "--state", - "./state", - ], - expect_pass=True, - ) - assert len(results) == 1 - - # add a new row to the seed - changed_seed_contents = fixtures.seed_csv + "\n" + "3,carl" - write_file(changed_seed_contents, "seeds", "seed.csv") - - results = run_dbt( - ["ls", "--resource-type", "seed", "--select", "state:modified", "--state", "./state"] - ) - assert len(results) == 1 - assert results[0] == "test.seed" - - results = run_dbt( - [ - "ls", - "--resource-type", - "seed", - "--exclude", - "state:unmodified", - "--state", - "./state", - ] - ) - assert len(results) == 1 - assert results[0] == "test.seed" - - results = run_dbt( - ["ls", "--resource-type", "seed", "--select", "state:unmodified", "--state", "./state"] - ) - assert len(results) == 0 - - results = run_dbt(["ls", "--select", "state:modified", "--state", "./state"]) - assert len(results) == 1 - assert results[0] == "test.seed" - - results = run_dbt(["ls", "--exclude", "state:unmodified", "--state", "./state"]) - assert len(results) == 1 - assert results[0] == "test.seed" - - results = run_dbt(["ls", "--select", "state:unmodified", "--state", "./state"]) - assert len(results) == 6 - - results = run_dbt(["ls", "--select", "state:modified+", "--state", "./state"]) - assert len(results) == 7 - assert set(results) == { - "test.seed", - "test.table_model", - "test.view_model", - "test.ephemeral_model", - "test.not_null_view_model_id", - "test.unique_view_model_id", - "exposure:test.my_exposure", - } - - results = run_dbt(["ls", "--select", "state:unmodified+", "--state", "./state"]) - assert len(results) == 6 - assert set(results) == { - "test.table_model", - "test.view_model", - "test.ephemeral_model", - "test.not_null_view_model_id", - "test.unique_view_model_id", - "exposure:test.my_exposure", - } - - shutil.rmtree("./state") - self.copy_state() - - # make a very big seed - # assume each line is ~2 bytes + len(name) - target_size = 1 * 1024 * 1024 - line_size = 64 - num_lines = target_size // line_size - maxlines = num_lines + 4 - seed_lines = [fixtures.seed_csv] - for idx in range(4, maxlines): - value = "".join(random.choices(string.ascii_letters, k=62)) - seed_lines.append(f"{idx},{value}") - seed_contents = "\n".join(seed_lines) - write_file(seed_contents, "seeds", "seed.csv") - - # now if we run again, we should get a warning - results = run_dbt( - ["ls", "--resource-type", "seed", "--select", "state:modified", "--state", "./state"] - ) - assert len(results) == 1 - assert results[0] == "test.seed" - - with pytest.raises(CompilationError) as exc: - run_dbt( - [ - "--warn-error", - "ls", - "--resource-type", - "seed", - "--select", - "state:modified", - "--state", - "./state", - ] - ) - assert ">1MB" in str(exc.value) - - # now check if unmodified returns none - results = run_dbt( - ["ls", "--resource-type", "seed", "--select", "state:unmodified", "--state", "./state"] - ) - assert len(results) == 0 - - shutil.rmtree("./state") - self.copy_state() - - # once it"s in path mode, we don"t mark it as modified if it changes - write_file(seed_contents + "\n1,test", "seeds", "seed.csv") - - results = run_dbt( - ["ls", "--resource-type", "seed", "--select", "state:modified", "--state", "./state"], - expect_pass=True, - ) - assert len(results) == 0 - - results = run_dbt( - [ - "ls", - "--resource-type", - "seed", - "--exclude", - "state:unmodified", - "--state", - "./state", - ], - expect_pass=True, - ) - assert len(results) == 0 - - results = run_dbt( - [ - "ls", - "--resource-type", - "seed", - "--select", - "state:unmodified", - "--state", - "./state", - ], - expect_pass=True, - ) - assert len(results) == 1 - - -class TestChangedSeedConfig(BaseModifiedState): - def test_changed_seed_config(self, project): - self.run_and_save_state() - results = run_dbt( - ["ls", "--resource-type", "seed", "--select", "state:modified", "--state", "./state"], - expect_pass=True, - ) - assert len(results) == 0 - - results = run_dbt( - [ - "ls", - "--resource-type", - "seed", - "--exclude", - "state:unmodified", - "--state", - "./state", - ], - expect_pass=True, - ) - assert len(results) == 0 - - results = run_dbt( - [ - "ls", - "--resource-type", - "seed", - "--select", - "state:unmodified", - "--state", - "./state", - ], - expect_pass=True, - ) - assert len(results) == 1 - - update_config_file({"seeds": {"test": {"quote_columns": False}}}, "dbt_project.yml") - - # quoting change -> seed changed - results = run_dbt( - ["ls", "--resource-type", "seed", "--select", "state:modified", "--state", "./state"] - ) - assert len(results) == 1 - assert results[0] == "test.seed" - - results = run_dbt( - [ - "ls", - "--resource-type", - "seed", - "--exclude", - "state:unmodified", - "--state", - "./state", - ] - ) - assert len(results) == 1 - assert results[0] == "test.seed" - - results = run_dbt( - ["ls", "--resource-type", "seed", "--select", "state:unmodified", "--state", "./state"] - ) - assert len(results) == 0 - - -class TestUnrenderedConfigSame(BaseModifiedState): - def test_unrendered_config_same(self, project): - self.run_and_save_state() - results = run_dbt( - ["ls", "--resource-type", "model", "--select", "state:modified", "--state", "./state"], - expect_pass=True, - ) - assert len(results) == 0 - - results = run_dbt( - [ - "ls", - "--resource-type", - "model", - "--exclude", - "state:unmodified", - "--state", - "./state", - ], - expect_pass=True, - ) - assert len(results) == 0 - - results = run_dbt( - [ - "ls", - "--resource-type", - "model", - "--select", - "state:unmodified", - "--state", - "./state", - ], - expect_pass=True, - ) - assert len(results) == 3 - - # although this is the default value, dbt will recognize it as a change - # for previously-unconfigured models, because it"s been explicitly set - update_config_file({"models": {"test": {"materialized": "view"}}}, "dbt_project.yml") - results = run_dbt( - ["ls", "--resource-type", "model", "--select", "state:modified", "--state", "./state"] - ) - assert len(results) == 1 - assert results[0] == "test.view_model" - - # converse of above statement - results = run_dbt( - [ - "ls", - "--resource-type", - "model", - "--exclude", - "state:unmodified", - "--state", - "./state", - ] - ) - assert len(results) == 1 - assert results[0] == "test.view_model" - - results = run_dbt( - [ - "ls", - "--resource-type", - "model", - "--select", - "state:unmodified", - "--state", - "./state", - ] - ) - assert len(results) == 2 - assert set(results) == { - "test.table_model", - "test.ephemeral_model", - } - - -class TestChangedModelContents(BaseModifiedState): - def test_changed_model_contents(self, project): - self.run_and_save_state() - results = run_dbt(["run", "--models", "state:modified", "--state", "./state"]) - assert len(results) == 0 - - table_model_update = """ - {{ config(materialized="table") }} - - select * from {{ ref("seed") }} - """ - - write_file(table_model_update, "models", "table_model.sql") - - results = run_dbt(["run", "--models", "state:modified", "--state", "./state"]) - assert len(results) == 1 - assert results[0].node.name == "table_model" - - results = run_dbt(["run", "--exclude", "state:unmodified", "--state", "./state"]) - assert len(results) == 1 - assert results[0].node.name == "table_model" - - -class TestNewMacro(BaseModifiedState): - def test_new_macro(self, project): - self.run_and_save_state() - - new_macro = """ - {% macro my_other_macro() %} - {% endmacro %} - """ - - # add a new macro to a new file - write_file(new_macro, "macros", "second_macro.sql") - - results = run_dbt(["run", "--models", "state:modified", "--state", "./state"]) - assert len(results) == 0 - - os.remove("macros/second_macro.sql") - # add a new macro to the existing file - with open("macros/macros.sql", "a") as fp: - fp.write(new_macro) - - results = run_dbt(["run", "--models", "state:modified", "--state", "./state"]) - assert len(results) == 0 - - results = run_dbt(["run", "--exclude", "state:unmodified", "--state", "./state"]) - assert len(results) == 0 - - -class TestChangedMacroContents(BaseModifiedState): - def test_changed_macro_contents(self, project): - self.run_and_save_state() - - # modify an existing macro - updated_macro = """ - {% macro my_macro() %} - {% do log("in a macro", info=True) %} - {% endmacro %} - """ - write_file(updated_macro, "macros", "macros.sql") - - # table_model calls this macro - results = run_dbt(["run", "--models", "state:modified", "--state", "./state"]) - assert len(results) == 1 - - results = run_dbt(["run", "--exclude", "state:unmodified", "--state", "./state"]) - assert len(results) == 1 - - -class TestChangedExposure(BaseModifiedState): - def test_changed_exposure(self, project): - self.run_and_save_state() - - # add an "owner.name" to existing exposure - updated_exposure = fixtures.exposures_yml + "\n name: John Doe\n" - write_file(updated_exposure, "models", "exposures.yml") - - results = run_dbt(["run", "--models", "+state:modified", "--state", "./state"]) - assert len(results) == 1 - assert results[0].node.name == "view_model" - - results = run_dbt(["run", "--exclude", "state:unmodified", "--state", "./state"]) - assert len(results) == 0 - - -class TestChangedContractUnversioned(BaseModifiedState): - MODEL_UNIQUE_ID = "model.test.table_model" - CONTRACT_SCHEMA_YML = fixtures.contract_schema_yml - MODIFIED_SCHEMA_YML = fixtures.modified_contract_schema_yml - DISABLED_SCHEMA_YML = fixtures.disabled_contract_schema_yml - NO_CONTRACT_SCHEMA_YML = fixtures.no_contract_schema_yml - - def test_changed_contract(self, project): - self.run_and_save_state() - - # update contract for table_model - write_file(self.CONTRACT_SCHEMA_YML, "models", "schema.yml") - - # This will find the table_model node modified both through a config change - # and by a non-breaking change to contract: true - results = run_dbt(["run", "--models", "state:modified", "--state", "./state"]) - assert len(results) == 1 - assert results[0].node.name == "table_model" - - results = run_dbt(["run", "--exclude", "state:unmodified", "--state", "./state"]) - assert len(results) == 1 - assert results[0].node.name == "table_model" - - manifest = get_manifest(project.project_root) - model_unique_id = self.MODEL_UNIQUE_ID - model = manifest.nodes[model_unique_id] - expected_unrendered_config = {"contract": {"enforced": True}, "materialized": "table"} - assert model.unrendered_config == expected_unrendered_config - - # Run it again with "state:modified:contract", still finds modified due to contract: true - results = run_dbt(["run", "--models", "state:modified.contract", "--state", "./state"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - model = manifest.nodes[model_unique_id] - first_contract_checksum = model.contract.checksum - assert first_contract_checksum - # save a new state - self.copy_state() - - # This should raise because a column name has changed - write_file(self.MODIFIED_SCHEMA_YML, "models", "schema.yml") - results = run_dbt(["run"], expect_pass=False) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - model = manifest.nodes[model_unique_id] - second_contract_checksum = model.contract.checksum - # double check different contract_checksums - assert first_contract_checksum != second_contract_checksum - - _, logs = run_dbt_and_capture( - ["run", "--models", "state:modified.contract", "--state", "./state"], expect_pass=False - ) - expected_error = "This model has an enforced contract that failed." - expected_warning = "While comparing to previous project state, dbt detected a breaking change to an unversioned model" - expected_change = "Please ensure the name, data_type, and number of columns in your contract match the columns in your model's definition" - assert expected_error in logs - assert expected_warning in logs - assert expected_change in logs - - # Go back to schema file without contract. Should throw a warning. - write_file(self.NO_CONTRACT_SCHEMA_YML, "models", "schema.yml") - _, logs = run_dbt_and_capture( - ["run", "--models", "state:modified.contract", "--state", "./state"] - ) - expected_warning = "While comparing to previous project state, dbt detected a breaking change to an unversioned model" - expected_change = "Contract enforcement was removed" - - # Now disable the contract. Should throw a warning - force warning into an error. - write_file(self.DISABLED_SCHEMA_YML, "models", "schema.yml") - with pytest.raises(CompilationError): - _, logs = run_dbt_and_capture( - [ - "--warn-error", - "run", - "--models", - "state:modified.contract", - "--state", - "./state", - ] - ) - expected_warning = "While comparing to previous project state, dbt detected a breaking change to an unversioned model" - expected_change = "Contract enforcement was removed" - - -class TestChangedContractVersioned(BaseModifiedState): - MODEL_UNIQUE_ID = "model.test.table_model.v1" - CONTRACT_SCHEMA_YML = fixtures.versioned_contract_schema_yml - MODIFIED_SCHEMA_YML = fixtures.versioned_modified_contract_schema_yml - DISABLED_SCHEMA_YML = fixtures.versioned_disabled_contract_schema_yml - NO_CONTRACT_SCHEMA_YML = fixtures.versioned_no_contract_schema_yml - - def test_changed_contract_versioned(self, project): - self.run_and_save_state() - - # update contract for table_model - write_file(self.CONTRACT_SCHEMA_YML, "models", "schema.yml") - - # This will find the table_model node modified both through a config change - # and by a non-breaking change to contract: true - results = run_dbt(["run", "--models", "state:modified", "--state", "./state"]) - assert len(results) == 1 - assert results[0].node.name == "table_model" - - results = run_dbt(["run", "--exclude", "state:unmodified", "--state", "./state"]) - assert len(results) == 1 - assert results[0].node.name == "table_model" - - manifest = get_manifest(project.project_root) - model_unique_id = self.MODEL_UNIQUE_ID - model = manifest.nodes[model_unique_id] - expected_unrendered_config = {"contract": {"enforced": True}, "materialized": "table"} - assert model.unrendered_config == expected_unrendered_config - - # Run it again with "state:modified:contract", still finds modified due to contract: true - results = run_dbt(["run", "--models", "state:modified.contract", "--state", "./state"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - model = manifest.nodes[model_unique_id] - first_contract_checksum = model.contract.checksum - assert first_contract_checksum - # save a new state - self.copy_state() - - # This should raise because a column name has changed - write_file(self.MODIFIED_SCHEMA_YML, "models", "schema.yml") - results = run_dbt(["run"], expect_pass=False) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - model = manifest.nodes[model_unique_id] - second_contract_checksum = model.contract.checksum - # double check different contract_checksums - assert first_contract_checksum != second_contract_checksum - with pytest.raises(ContractBreakingChangeError): - results = run_dbt(["run", "--models", "state:modified.contract", "--state", "./state"]) - - # Go back to schema file without contract. Should raise an error. - write_file(self.NO_CONTRACT_SCHEMA_YML, "models", "schema.yml") - with pytest.raises(ContractBreakingChangeError): - results = run_dbt(["run", "--models", "state:modified.contract", "--state", "./state"]) - - # Now disable the contract. Should raise an error. - write_file(self.DISABLED_SCHEMA_YML, "models", "schema.yml") - with pytest.raises(ContractBreakingChangeError): - results = run_dbt(["run", "--models", "state:modified.contract", "--state", "./state"]) - - -class TestChangedConstraintUnversioned(BaseModifiedState): - def test_changed_constraint(self, project): - self.run_and_save_state() - - # update constraint for table_model - write_file(fixtures.constraint_schema_yml, "models", "schema.yml") - - # This will find the table_model node modified both through adding constraint - # and by a non-breaking change to contract: true - results = run_dbt(["run", "--models", "state:modified", "--state", "./state"]) - assert len(results) == 1 - assert results[0].node.name == "table_model" - - results = run_dbt(["run", "--exclude", "state:unmodified", "--state", "./state"]) - assert len(results) == 1 - assert results[0].node.name == "table_model" - - manifest = get_manifest(project.project_root) - model_unique_id = "model.test.table_model" - model = manifest.nodes[model_unique_id] - expected_unrendered_config = {"contract": {"enforced": True}, "materialized": "table"} - assert model.unrendered_config == expected_unrendered_config - - # Run it again with "state:modified:contract", still finds modified due to contract: true - results = run_dbt(["run", "--models", "state:modified.contract", "--state", "./state"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - model = manifest.nodes[model_unique_id] - first_contract_checksum = model.contract.checksum - assert first_contract_checksum - # save a new state - self.copy_state() - - # This should raise because a column level constraint was removed - write_file(fixtures.modified_column_constraint_schema_yml, "models", "schema.yml") - # we don't have a way to know this failed unless we have a previous state to refer to, so the run succeeds - results = run_dbt(["run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - model = manifest.nodes[model_unique_id] - second_contract_checksum = model.contract.checksum - # double check different contract_checksums - assert first_contract_checksum != second_contract_checksum - # since the models are unversioned, they raise a warning but not an error - _, logs = run_dbt_and_capture( - ["run", "--models", "state:modified.contract", "--state", "./state"] - ) - expected_warning = "While comparing to previous project state, dbt detected a breaking change to an unversioned model" - expected_change = "Enforced column level constraints were removed" - assert expected_warning in logs - assert expected_change in logs - - # This should raise because a model level constraint was removed (primary_key on id) - write_file(fixtures.modified_model_constraint_schema_yml, "models", "schema.yml") - # we don't have a way to know this failed unless we have a previous state to refer to, so the run succeeds - results = run_dbt(["run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - model = manifest.nodes[model_unique_id] - second_contract_checksum = model.contract.checksum - # double check different contract_checksums - assert first_contract_checksum != second_contract_checksum - _, logs = run_dbt_and_capture( - ["run", "--models", "state:modified.contract", "--state", "./state"] - ) - expected_warning = "While comparing to previous project state, dbt detected a breaking change to an unversioned model" - expected_change = "Enforced model level constraints were removed" - assert expected_warning in logs - assert expected_change in logs - - -class TestChangedMaterializationConstraint(BaseModifiedState): - def test_changed_materialization(self, project): - self.run_and_save_state() - - # update constraint for table_model - write_file(fixtures.constraint_schema_yml, "models", "schema.yml") - - # This will find the table_model node modified both through adding constraint - # and by a non-breaking change to contract: true - results = run_dbt(["run", "--models", "state:modified", "--state", "./state"]) - assert len(results) == 1 - assert results[0].node.name == "table_model" - - results = run_dbt(["run", "--exclude", "state:unmodified", "--state", "./state"]) - assert len(results) == 1 - assert results[0].node.name == "table_model" - - manifest = get_manifest(project.project_root) - model_unique_id = "model.test.table_model" - model = manifest.nodes[model_unique_id] - expected_unrendered_config = {"contract": {"enforced": True}, "materialized": "table"} - assert model.unrendered_config == expected_unrendered_config - - # Run it again with "state:modified:contract", still finds modified due to contract: true - results = run_dbt(["run", "--models", "state:modified.contract", "--state", "./state"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - model = manifest.nodes[model_unique_id] - first_contract_checksum = model.contract.checksum - assert first_contract_checksum - # save a new state - self.copy_state() - - # This should raise because materialization changed from table to view - write_file(fixtures.table_model_now_view_sql, "models", "table_model.sql") - # we don't have a way to know this failed unless we have a previous state to refer to, so the run succeeds - results = run_dbt(["run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - model = manifest.nodes[model_unique_id] - second_contract_checksum = model.contract.checksum - # double check different contract_checksums - assert first_contract_checksum != second_contract_checksum - _, logs = run_dbt_and_capture( - ["run", "--models", "state:modified.contract", "--state", "./state"] - ) - expected_warning = "While comparing to previous project state, dbt detected a breaking change to an unversioned model" - expected_change = "Materialization changed with enforced constraints" - assert expected_warning in logs - assert expected_change in logs - - # This should not raise because materialization changed from table to incremental, both enforce constraints - write_file(fixtures.table_model_now_incremental_sql, "models", "table_model.sql") - # we don't have a way to know this failed unless we have a previous state to refer to, so the run succeeds - results = run_dbt(["run"]) - assert len(results) == 2 - - # This should pass because materialization changed from view to table which is the same as just adding new constraint, not breaking - write_file(fixtures.view_model_now_table_sql, "models", "view_model.sql") - write_file(fixtures.table_model_sql, "models", "table_model.sql") - results = run_dbt(["run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - model = manifest.nodes[model_unique_id] - second_contract_checksum = model.contract.checksum - # contract_checksums should be equal because we only save constraint related changes if the materialization is table/incremental - assert first_contract_checksum == second_contract_checksum - run_dbt(["run", "--models", "state:modified.contract", "--state", "./state"]) - assert len(results) == 2 - - -my_model_sql = """ -select 1 as id -""" - -modified_my_model_sql = """ --- a comment -select 1 as id -""" - -modified_my_model_non_breaking_sql = """ --- a comment -select 1 as id, 'blue' as color -""" - -my_model_yml = """ -models: - - name: my_model - latest_version: 1 - config: - contract: - enforced: true - columns: - - name: id - data_type: int - versions: - - v: 1 -""" - -modified_my_model_yml = """ -models: - - name: my_model - latest_version: 1 - config: - contract: - enforced: true - columns: - - name: id - data_type: text - versions: - - v: 1 -""" - -modified_my_model_non_breaking_yml = """ -models: - - name: my_model - latest_version: 1 - config: - contract: - enforced: true - columns: - - name: id - data_type: int - - name: color - data_type: text - versions: - - v: 1 -""" - - -class TestModifiedBodyAndContract: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_model_sql, - "my_model.yml": my_model_yml, - } - - def copy_state(self): - if not os.path.exists("state"): - os.makedirs("state") - shutil.copyfile("target/manifest.json", "state/manifest.json") - - def test_modified_body_and_contract(self, project): - results = run_dbt(["run"]) - assert len(results) == 1 - self.copy_state() - - # Change both body and contract in a *breaking* way (= changing data_type of existing column) - write_file(modified_my_model_yml, "models", "my_model.yml") - write_file(modified_my_model_sql, "models", "my_model.sql") - - # Should raise even without specifying state:modified.contract - with pytest.raises(ContractBreakingChangeError): - results = run_dbt(["run", "-s", "state:modified", "--state", "./state"]) - - with pytest.raises(ContractBreakingChangeError): - results = run_dbt(["run", "--exclude", "state:unmodified", "--state", "./state"]) - - # Change both body and contract in a *non-breaking* way (= adding a new column) - write_file(modified_my_model_non_breaking_yml, "models", "my_model.yml") - write_file(modified_my_model_non_breaking_sql, "models", "my_model.sql") - - # Should pass - run_dbt(["run", "-s", "state:modified", "--state", "./state"]) - - # The model's contract has changed, even if non-breaking, so it should be selected by 'state:modified.contract' - results = run_dbt(["list", "-s", "state:modified.contract", "--state", "./state"]) - assert results == ["test.my_model.v1"] - - -modified_table_model_access_yml = """ -version: 2 -models: - - name: table_model - access: public -""" - - -class TestModifiedAccess(BaseModifiedState): - def test_changed_access(self, project): - self.run_and_save_state() - - # No access change - assert not run_dbt(["list", "-s", "state:modified", "--state", "./state"]) - - # Modify access (protected -> public) - write_file(modified_table_model_access_yml, "models", "schema.yml") - assert run_dbt(["list", "-s", "state:modified", "--state", "./state"]) - - results = run_dbt(["list", "-s", "state:modified", "--state", "./state"]) - assert results == ["test.table_model"] - - -modified_table_model_access_yml = """ -version: 2 -models: - - name: table_model - deprecation_date: 2020-01-01 -""" - - -class TestModifiedDeprecationDate(BaseModifiedState): - def test_changed_access(self, project): - self.run_and_save_state() - - # No access change - assert not run_dbt(["list", "-s", "state:modified", "--state", "./state"]) - - # Modify deprecation_date (None -> 2020-01-01) - write_file(modified_table_model_access_yml, "models", "schema.yml") - assert run_dbt(["list", "-s", "state:modified", "--state", "./state"]) - - results = run_dbt(["list", "-s", "state:modified", "--state", "./state"]) - assert results == ["test.table_model"] - - -modified_table_model_version_yml = """ -version: 2 -models: - - name: table_model - versions: - - v: 1 - defined_in: table_model -""" - - -class TestModifiedVersion(BaseModifiedState): - def test_changed_access(self, project): - self.run_and_save_state() - - # Change version (null -> v1) - write_file(modified_table_model_version_yml, "models", "schema.yml") - - results = run_dbt(["list", "-s", "state:modified", "--state", "./state"]) - assert results == ["test.table_model.v1"] - - -table_model_latest_version_yml = """ -version: 2 -models: - - name: table_model - latest_version: 1 - versions: - - v: 1 - defined_in: table_model -""" - - -modified_table_model_latest_version_yml = """ -version: 2 -models: - - name: table_model - latest_version: 2 - versions: - - v: 1 - defined_in: table_model - - v: 2 -""" - - -class TestModifiedLatestVersion(BaseModifiedState): - def test_changed_access(self, project): - # Setup initial latest_version: 1 - write_file(table_model_latest_version_yml, "models", "schema.yml") - - self.run_and_save_state() - - # Bump latest version - write_file(fixtures.table_model_sql, "models", "table_model_v2.sql") - write_file(modified_table_model_latest_version_yml, "models", "schema.yml") - - results = run_dbt(["list", "-s", "state:modified", "--state", "./state"]) - assert results == ["test.table_model.v1", "test.table_model.v2"] diff --git a/tests/functional/defer_state/test_run_results_state.py b/tests/functional/defer_state/test_run_results_state.py deleted file mode 100644 index ae5941c7..00000000 --- a/tests/functional/defer_state/test_run_results_state.py +++ /dev/null @@ -1,481 +0,0 @@ -import os -import shutil - -from dbt.tests.util import run_dbt, write_file -import pytest - -from tests.functional.defer_state import fixtures - - -class BaseRunResultsState: - @pytest.fixture(scope="class") - def models(self): - return { - "table_model.sql": fixtures.table_model_sql, - "view_model.sql": fixtures.view_model_sql, - "ephemeral_model.sql": fixtures.ephemeral_model_sql, - "schema.yml": fixtures.schema_yml, - "exposures.yml": fixtures.exposures_yml, - } - - @pytest.fixture(scope="class") - def macros(self): - return { - "macros.sql": fixtures.macros_sql, - "infinite_macros.sql": fixtures.infinite_macros_sql, - } - - @pytest.fixture(scope="class") - def seeds(self): - return {"seed.csv": fixtures.seed_csv} - - @property - def project_config_update(self): - return { - "seeds": { - "test": { - "quote_columns": False, - } - } - } - - def clear_state(self): - shutil.rmtree("./state") - - def copy_state(self): - if not os.path.exists("state"): - os.makedirs("state") - shutil.copyfile("target/manifest.json", "state/manifest.json") - shutil.copyfile("target/run_results.json", "state/run_results.json") - - def run_and_save_state(self): - run_dbt(["build"]) - self.copy_state() - - def rebuild_run_dbt(self, expect_pass=True): - self.clear_state() - run_dbt(["build"], expect_pass=expect_pass) - self.copy_state() - - def update_view_model_bad_sql(self): - # update view model to generate a failure case - not_unique_sql = "select * from forced_error" - write_file(not_unique_sql, "models", "view_model.sql") - - def update_view_model_failing_tests(self, with_dupes=True, with_nulls=False): - # test failure on build tests - # fail the unique test - select_1 = "select 1 as id" - select_stmts = [select_1] - if with_dupes: - select_stmts.append(select_1) - if with_nulls: - select_stmts.append("select null as id") - failing_tests_sql = " union all ".join(select_stmts) - write_file(failing_tests_sql, "models", "view_model.sql") - - def update_unique_test_severity_warn(self): - # change the unique test severity from error to warn and reuse the same view_model.sql changes above - new_config = fixtures.schema_yml.replace("error", "warn") - write_file(new_config, "models", "schema.yml") - - -class TestSeedRunResultsState(BaseRunResultsState): - def test_seed_run_results_state(self, project): - self.run_and_save_state() - self.clear_state() - run_dbt(["seed"]) - self.copy_state() - results = run_dbt( - ["ls", "--resource-type", "seed", "--select", "result:success", "--state", "./state"], - expect_pass=True, - ) - assert len(results) == 1 - assert results[0] == "test.seed" - - results = run_dbt(["ls", "--select", "result:success", "--state", "./state"]) - assert len(results) == 1 - assert results[0] == "test.seed" - - results = run_dbt(["ls", "--select", "result:success+", "--state", "./state"]) - assert len(results) == 7 - assert set(results) == { - "test.seed", - "test.table_model", - "test.view_model", - "test.ephemeral_model", - "test.not_null_view_model_id", - "test.unique_view_model_id", - "exposure:test.my_exposure", - } - - # add a new faulty row to the seed - changed_seed_contents = fixtures.seed_csv + "\n" + "\\\3,carl" - write_file(changed_seed_contents, "seeds", "seed.csv") - - self.clear_state() - run_dbt(["seed"], expect_pass=False) - self.copy_state() - - results = run_dbt( - ["ls", "--resource-type", "seed", "--select", "result:error", "--state", "./state"], - expect_pass=True, - ) - assert len(results) == 1 - assert results[0] == "test.seed" - - results = run_dbt(["ls", "--select", "result:error", "--state", "./state"]) - assert len(results) == 1 - assert results[0] == "test.seed" - - results = run_dbt(["ls", "--select", "result:error+", "--state", "./state"]) - assert len(results) == 7 - assert set(results) == { - "test.seed", - "test.table_model", - "test.view_model", - "test.ephemeral_model", - "test.not_null_view_model_id", - "test.unique_view_model_id", - "exposure:test.my_exposure", - } - - -class TestBuildRunResultsState(BaseRunResultsState): - def test_build_run_results_state(self, project): - self.run_and_save_state() - results = run_dbt(["build", "--select", "result:error", "--state", "./state"]) - assert len(results) == 0 - - self.update_view_model_bad_sql() - self.rebuild_run_dbt(expect_pass=False) - - results = run_dbt( - ["build", "--select", "result:error", "--state", "./state"], expect_pass=False - ) - assert len(results) == 3 - nodes = set([elem.node.name for elem in results]) - assert nodes == {"view_model", "not_null_view_model_id", "unique_view_model_id"} - - results = run_dbt(["ls", "--select", "result:error", "--state", "./state"]) - assert len(results) == 3 - assert set(results) == { - "test.view_model", - "test.not_null_view_model_id", - "test.unique_view_model_id", - } - - results = run_dbt( - ["build", "--select", "result:error+", "--state", "./state"], expect_pass=False - ) - assert len(results) == 4 - nodes = set([elem.node.name for elem in results]) - assert nodes == { - "table_model", - "view_model", - "not_null_view_model_id", - "unique_view_model_id", - } - - results = run_dbt(["ls", "--select", "result:error+", "--state", "./state"]) - assert len(results) == 6 # includes exposure - assert set(results) == { - "test.table_model", - "test.view_model", - "test.ephemeral_model", - "test.not_null_view_model_id", - "test.unique_view_model_id", - "exposure:test.my_exposure", - } - - self.update_view_model_failing_tests() - self.rebuild_run_dbt(expect_pass=False) - - results = run_dbt( - ["build", "--select", "result:fail", "--state", "./state"], expect_pass=False - ) - assert len(results) == 1 - assert results[0].node.name == "unique_view_model_id" - - results = run_dbt(["ls", "--select", "result:fail", "--state", "./state"]) - assert len(results) == 1 - assert results[0] == "test.unique_view_model_id" - - results = run_dbt( - ["build", "--select", "result:fail+", "--state", "./state"], expect_pass=False - ) - assert len(results) == 1 - nodes = set([elem.node.name for elem in results]) - assert nodes == {"unique_view_model_id"} - - results = run_dbt(["ls", "--select", "result:fail+", "--state", "./state"]) - assert len(results) == 1 - assert set(results) == {"test.unique_view_model_id"} - - self.update_unique_test_severity_warn() - self.rebuild_run_dbt(expect_pass=True) - - results = run_dbt( - ["build", "--select", "result:warn", "--state", "./state"], expect_pass=True - ) - assert len(results) == 1 - assert results[0].node.name == "unique_view_model_id" - - results = run_dbt(["ls", "--select", "result:warn", "--state", "./state"]) - assert len(results) == 1 - assert results[0] == "test.unique_view_model_id" - - results = run_dbt( - ["build", "--select", "result:warn+", "--state", "./state"], expect_pass=True - ) - assert len(results) == 1 - nodes = set([elem.node.name for elem in results]) - assert nodes == {"unique_view_model_id"} - - results = run_dbt(["ls", "--select", "result:warn+", "--state", "./state"]) - assert len(results) == 1 - assert set(results) == {"test.unique_view_model_id"} - - -class TestRunRunResultsState(BaseRunResultsState): - def test_run_run_results_state(self, project): - self.run_and_save_state() - results = run_dbt( - ["run", "--select", "result:success", "--state", "./state"], expect_pass=True - ) - assert len(results) == 2 - assert results[0].node.name == "view_model" - assert results[1].node.name == "table_model" - - # clear state and rerun upstream view model to test + operator - self.clear_state() - run_dbt(["run", "--select", "view_model"], expect_pass=True) - self.copy_state() - results = run_dbt( - ["run", "--select", "result:success+", "--state", "./state"], expect_pass=True - ) - assert len(results) == 2 - assert results[0].node.name == "view_model" - assert results[1].node.name == "table_model" - - # check we are starting from a place with 0 errors - results = run_dbt(["run", "--select", "result:error", "--state", "./state"]) - assert len(results) == 0 - - self.update_view_model_bad_sql() - self.clear_state() - run_dbt(["run"], expect_pass=False) - self.copy_state() - - # test single result selector on error - results = run_dbt( - ["run", "--select", "result:error", "--state", "./state"], expect_pass=False - ) - assert len(results) == 1 - assert results[0].node.name == "view_model" - - # test + operator selection on error - results = run_dbt( - ["run", "--select", "result:error+", "--state", "./state"], expect_pass=False - ) - assert len(results) == 2 - assert results[0].node.name == "view_model" - assert results[1].node.name == "table_model" - - # single result selector on skipped. Expect this to pass becase underlying view already defined above - results = run_dbt( - ["run", "--select", "result:skipped", "--state", "./state"], expect_pass=True - ) - assert len(results) == 1 - assert results[0].node.name == "table_model" - - # add a downstream model that depends on table_model for skipped+ selector - downstream_model_sql = "select * from {{ref('table_model')}}" - write_file(downstream_model_sql, "models", "table_model_downstream.sql") - - self.clear_state() - run_dbt(["run"], expect_pass=False) - self.copy_state() - - results = run_dbt( - ["run", "--select", "result:skipped+", "--state", "./state"], expect_pass=True - ) - assert len(results) == 2 - assert results[0].node.name == "table_model" - assert results[1].node.name == "table_model_downstream" - - -class TestTestRunResultsState(BaseRunResultsState): - def test_test_run_results_state(self, project): - self.run_and_save_state() - # run passed nodes - results = run_dbt( - ["test", "--select", "result:pass", "--state", "./state"], expect_pass=True - ) - assert len(results) == 2 - nodes = set([elem.node.name for elem in results]) - assert nodes == {"unique_view_model_id", "not_null_view_model_id"} - - # run passed nodes with + operator - results = run_dbt( - ["test", "--select", "result:pass+", "--state", "./state"], expect_pass=True - ) - assert len(results) == 2 - nodes = set([elem.node.name for elem in results]) - assert nodes == {"unique_view_model_id", "not_null_view_model_id"} - - self.update_view_model_failing_tests() - self.rebuild_run_dbt(expect_pass=False) - - # test with failure selector - results = run_dbt( - ["test", "--select", "result:fail", "--state", "./state"], expect_pass=False - ) - assert len(results) == 1 - assert results[0].node.name == "unique_view_model_id" - - # test with failure selector and + operator - results = run_dbt( - ["test", "--select", "result:fail+", "--state", "./state"], expect_pass=False - ) - assert len(results) == 1 - assert results[0].node.name == "unique_view_model_id" - - self.update_unique_test_severity_warn() - # rebuild - expect_pass = True because we changed the error to a warning this time around - self.rebuild_run_dbt(expect_pass=True) - - # test with warn selector - results = run_dbt( - ["test", "--select", "result:warn", "--state", "./state"], expect_pass=True - ) - assert len(results) == 1 - assert results[0].node.name == "unique_view_model_id" - - # test with warn selector and + operator - results = run_dbt( - ["test", "--select", "result:warn+", "--state", "./state"], expect_pass=True - ) - assert len(results) == 1 - assert results[0].node.name == "unique_view_model_id" - - -class TestConcurrentSelectionRunResultsState(BaseRunResultsState): - def test_concurrent_selection_run_run_results_state(self, project): - self.run_and_save_state() - results = run_dbt( - ["run", "--select", "state:modified+", "result:error+", "--state", "./state"] - ) - assert len(results) == 0 - - self.update_view_model_bad_sql() - self.clear_state() - run_dbt(["run"], expect_pass=False) - self.copy_state() - - # add a new failing dbt model - bad_sql = "select * from forced_error" - write_file(bad_sql, "models", "table_model_modified_example.sql") - - results = run_dbt( - ["run", "--select", "state:modified+", "result:error+", "--state", "./state"], - expect_pass=False, - ) - assert len(results) == 3 - nodes = set([elem.node.name for elem in results]) - assert nodes == {"view_model", "table_model_modified_example", "table_model"} - - -class TestConcurrentSelectionTestRunResultsState(BaseRunResultsState): - def test_concurrent_selection_test_run_results_state(self, project): - self.run_and_save_state() - # create failure test case for result:fail selector - self.update_view_model_failing_tests(with_nulls=True) - - # run dbt build again to trigger test errors - self.rebuild_run_dbt(expect_pass=False) - - # get the failures from - results = run_dbt( - [ - "test", - "--select", - "result:fail", - "--exclude", - "not_null_view_model_id", - "--state", - "./state", - ], - expect_pass=False, - ) - assert len(results) == 1 - nodes = set([elem.node.name for elem in results]) - assert nodes == {"unique_view_model_id"} - - -class TestConcurrentSelectionBuildRunResultsState(BaseRunResultsState): - def test_concurrent_selectors_build_run_results_state(self, project): - self.run_and_save_state() - results = run_dbt( - ["build", "--select", "state:modified+", "result:error+", "--state", "./state"] - ) - assert len(results) == 0 - - self.update_view_model_bad_sql() - self.rebuild_run_dbt(expect_pass=False) - - # add a new failing dbt model - bad_sql = "select * from forced_error" - write_file(bad_sql, "models", "table_model_modified_example.sql") - - results = run_dbt( - ["build", "--select", "state:modified+", "result:error+", "--state", "./state"], - expect_pass=False, - ) - assert len(results) == 5 - nodes = set([elem.node.name for elem in results]) - assert nodes == { - "table_model_modified_example", - "view_model", - "table_model", - "not_null_view_model_id", - "unique_view_model_id", - } - - self.update_view_model_failing_tests() - - # create error model case for result:error selector - more_bad_sql = "select 1 as id from not_exists" - write_file(more_bad_sql, "models", "error_model.sql") - - # create something downstream from the error model to rerun - downstream_model_sql = "select * from {{ ref('error_model') }} )" - write_file(downstream_model_sql, "models", "downstream_of_error_model.sql") - - # regenerate build state - self.rebuild_run_dbt(expect_pass=False) - - # modify model again to trigger the state:modified selector - bad_again_sql = "select * from forced_anothererror" - write_file(bad_again_sql, "models", "table_model_modified_example.sql") - - results = run_dbt( - [ - "build", - "--select", - "state:modified+", - "result:error+", - "result:fail+", - "--state", - "./state", - ], - expect_pass=False, - ) - assert len(results) == 4 - nodes = set([elem.node.name for elem in results]) - assert nodes == { - "error_model", - "downstream_of_error_model", - "table_model_modified_example", - "unique_view_model_id", - }