From 8446b977a01b1443f1ae14b78ce4d58d46f765c8 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Thu, 2 May 2024 18:29:58 -0400 Subject: [PATCH] [Tech Debt] Remove context methods test suite (#83) --- .../context_methods/first_dependency.py | 95 -------- .../context_methods/test_builtin_functions.py | 143 ------------ .../context_methods/test_cli_var_override.py | 66 ------ .../context_methods/test_cli_vars.py | 205 ------------------ .../context_methods/test_custom_env_vars.py | 36 --- .../context_methods/test_env_vars.py | 195 ----------------- .../context_methods/test_secret_env_vars.py | 185 ---------------- .../context_methods/test_var_dependency.py | 82 ------- .../test_var_in_generate_name.py | 43 ---- .../context_methods/test_yaml_functions.py | 49 ----- 10 files changed, 1099 deletions(-) delete mode 100644 tests/functional/context_methods/first_dependency.py delete mode 100644 tests/functional/context_methods/test_builtin_functions.py delete mode 100644 tests/functional/context_methods/test_cli_var_override.py delete mode 100644 tests/functional/context_methods/test_cli_vars.py delete mode 100644 tests/functional/context_methods/test_custom_env_vars.py delete mode 100644 tests/functional/context_methods/test_env_vars.py delete mode 100644 tests/functional/context_methods/test_secret_env_vars.py delete mode 100644 tests/functional/context_methods/test_var_dependency.py delete mode 100644 tests/functional/context_methods/test_var_in_generate_name.py delete mode 100644 tests/functional/context_methods/test_yaml_functions.py diff --git a/tests/functional/context_methods/first_dependency.py b/tests/functional/context_methods/first_dependency.py deleted file mode 100644 index 8e1365be..00000000 --- a/tests/functional/context_methods/first_dependency.py +++ /dev/null @@ -1,95 +0,0 @@ -from dbt.tests.fixtures.project import write_project_files -import pytest - - -first_dependency__dbt_project_yml = """ -name: 'first_dep' -version: '1.0' -config-version: 2 - -profile: 'default' - -model-paths: ["models"] -analysis-paths: ["analyses"] -test-paths: ["tests"] -seed-paths: ["seeds"] -macro-paths: ["macros"] - -require-dbt-version: '>=0.1.0' - -target-path: "target" # directory which will store compiled SQL files -clean-targets: # directories to be removed by `dbt clean` - - "target" - - "dbt_packages" - -vars: - first_dep: - first_dep_global: 'first_dep_global_value_overridden' - test_config_root_override: 'configured_from_dependency' - test_config_package: 'configured_from_dependency' - -seeds: - quote_columns: True - -""" - -first_dependency__models__nested__first_dep_model_sql = """ -select - '{{ var("first_dep_global") }}' as first_dep_global, - '{{ var("from_root_to_first") }}' as from_root -""" - -first_dependency__seeds__first_dep_expected_csv = """first_dep_global,from_root -first_dep_global_value_overridden,root_first_value -""" - -first_dependency__models__nested__first_dep_model_var_expected_csv = """test_config_root_override,test_config_package -configured_from_root,configured_from_dependency -""" - -first_dependency__models__nested__first_dep_model_var_sql = """ -select - '{{ config.get("test_config_root_override") }}' as test_config_root_override, - '{{ config.get("test_config_package") }}' as test_config_package -""" - -first_dependency__model_var_in_config_schema = """ -models: -- name: first_dep_model - config: - test_config_root_override: "{{ var('test_config_root_override') }}" - test_config_package: "{{ var('test_config_package') }}" -""" - - -class FirstDependencyProject: - @pytest.fixture(scope="class") - def first_dependency(self, project): - first_dependency_files = { - "dbt_project.yml": first_dependency__dbt_project_yml, - "models": { - "nested": { - "first_dep_model.sql": first_dependency__models__nested__first_dep_model_sql - } - }, - "seeds": {"first_dep_expected.csv": first_dependency__seeds__first_dep_expected_csv}, - } - write_project_files(project.project_root, "first_dependency", first_dependency_files) - - -class FirstDependencyConfigProject: - @pytest.fixture(scope="class") - def first_dependency(self, project): - first_dependency_files = { - "dbt_project.yml": first_dependency__dbt_project_yml, - "models": { - "nested": { - "first_dep_model.sql": first_dependency__models__nested__first_dep_model_var_sql, - "schema.yml": first_dependency__model_var_in_config_schema, - } - }, - "seeds": { - "first_dep_expected.csv": first_dependency__models__nested__first_dep_model_var_expected_csv - }, - } - write_project_files(project.project_root, "first_dependency", first_dependency_files) diff --git a/tests/functional/context_methods/test_builtin_functions.py b/tests/functional/context_methods/test_builtin_functions.py deleted file mode 100644 index b8a47b34..00000000 --- a/tests/functional/context_methods/test_builtin_functions.py +++ /dev/null @@ -1,143 +0,0 @@ -import json - -from dbt.tests.util import write_file -from dbt_common.exceptions import CompilationError -import pytest - -from tests.functional.utils import run_dbt, run_dbt_and_capture - - -macros__validate_set_sql = """ -{% macro validate_set() %} - {% set set_result = set([1, 2, 2, 3, 'foo', False]) %} - {{ log("set_result: " ~ set_result) }} - {% set set_strict_result = set_strict([1, 2, 2, 3, 'foo', False]) %} - {{ log("set_strict_result: " ~ set_strict_result) }} -{% endmacro %} -""" - -macros__validate_zip_sql = """ -{% macro validate_zip() %} - {% set list_a = [1, 2] %} - {% set list_b = ['foo', 'bar'] %} - {% set zip_result = zip(list_a, list_b) | list %} - {{ log("zip_result: " ~ zip_result) }} - {% set zip_strict_result = zip_strict(list_a, list_b) | list %} - {{ log("zip_strict_result: " ~ zip_strict_result) }} -{% endmacro %} -""" - -macros__validate_invocation_sql = """ -{% macro validate_invocation(my_variable) %} - -- check a specific value - {{ log("use_colors: "~ invocation_args_dict['use_colors']) }} - -- whole dictionary (as string) - {{ log("invocation_result: "~ invocation_args_dict) }} -{% endmacro %} -""" - -macros__validate_dbt_metadata_envs_sql = """ -{% macro validate_dbt_metadata_envs() %} - {{ log("dbt_metadata_envs_result:"~ dbt_metadata_envs) }} -{% endmacro %} -""" - -models__set_exception_sql = """ -{% set set_strict_result = set_strict(1) %} -""" - -models__zip_exception_sql = """ -{% set zip_strict_result = zip_strict(1) %} -""" - - -def parse_json_logs(json_log_output): - parsed_logs = [] - for line in json_log_output.split("\n"): - try: - log = json.loads(line) - except ValueError: - continue - - parsed_logs.append(log) - - return parsed_logs - - -def find_result_in_parsed_logs(parsed_logs, result_name): - return next( - ( - item["data"]["msg"] - for item in parsed_logs - if result_name in item["data"].get("msg", "msg") - ), - False, - ) - - -class TestContextBuiltins: - @pytest.fixture(scope="class") - def macros(self): - return { - "validate_set.sql": macros__validate_set_sql, - "validate_zip.sql": macros__validate_zip_sql, - "validate_invocation.sql": macros__validate_invocation_sql, - "validate_dbt_metadata_envs.sql": macros__validate_dbt_metadata_envs_sql, - } - - def test_builtin_set_function(self, project): - _, log_output = run_dbt_and_capture(["--debug", "run-operation", "validate_set"]) - - # The order of the set isn't guaranteed so we can't check for the actual set in the logs - assert "set_result: " in log_output - assert "False" in log_output - assert "set_strict_result: " in log_output - - def test_builtin_zip_function(self, project): - _, log_output = run_dbt_and_capture(["--debug", "run-operation", "validate_zip"]) - - expected_zip = [(1, "foo"), (2, "bar")] - assert f"zip_result: {expected_zip}" in log_output - assert f"zip_strict_result: {expected_zip}" in log_output - - def test_builtin_invocation_args_dict_function(self, project): - _, log_output = run_dbt_and_capture( - [ - "--debug", - "--log-format=json", - "run-operation", - "validate_invocation", - "--args", - "{my_variable: test_variable}", - ] - ) - - parsed_logs = parse_json_logs(log_output) - use_colors = result = find_result_in_parsed_logs(parsed_logs, "use_colors") - assert use_colors == "use_colors: True" - invocation_dict = find_result_in_parsed_logs(parsed_logs, "invocation_result") - assert result - # The result should include a dictionary of all flags with values that aren't None - expected = ( - "'send_anonymous_usage_stats': False", - "'quiet': False", - "'print': True", - "'cache_selected_only': False", - "'macro': 'validate_invocation'", - "'args': {'my_variable': 'test_variable'}", - "'which': 'run-operation'", - "'indirect_selection': 'eager'", - ) - assert all(element in invocation_dict for element in expected) - - -class TestContextBuiltinExceptions: - # Assert compilation errors are raised with _strict equivalents - def test_builtin_function_exception(self, project): - write_file(models__set_exception_sql, project.project_root, "models", "raise.sql") - with pytest.raises(CompilationError): - run_dbt(["compile"]) - - write_file(models__zip_exception_sql, project.project_root, "models", "raise.sql") - with pytest.raises(CompilationError): - run_dbt(["compile"]) diff --git a/tests/functional/context_methods/test_cli_var_override.py b/tests/functional/context_methods/test_cli_var_override.py deleted file mode 100644 index 757ab521..00000000 --- a/tests/functional/context_methods/test_cli_var_override.py +++ /dev/null @@ -1,66 +0,0 @@ -from dbt.tests.util import run_dbt -import pytest - - -models_override__schema_yml = """ -version: 2 -models: -- name: test_vars - columns: - - name: field - data_tests: - - accepted_values: - values: - - override -""" - -models_override__test_vars_sql = """ -select '{{ var("required") }}'::varchar as field -""" - - -# Tests that cli vars override vars set in the project config -class TestCLIVarOverride: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_override__schema_yml, - "test_vars.sql": models_override__test_vars_sql, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "vars": { - "required": "present", - }, - } - - def test__override_vars_global(self, project): - run_dbt(["run", "--vars", "{required: override}"]) - run_dbt(["test"]) - - -# This one switches to setting a var in 'test' -class TestCLIVarOverridePorject: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_override__schema_yml, - "test_vars.sql": models_override__test_vars_sql, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "vars": { - "test": { - "required": "present", - }, - }, - } - - def test__override_vars_project_level(self, project): - # This should be "override" - run_dbt(["run", "--vars", "{required: override}"]) - run_dbt(["test"]) diff --git a/tests/functional/context_methods/test_cli_vars.py b/tests/functional/context_methods/test_cli_vars.py deleted file mode 100644 index 8f6d6e8d..00000000 --- a/tests/functional/context_methods/test_cli_vars.py +++ /dev/null @@ -1,205 +0,0 @@ -from dbt.tests.fixtures.project import write_project_files -from dbt.tests.util import get_artifact, run_dbt, write_config_file -from dbt_common.exceptions import CompilationError, DbtRuntimeError -import pytest -import yaml - - -models_complex__schema_yml = """ -version: 2 -models: -- name: complex_model - columns: - - name: var_1 - data_tests: - - accepted_values: - values: - - abc - - name: var_2 - data_tests: - - accepted_values: - values: - - def - - name: var_3 - data_tests: - - accepted_values: - values: - - jkl -""" - -models_complex__complex_model_sql = """ -select - '{{ var("variable_1") }}'::varchar as var_1, - '{{ var("variable_2")[0] }}'::varchar as var_2, - '{{ var("variable_3")["value"] }}'::varchar as var_3 -""" - -models_simple__schema_yml = """ -version: 2 -models: -- name: simple_model - columns: - - name: simple - data_tests: - - accepted_values: - values: - - abc -""" - -models_simple__simple_model_sql = """ -select - '{{ var("simple") }}'::varchar as simple -""" - -really_simple_model_sql = """ -select 'abc' as simple -""" - - -class TestCLIVars: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_complex__schema_yml, - "complex_model.sql": models_complex__complex_model_sql, - } - - def test__cli_vars_longform(self, project): - cli_vars = { - "variable_1": "abc", - "variable_2": ["def", "ghi"], - "variable_3": {"value": "jkl"}, - } - results = run_dbt(["run", "--vars", yaml.dump(cli_vars)]) - assert len(results) == 1 - results = run_dbt(["test", "--vars", yaml.dump(cli_vars)]) - assert len(results) == 3 - - -class TestCLIVarsSimple: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_simple__schema_yml, - "simple_model.sql": models_simple__simple_model_sql, - } - - def test__cli_vars_shorthand(self, project): - results = run_dbt(["run", "--vars", "simple: abc"]) - assert len(results) == 1 - results = run_dbt(["test", "--vars", "simple: abc"]) - assert len(results) == 1 - - def test__cli_vars_longer(self, project): - results = run_dbt(["run", "--vars", "{simple: abc, unused: def}"]) - assert len(results) == 1 - results = run_dbt(["test", "--vars", "{simple: abc, unused: def}"]) - assert len(results) == 1 - run_results = get_artifact(project.project_root, "target", "run_results.json") - assert run_results["args"]["vars"] == {"simple": "abc", "unused": "def"} - - -class TestCLIVarsProfile: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_simple__schema_yml, - "simple_model.sql": really_simple_model_sql, - } - - def test_cli_vars_in_profile(self, project, dbt_profile_data): - profile = dbt_profile_data - profile["test"]["outputs"]["default"]["host"] = "{{ var('db_host') }}" - write_config_file(profile, project.profiles_dir, "profiles.yml") - with pytest.raises(DbtRuntimeError): - results = run_dbt(["run"]) - results = run_dbt(["run", "--vars", "db_host: localhost"]) - assert len(results) == 1 - - -class TestCLIVarsPackages: - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project_root, dbt_integration_project): # noqa: F811 - write_project_files(project_root, "dbt_integration_project", dbt_integration_project) - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_simple__schema_yml, - "simple_model.sql": really_simple_model_sql, - } - - @pytest.fixture(scope="class") - def packages_config(self): - return {"packages": [{"local": "dbt_integration_project"}]} - - def test_cli_vars_in_packages(self, project, packages_config): - # Run working deps and run commands - run_dbt(["deps"]) - results = run_dbt(["run"]) - assert len(results) == 1 - - # Change packages.yml to contain a var - packages = packages_config - packages["packages"][0]["local"] = "{{ var('path_to_project') }}" - write_config_file(packages, project.project_root, "packages.yml") - - # Without vars args deps fails - with pytest.raises(DbtRuntimeError): - run_dbt(["deps"]) - - # With vars arg deps succeeds - results = run_dbt(["deps", "--vars", "path_to_project: dbt_integration_project"]) - assert results is None - - -initial_selectors_yml = """ -selectors: - - name: dev_defer_snapshots - default: "{{ target.name == 'dev' | as_bool }}" - definition: - method: fqn - value: '*' - exclude: - - method: config.materialized - value: snapshot -""" - -var_selectors_yml = """ -selectors: - - name: dev_defer_snapshots - default: "{{ var('snapshot_target') == 'dev' | as_bool }}" - definition: - method: fqn - value: '*' - exclude: - - method: config.materialized - value: snapshot -""" - - -class TestCLIVarsSelectors: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_simple__schema_yml, - "simple_model.sql": really_simple_model_sql, - } - - @pytest.fixture(scope="class") - def selectors(self): - return initial_selectors_yml - - def test_vars_in_selectors(self, project): - # initially runs ok - results = run_dbt(["run"]) - assert len(results) == 1 - - # Update the selectors.yml file to have a var - write_config_file(var_selectors_yml, project.project_root, "selectors.yml") - with pytest.raises(CompilationError): - run_dbt(["run"]) - - # Var in cli_vars works - results = run_dbt(["run", "--vars", "snapshot_target: dev"]) - assert len(results) == 1 diff --git a/tests/functional/context_methods/test_custom_env_vars.py b/tests/functional/context_methods/test_custom_env_vars.py deleted file mode 100644 index 50a9b00c..00000000 --- a/tests/functional/context_methods/test_custom_env_vars.py +++ /dev/null @@ -1,36 +0,0 @@ -import json -import os - -import pytest - -from tests.functional.utils import run_dbt_and_capture - - -def parse_json_logs(json_log_output): - parsed_logs = [] - for line in json_log_output.split("\n"): - try: - log = json.loads(line) - except ValueError: - continue - - parsed_logs.append(log) - - return parsed_logs - - -class TestCustomVarInLogs: - @pytest.fixture(scope="class", autouse=True) - def setup(self): - # on windows, python uppercases env var names because windows is case insensitive - os.environ["DBT_ENV_CUSTOM_ENV_SOME_VAR"] = "value" - yield - del os.environ["DBT_ENV_CUSTOM_ENV_SOME_VAR"] - - def test_extra_filled(self, project): - _, log_output = run_dbt_and_capture( - ["--log-format=json", "deps"], - ) - logs = parse_json_logs(log_output) - for log in logs: - assert log["info"].get("extra") == {"SOME_VAR": "value"} diff --git a/tests/functional/context_methods/test_env_vars.py b/tests/functional/context_methods/test_env_vars.py deleted file mode 100644 index 0bfbd01c..00000000 --- a/tests/functional/context_methods/test_env_vars.py +++ /dev/null @@ -1,195 +0,0 @@ -import os - -from dbt.constants import DEFAULT_ENV_PLACEHOLDER, SECRET_ENV_PREFIX -from dbt.tests.util import get_manifest -import pytest - -from tests.functional.utils import run_dbt, run_dbt_and_capture - - -context_sql = """ - -{{ - config( - materialized='table' - ) -}} - -select - - -- compile-time variables - '{{ this }}' as "this", - '{{ this.name }}' as "this.name", - '{{ this.schema }}' as "this.schema", - '{{ this.table }}' as "this.table", - - '{{ target.dbname }}' as "target.dbname", - '{{ target.host }}' as "target.host", - '{{ target.name }}' as "target.name", - '{{ target.schema }}' as "target.schema", - '{{ target.type }}' as "target.type", - '{{ target.user }}' as "target.user", - '{{ target.get("pass", "") }}' as "target.pass", -- not actually included, here to test that it is _not_ present! - {{ target.port }} as "target.port", - {{ target.threads }} as "target.threads", - - -- runtime variables - '{{ run_started_at }}' as run_started_at, - '{{ invocation_id }}' as invocation_id, - '{{ thread_id }}' as thread_id, - - '{{ env_var("DBT_TEST_ENV_VAR") }}' as env_var, - '{{ env_var("DBT_TEST_IGNORE_DEFAULT", "ignored_default_val") }}' as env_var_ignore_default, - '{{ env_var("DBT_TEST_USE_DEFAULT", "use_my_default_val") }}' as env_var_use_default, - 'secret_variable' as env_var_secret, -- make sure the value itself is scrubbed from the logs - '{{ env_var("DBT_TEST_NOT_SECRET") }}' as env_var_not_secret - -""" - - -class TestEnvVars: - @pytest.fixture(scope="class") - def models(self): - return {"context.sql": context_sql} - - @pytest.fixture(scope="class", autouse=True) - def setup(self): - os.environ["DBT_TEST_ENV_VAR"] = "1" - os.environ["DBT_TEST_USER"] = "root" - os.environ["DBT_TEST_PASS"] = "password" - os.environ[SECRET_ENV_PREFIX + "SECRET"] = "secret_variable" - os.environ["DBT_TEST_NOT_SECRET"] = "regular_variable" - os.environ["DBT_TEST_IGNORE_DEFAULT"] = "ignored_default" - yield - del os.environ["DBT_TEST_ENV_VAR"] - del os.environ["DBT_TEST_USER"] - del os.environ[SECRET_ENV_PREFIX + "SECRET"] - del os.environ["DBT_TEST_NOT_SECRET"] - del os.environ["DBT_TEST_IGNORE_DEFAULT"] - - @pytest.fixture(scope="class") - def profiles_config_update(self, unique_schema): - return { - "test": { - "outputs": { - # don't use env_var's here so the integration tests can run - # seed sql statements and the like. default target is used - "dev": { - "type": "postgres", - "threads": 1, - "host": "localhost", - "port": 5432, - "user": "root", - "pass": "password", - "dbname": "dbt", - "schema": unique_schema, - }, - "prod": { - "type": "postgres", - "threads": 1, - "host": "localhost", - "port": 5432, - # root/password - "user": "{{ env_var('DBT_TEST_USER') }}", - "pass": "{{ env_var('DBT_TEST_PASS') }}", - "dbname": "dbt", - "schema": unique_schema, - }, - }, - "target": "dev", - } - } - - def get_ctx_vars(self, project): - fields = [ - "this", - "this.name", - "this.schema", - "this.table", - "target.dbname", - "target.host", - "target.name", - "target.port", - "target.schema", - "target.threads", - "target.type", - "target.user", - "target.pass", - "run_started_at", - "invocation_id", - "thread_id", - "env_var", - ] - field_list = ", ".join(['"{}"'.format(f) for f in fields]) - query = "select {field_list} from {schema}.context".format( - field_list=field_list, schema=project.test_schema - ) - vals = project.run_sql(query, fetch="all") - ctx = dict([(k, v) for (k, v) in zip(fields, vals[0])]) - return ctx - - def test_env_vars_dev( - self, - project, - ): - results = run_dbt(["run"]) - assert len(results) == 1 - ctx = self.get_ctx_vars(project) - - manifest = get_manifest(project.project_root) - expected = { - "DBT_TEST_ENV_VAR": "1", - "DBT_TEST_NOT_SECRET": "regular_variable", - "DBT_TEST_IGNORE_DEFAULT": "ignored_default", - "DBT_TEST_USE_DEFAULT": DEFAULT_ENV_PLACEHOLDER, - } - assert manifest.env_vars == expected - - this = '"{}"."{}"."context"'.format(project.database, project.test_schema) - assert ctx["this"] == this - - assert ctx["this.name"] == "context" - assert ctx["this.schema"] == project.test_schema - assert ctx["this.table"] == "context" - - assert ctx["target.dbname"] == "dbt" - assert ctx["target.host"] == "localhost" - assert ctx["target.name"] == "dev" - assert ctx["target.port"] == 5432 - assert ctx["target.schema"] == project.test_schema - assert ctx["target.threads"] == 1 - assert ctx["target.type"] == "postgres" - assert ctx["target.user"] == "root" - assert ctx["target.pass"] == "" - - assert ctx["env_var"] == "1" - - def test_env_vars_prod(self, project): - results = run_dbt(["run", "--target", "prod"]) - assert len(results) == 1 - ctx = self.get_ctx_vars(project) - - this = '"{}"."{}"."context"'.format(project.database, project.test_schema) - assert ctx["this"] == this - - assert ctx["this.name"] == "context" - assert ctx["this.schema"] == project.test_schema - assert ctx["this.table"] == "context" - - assert ctx["target.dbname"] == "dbt" - assert ctx["target.host"] == "localhost" - assert ctx["target.name"] == "prod" - assert ctx["target.port"] == 5432 - assert ctx["target.schema"] == project.test_schema - assert ctx["target.threads"] == 1 - assert ctx["target.type"] == "postgres" - assert ctx["target.user"] == "root" - assert ctx["target.pass"] == "" - assert ctx["env_var"] == "1" - - def test_env_vars_secrets(self, project): - os.environ["DBT_DEBUG"] = "True" - _, log_output = run_dbt_and_capture(["run", "--target", "prod"]) - - assert not ("secret_variable" in log_output) - assert "regular_variable" in log_output diff --git a/tests/functional/context_methods/test_secret_env_vars.py b/tests/functional/context_methods/test_secret_env_vars.py deleted file mode 100644 index a6a5537a..00000000 --- a/tests/functional/context_methods/test_secret_env_vars.py +++ /dev/null @@ -1,185 +0,0 @@ -import os - -from dbt.constants import SECRET_ENV_PREFIX -from dbt.exceptions import ParsingError -from dbt.tests.util import read_file -from dbt_common.exceptions import DbtInternalError -import pytest - -from tests.functional.context_methods.first_dependency import FirstDependencyProject -from tests.functional.utils import run_dbt, run_dbt_and_capture - - -secret_bad__context_sql = """ - -{{ - config( - materialized='table' - ) -}} - -select - - '{{ env_var("DBT_TEST_ENV_VAR") }}' as env_var, - '{{ env_var("DBT_ENV_SECRET_SECRET") }}' as env_var_secret, -- this should raise an error! - '{{ env_var("DBT_TEST_NOT_SECRET") }}' as env_var_not_secret - -""" - - -class TestDisallowSecretModel: - @pytest.fixture(scope="class") - def models(self): - return {"context.sql": secret_bad__context_sql} - - def test_disallow_secret(self, project): - with pytest.raises(ParsingError): - run_dbt(["compile"]) - - -models__context_sql = """ -{{ - config( - materialized='table' - ) -}} - -select - - -- compile-time variables - '{{ this }}' as "this", - '{{ this.name }}' as "this.name", - '{{ this.schema }}' as "this.schema", - '{{ this.table }}' as "this.table", - - '{{ target.dbname }}' as "target.dbname", - '{{ target.host }}' as "target.host", - '{{ target.name }}' as "target.name", - '{{ target.schema }}' as "target.schema", - '{{ target.type }}' as "target.type", - '{{ target.user }}' as "target.user", - '{{ target.get("pass", "") }}' as "target.pass", -- not actually included, here to test that it is _not_ present! - {{ target.port }} as "target.port", - {{ target.threads }} as "target.threads", - - -- runtime variables - '{{ run_started_at }}' as run_started_at, - '{{ invocation_id }}' as invocation_id, - '{{ thread_id }}' as thread_id, - - '{{ env_var("DBT_TEST_ENV_VAR") }}' as env_var, - 'secret_variable' as env_var_secret, -- make sure the value itself is scrubbed from the logs - '{{ env_var("DBT_TEST_NOT_SECRET") }}' as env_var_not_secret -""" - - -class TestAllowSecretProfilePackage(FirstDependencyProject): - @pytest.fixture(scope="class", autouse=True) - def setup(self): - os.environ[SECRET_ENV_PREFIX + "USER"] = "root" - os.environ[SECRET_ENV_PREFIX + "PASS"] = "password" - os.environ[SECRET_ENV_PREFIX + "PACKAGE"] = "first_dependency" - os.environ[SECRET_ENV_PREFIX + "GIT_TOKEN"] = "abc123" - yield - del os.environ[SECRET_ENV_PREFIX + "USER"] - del os.environ[SECRET_ENV_PREFIX + "PASS"] - del os.environ[SECRET_ENV_PREFIX + "PACKAGE"] - del os.environ[SECRET_ENV_PREFIX + "GIT_TOKEN"] - - @pytest.fixture(scope="class") - def models(self): - return {"context.sql": models__context_sql} - - @pytest.fixture(scope="class") - def packages(self): - return { - "packages": [ - { - # the raw value of this secret *will* be written to lock file - "local": "{{ env_var('DBT_ENV_SECRET_PACKAGE') }}" - }, - { - # this secret env var will *not* be written to lock file - "git": "https://{{ env_var('DBT_ENV_SECRET_GIT_TOKEN') }}@github.com/dbt-labs/dbt-external-tables.git" - }, - { - # this secret env var will *not* be written to lock file - "tarball": "https://{{ env_var('DBT_ENV_SECRET_GIT_TOKEN') }}@github.com/dbt-labs/dbt-utils/archive/refs/tags/1.1.1.tar.gz", - "name": "dbt_utils", - }, - ] - } - - @pytest.fixture(scope="class") - def profile_target(self): - return { - "type": "postgres", - "threads": 1, - "host": "localhost", - "port": 5432, - # root/password - "user": "{{ env_var('DBT_ENV_SECRET_USER') }}", - "pass": "{{ env_var('DBT_ENV_SECRET_PASS') }}", - "dbname": "dbt", - } - - def test_allow_secrets(self, project, first_dependency): - _, log_output = run_dbt_and_capture(["deps"]) - lock_file_contents = read_file("package-lock.yml") - - # this will not be written to logs or lock file - assert not ("abc123" in log_output) - assert not ("abc123" in lock_file_contents) - assert "{{ env_var('DBT_ENV_SECRET_GIT_TOKEN') }}" in lock_file_contents - - # this will be scrubbed from logs, but not from the lock file - assert not ("first_dependency" in log_output) - assert "first_dependency" in lock_file_contents - - -class TestCloneFailSecretScrubbed: - @pytest.fixture(scope="class", autouse=True) - def setup(self): - os.environ[SECRET_ENV_PREFIX + "GIT_TOKEN"] = "abc123" - - @pytest.fixture(scope="class") - def models(self): - return {"context.sql": models__context_sql} - - @pytest.fixture(scope="class") - def packages(self): - return { - "packages": [ - { - "git": "https://fakeuser:{{ env_var('DBT_ENV_SECRET_GIT_TOKEN') }}@github.com/dbt-labs/fake-repo.git" - }, - ] - } - - def test_fail_clone_with_scrubbing(self, project): - with pytest.raises(DbtInternalError) as excinfo: - _, log_output = run_dbt_and_capture(["deps"]) - - assert "abc123" not in str(excinfo.value) - - -class TestCloneFailSecretNotRendered(TestCloneFailSecretScrubbed): - # as above, with some Jinja manipulation - @pytest.fixture(scope="class") - def packages(self): - return { - "packages": [ - { - "git": "https://fakeuser:{{ env_var('DBT_ENV_SECRET_GIT_TOKEN') | join(' ') }}@github.com/dbt-labs/fake-repo.git" - }, - ] - } - - def test_fail_clone_with_scrubbing(self, project): - with pytest.raises(DbtInternalError) as excinfo: - _, log_output = run_dbt_and_capture(["deps"]) - - # we should not see any manipulated form of the secret value (abc123) here - # we should see a manipulated form of the placeholder instead - assert "a b c 1 2 3" not in str(excinfo.value) - assert "D B T _ E N V _ S E C R E T _ G I T _ T O K E N" in str(excinfo.value) diff --git a/tests/functional/context_methods/test_var_dependency.py b/tests/functional/context_methods/test_var_dependency.py deleted file mode 100644 index a0c06db7..00000000 --- a/tests/functional/context_methods/test_var_dependency.py +++ /dev/null @@ -1,82 +0,0 @@ -from dbt.tests.util import check_relations_equal, run_dbt -import pytest - -from tests.functional.context_methods.first_dependency import ( - FirstDependencyConfigProject, - FirstDependencyProject, -) - - -dependency_seeds__root_model_expected_csv = """first_dep_global,from_root -dep_never_overridden,root_root_value -""" - -dependency_models__inside__model_sql = """ -select - '{{ var("first_dep_override") }}' as first_dep_global, - '{{ var("from_root_to_root") }}' as from_root - -""" - - -class TestVarDependencyInheritance(FirstDependencyProject): - @pytest.fixture(scope="class") - def seeds(self): - return {"root_model_expected.csv": dependency_seeds__root_model_expected_csv} - - @pytest.fixture(scope="class") - def models(self): - return {"inside": {"model.sql": dependency_models__inside__model_sql}} - - @pytest.fixture(scope="class") - def packages(self): - return { - "packages": [ - {"local": "first_dependency"}, - ] - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "vars": { - "first_dep_override": "dep_never_overridden", - "test": { - "from_root_to_root": "root_root_value", - }, - "first_dep": { - "from_root_to_first": "root_first_value", - }, - }, - } - - def test_var_mutual_overrides_v1_conversion(self, project, first_dependency): - run_dbt(["deps"]) - assert len(run_dbt(["seed"])) == 2 - assert len(run_dbt(["run"])) == 2 - check_relations_equal(project.adapter, ["root_model_expected", "model"]) - check_relations_equal(project.adapter, ["first_dep_expected", "first_dep_model"]) - - -class TestVarConfigDependencyInheritance(FirstDependencyConfigProject): - @pytest.fixture(scope="class") - def packages(self): - return { - "packages": [ - {"local": "first_dependency"}, - ] - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "vars": { - "test_config_root_override": "configured_from_root", - }, - } - - def test_root_var_overrides_package_var(self, project, first_dependency): - run_dbt(["deps"]) - run_dbt(["seed"]) - assert len(run_dbt(["run"])) == 1 - check_relations_equal(project.adapter, ["first_dep_expected", "first_dep_model"]) diff --git a/tests/functional/context_methods/test_var_in_generate_name.py b/tests/functional/context_methods/test_var_in_generate_name.py deleted file mode 100644 index f36bec3a..00000000 --- a/tests/functional/context_methods/test_var_in_generate_name.py +++ /dev/null @@ -1,43 +0,0 @@ -from dbt.tests.util import run_dbt, update_config_file -from dbt_common.exceptions import CompilationError -import pytest - - -model_sql = """ -select 1 as id -""" - -bad_generate_macros__generate_names_sql = """ -{% macro generate_schema_name(custom_schema_name, node) -%} - {% do var('somevar') %} - {% do return(dbt.generate_schema_name(custom_schema_name, node)) %} -{%- endmacro %} - -""" - - -class TestMissingVarGenerateNameMacro: - @pytest.fixture(scope="class") - def macros(self): - return {"generate_names.sql": bad_generate_macros__generate_names_sql} - - @pytest.fixture(scope="class") - def models(self): - return {"model.sql": model_sql} - - def test_generate_schema_name_var(self, project): - # var isn't set, so generate_name macro fails - with pytest.raises(CompilationError) as excinfo: - run_dbt(["compile"]) - - assert "Required var 'somevar' not found in config" in str(excinfo.value) - - # globally scoped -- var is set at top-level - update_config_file({"vars": {"somevar": 1}}, project.project_root, "dbt_project.yml") - run_dbt(["compile"]) - - # locally scoped -- var is set in 'test' scope - update_config_file( - {"vars": {"test": {"somevar": 1}}}, project.project_root, "dbt_project.yml" - ) - run_dbt(["compile"]) diff --git a/tests/functional/context_methods/test_yaml_functions.py b/tests/functional/context_methods/test_yaml_functions.py deleted file mode 100644 index 8996abc9..00000000 --- a/tests/functional/context_methods/test_yaml_functions.py +++ /dev/null @@ -1,49 +0,0 @@ -from dbt.tests.util import run_dbt -import pytest - - -tests__from_yaml_sql = """ -{% set simplest = (fromyaml('a: 1') == {'a': 1}) %} -{% set nested_data %} -a: - b: - - c: 1 - d: 2 - - c: 3 - d: 4 -{% endset %} -{% set nested = (fromyaml(nested_data) == {'a': {'b': [{'c': 1, 'd': 2}, {'c': 3, 'd': 4}]}}) %} - -(select 'simplest' as name {% if simplest %}limit 0{% endif %}) -union all -(select 'nested' as name {% if nested %}limit 0{% endif %}) -""" - -tests__to_yaml_sql = """ -{% set simplest = (toyaml({'a': 1}) == 'a: 1\\n') %} -{% set default_sort = (toyaml({'b': 2, 'a': 1}) == 'b: 2\\na: 1\\n') %} -{% set unsorted = (toyaml({'b': 2, 'a': 1}, sort_keys=False) == 'b: 2\\na: 1\\n') %} -{% set sorted = (toyaml({'b': 2, 'a': 1}, sort_keys=True) == 'a: 1\\nb: 2\\n') %} -{% set default_results = (toyaml({'a': adapter}, 'failed') == 'failed') %} - -(select 'simplest' as name {% if simplest %}limit 0{% endif %}) -union all -(select 'default_sort' as name {% if default_sort %}limit 0{% endif %}) -union all -(select 'unsorted' as name {% if unsorted %}limit 0{% endif %}) -union all -(select 'sorted' as name {% if sorted %}limit 0{% endif %}) -union all -(select 'default_results' as name {% if default_results %}limit 0{% endif %}) -""" - - -class TestContextVars: - # This test has no actual models - - @pytest.fixture(scope="class") - def tests(self): - return {"from_yaml.sql": tests__from_yaml_sql, "to_yaml.sql": tests__to_yaml_sql} - - def test_json_data_tests(self, project): - assert len(run_dbt(["test"])) == 2