From ea33cbec035c32052008c07c90ef799e024aa7dc Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Tue, 22 Oct 2024 10:08:10 -0700 Subject: [PATCH 1/4] Add url. (#1222) --- dbt/include/snowflake/macros/relations/table/create.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/snowflake/macros/relations/table/create.sql b/dbt/include/snowflake/macros/relations/table/create.sql index e60b93039..e2141df4d 100644 --- a/dbt/include/snowflake/macros/relations/table/create.sql +++ b/dbt/include/snowflake/macros/relations/table/create.sql @@ -1,7 +1,7 @@ {% macro snowflake__create_table_as(temporary, relation, compiled_code, language='sql') -%} {%- if relation.is_iceberg_format and not adapter.behavior.enable_iceberg_materializations.no_warn %} - {% do exceptions.raise_compiler_error('Was unable to create model as Iceberg Table Format. Please set the `enable_iceberg_materializations` behavior flag to True in your dbt_project.yml. For more information, go to .') %} + {% do exceptions.raise_compiler_error('Was unable to create model as Iceberg Table Format. Please set the `enable_iceberg_materializations` behavior flag to True in your dbt_project.yml. For more information, go to https://docs.getdbt.com/reference/resource-configs/snowflake-configs.') %} {%- endif %} {%- set materialization_prefix = relation.get_ddl_prefix_for_create(config.model.config, temporary) -%} From df643a0ed4bec6e0e961594d8171eb2c808d249b Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:20:45 -0700 Subject: [PATCH 2/4] Add snowflake telemetry. (#1209) * Add snowflake telemetry. * Add changelog. * Temporary dev branch switch. * Correct version import * bump ci * Temporary dev branch switch. * Temporary dev branch switch take 2. * Alter to meet new base schema changes. * Fix input args. * Change field name. --------- Co-authored-by: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> --- .../Under the Hood-20241016-035544.yaml | 6 +++++ dbt/adapters/snowflake/impl.py | 17 ++++++++++++ dev-requirements.txt | 2 +- setup.py | 2 +- tests/unit/test_adapter_telemetry.py | 27 +++++++++++++++++++ 5 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20241016-035544.yaml create mode 100644 tests/unit/test_adapter_telemetry.py diff --git a/.changes/unreleased/Under the Hood-20241016-035544.yaml b/.changes/unreleased/Under the Hood-20241016-035544.yaml new file mode 100644 index 000000000..59e4f70de --- /dev/null +++ b/.changes/unreleased/Under the Hood-20241016-035544.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add telemetry function +time: 2024-10-16T03:55:44.144174-07:00 +custom: + Author: versusfacit + Issue: "301" diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 6320893e1..89c21f531 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -4,6 +4,7 @@ from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport from dbt.adapters.base.meta import available from dbt.adapters.capability import CapabilityDict, CapabilitySupport, Support, Capability +from dbt.adapters.contracts.relation import RelationConfig from dbt.adapters.sql import SQLAdapter from dbt.adapters.sql.impl import ( LIST_SCHEMAS_MACRO_NAME, @@ -25,6 +26,7 @@ SnowflakeRelationType, TableFormat, ) + from dbt.adapters.snowflake import SnowflakeColumn from dbt.adapters.snowflake import SnowflakeConnectionManager from dbt.adapters.snowflake import SnowflakeRelation @@ -419,3 +421,18 @@ def valid_incremental_strategies(self): def debug_query(self): """Override for DebugTask method""" self.execute("select 1 as id") + + @classmethod + def _get_adapter_specific_run_info(cls, config: RelationConfig) -> Dict[str, Any]: + table_format: Optional[str] = None + if ( + config + and hasattr(config, "_extra") + and (relation_format := config._extra.get("table_format")) + ): + table_format = relation_format + + return { + "adapter_type": "snowflake", + "table_format": table_format, + } diff --git a/dev-requirements.txt b/dev-requirements.txt index 906003768..0ceb365db 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,6 @@ # install latest changes in dbt-core git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-adapters.git +git+https://github.com/dbt-labs/dbt-adapters.git@ADAP-301/add-adapter-telemetry git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter git+https://github.com/dbt-labs/dbt-common.git diff --git a/setup.py b/setup.py index c0716341d..fcea43309 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ def _plugin_version() -> str: include_package_data=True, install_requires=[ "dbt-common>=1.10,<2.0", - "dbt-adapters>=1.7,<2.0", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@ADAP-301/add-adapter-telemetry", "snowflake-connector-python[secure-local-storage]~=3.0", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", diff --git a/tests/unit/test_adapter_telemetry.py b/tests/unit/test_adapter_telemetry.py new file mode 100644 index 000000000..498676b77 --- /dev/null +++ b/tests/unit/test_adapter_telemetry.py @@ -0,0 +1,27 @@ +from unittest import mock + +import dbt.adapters.snowflake.__version__ + +from dbt.adapters.snowflake.impl import SnowflakeAdapter +from dbt.adapters.base.relation import AdapterTrackingRelationInfo + + +def test_telemetry_with_snowflake_details(): + mock_model_config = mock.MagicMock() + mock_model_config._extra = mock.MagicMock() + mock_model_config._extra = { + "adapter_type": "snowflake", + "table_format": "iceberg", + } + + res = SnowflakeAdapter.get_adapter_run_info(mock_model_config) + + assert res.adapter_name == "snowflake" + assert res.base_adapter_version == dbt.adapters.__about__.version + assert res.adapter_version == dbt.adapters.snowflake.__version__.version + assert res.model_adapter_details == { + "adapter_type": "snowflake", + "table_format": "iceberg", + } + + assert type(res) is AdapterTrackingRelationInfo From ad4ccadb0411826bd3053800431bbc7cfc9a78c2 Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Tue, 29 Oct 2024 07:50:49 -0700 Subject: [PATCH 3/4] Revert requirements. (#1225) Co-authored-by: VersusFacit --- dev-requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 0ceb365db..906003768 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,6 @@ # install latest changes in dbt-core git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-adapters.git@ADAP-301/add-adapter-telemetry +git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter git+https://github.com/dbt-labs/dbt-common.git diff --git a/setup.py b/setup.py index fcea43309..c0716341d 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ def _plugin_version() -> str: include_package_data=True, install_requires=[ "dbt-common>=1.10,<2.0", - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@ADAP-301/add-adapter-telemetry", + "dbt-adapters>=1.7,<2.0", "snowflake-connector-python[secure-local-storage]~=3.0", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", From 1d299923e34c96f2e96a5215ac196658f86ce1d1 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Tue, 29 Oct 2024 12:35:37 -0400 Subject: [PATCH 4/4] Add test cases for generic tests config (#1172) * add test cases for generic tests config * update tests --------- Co-authored-by: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Co-authored-by: Colin --- .../functional/generic_test_tests/__init__.py | 0 tests/functional/generic_test_tests/_files.py | 90 +++++++++++++++++++ .../functional/generic_test_tests/_models.py | 0 .../functional/generic_test_tests/_schemas.py | 0 .../generic_test_tests/test_generic_tests.py | 54 +++++++++++ 5 files changed, 144 insertions(+) create mode 100644 tests/functional/generic_test_tests/__init__.py create mode 100644 tests/functional/generic_test_tests/_files.py create mode 100644 tests/functional/generic_test_tests/_models.py create mode 100644 tests/functional/generic_test_tests/_schemas.py create mode 100644 tests/functional/generic_test_tests/test_generic_tests.py diff --git a/tests/functional/generic_test_tests/__init__.py b/tests/functional/generic_test_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional/generic_test_tests/_files.py b/tests/functional/generic_test_tests/_files.py new file mode 100644 index 000000000..a9743e43e --- /dev/null +++ b/tests/functional/generic_test_tests/_files.py @@ -0,0 +1,90 @@ +SCHEMA__CONTROL = """ +version: 2 +models: + - name: colors + columns: + - name: color + data_tests: + - not_null +""" + + +SCHEMA__EXPLICIT_WAREHOUSE = """ +version: 2 +models: + - name: colors + columns: + - name: color + data_tests: + - not_null: + config: + snowflake_warehouse: DBT_TESTING_ALT +""" + + +SCHEMA__NOT_NULL = """ +version: 2 +models: + - name: facts + columns: + - name: value + data_tests: + - not_null: + config: + snowflake_warehouse: DBT_TESTING_ALT +""" + + +SCHEMA__RELATIONSHIPS = """ +version: 2 +models: + - name: facts + columns: + - name: color + data_tests: + - relationships: + to: ref('my_colors') + field: color + config: + snowflake_warehouse: DBT_TESTING_ALT +""" + + +SCHEMA__ACCEPTED_VALUES = """ +version: 2 +models: + - name: colors + columns: + - name: color + data_tests: + - accepted_values: + values: ['blue', 'red', 'green'] + config: + snowflake_warehouse: DBT_TESTING_ALT +""" + + +SEED__COLORS = """ +color +blue +green +red +yellow +""".strip() + + +# record 10 is missing a value +# record 7 has a color that's not on COLORS +SEED__FACTS = """ +id,color,value +1,blue,10 +2,red,20 +3,green,30 +4,yellow,40 +5,blue,50 +6,red,60 +7,orange,70 +8,green,80 +9,yellow,90 +10,blue, +""".strip() diff --git a/tests/functional/generic_test_tests/_models.py b/tests/functional/generic_test_tests/_models.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional/generic_test_tests/_schemas.py b/tests/functional/generic_test_tests/_schemas.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional/generic_test_tests/test_generic_tests.py b/tests/functional/generic_test_tests/test_generic_tests.py new file mode 100644 index 000000000..a653a363b --- /dev/null +++ b/tests/functional/generic_test_tests/test_generic_tests.py @@ -0,0 +1,54 @@ +import pytest + +from dbt.tests.util import run_dbt, run_dbt_and_capture + +from tests.functional.generic_test_tests import _files + + +class TestWarehouseConfig: + + @pytest.fixture(scope="class") + def seeds(self): + return { + "colors.csv": _files.SEED__COLORS, + "facts.csv": _files.SEED__FACTS, + } + + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + yield + + +class TestWarehouseConfigControl(TestWarehouseConfig): + + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": _files.SCHEMA__CONTROL} + + def test_expected_warehouse(self, project): + results, logs = run_dbt_and_capture(["test"]) + assert len(results) == 1 + + +class TestWarehouseConfigExplicitWarehouse(TestWarehouseConfig): + + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": _files.SCHEMA__EXPLICIT_WAREHOUSE} + + def test_expected_warehouse(self, project): + _, logs = run_dbt_and_capture(["test", "--log-level", "debug"]) + assert "use warehouse " in logs + + +class TestWarehouseConfigNotNull(TestWarehouseConfig): + + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": _files.SCHEMA__NOT_NULL} + + def test_expected_warehouse(self, project): + _, logs = run_dbt_and_capture(["test", "--log-level", "debug"], expect_pass=False) + assert "use warehouse " in logs