From f1a7b3f3adc28d2c6819460c2273fe2df7240da6 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Tue, 12 Nov 2024 15:06:36 -0500 Subject: [PATCH 1/7] fix deprecation firing for microbatch model w custom strategy --- core/dbt/parser/manifest.py | 3 +- .../functional/microbatch/test_microbatch.py | 28 +++++++++---------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 6743bf5e6e7..a57e1fe6f5c 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -662,9 +662,10 @@ def check_for_microbatch_deprecations(self) -> None: has_microbatch_model = True break - if has_microbatch_model and self.manifest._microbatch_macro_is_core( + if has_microbatch_model and not self.manifest._microbatch_macro_is_core( self.root_project.project_name ): + breakpoint() dbt.deprecations.warn("microbatch-macro-outside-of-batches-deprecation") def load_and_parse_macros(self, project_parser_files): diff --git a/tests/functional/microbatch/test_microbatch.py b/tests/functional/microbatch/test_microbatch.py index b98ad38caa9..ee1b5ccd762 100644 --- a/tests/functional/microbatch/test_microbatch.py +++ b/tests/functional/microbatch/test_microbatch.py @@ -182,19 +182,16 @@ def test_use_custom_microbatch_strategy_by_default( project, deprecation_catcher: EventCatcher, ): - with mock.patch.object( - type(project.adapter), "valid_incremental_strategies", lambda _: [] - ): - # Initial run - run_dbt(["run"]) + # Initial run fires deprecation + run_dbt(["run"], callbacks=[deprecation_catcher.catch]) + # Deprecation warning about custom microbatch macro fired + assert len(deprecation_catcher.caught_events) == 1 - # Incremental run uses custom strategy - _, logs = run_dbt_and_capture(["run"]) - assert "custom microbatch strategy" in logs - # The custom strategy wasn't used with batch functionality - assert "START batch" not in logs - # Deprecation warning about custom microbatch macro fired - assert len(deprecation_catcher.caught_events) == 0 + # Incremental run uses custom strategy + _, logs = run_dbt_and_capture(["run"]) + assert "custom microbatch strategy" in logs + # The custom strategy wasn't used with batch functionality + assert "START batch" not in logs class TestMicrobatchCustomUserStrategyProjectFlagTrueValid(BaseMicrobatchCustomUserStrategy): @@ -208,7 +205,9 @@ def test_use_custom_microbatch_strategy_project_flag_true_invalid_incremental_st ): # Initial run with patch_microbatch_end_time("2020-01-03 13:57:00"): - run_dbt(["run"]) + run_dbt(["run"], callbacks=[deprecation_catcher.catch]) + # Deprecation warning about custom microbatch macro not fired + assert len(deprecation_catcher.caught_events) == 0 # Incremental run uses custom strategy with patch_microbatch_end_time("2020-01-03 13:57:00"): @@ -216,10 +215,9 @@ def test_use_custom_microbatch_strategy_project_flag_true_invalid_incremental_st assert "custom microbatch strategy" in logs # The custom strategy was used with batch functionality assert "START batch" in logs - # Deprecation warning about custom microbatch macro not fired - assert len(deprecation_catcher.caught_events) == 0 +# TODO: fix this test once adapter behaviour flag gating is fixed class TestMicrobatchCustomUserStrategyProjectFlagTrueInvalid(BaseMicrobatchCustomUserStrategy): def test_use_custom_microbatch_strategy_project_flag_true_invalid_incremental_strategy( self, project From 72ca6bf838c61e5085def05488264b6fe2ba73ab Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Tue, 12 Nov 2024 15:14:06 -0500 Subject: [PATCH 2/7] fix TestMicrobatchCustomUserStrategyProjectFlagTrueInvalid --- dev-requirements.txt | 2 +- tests/functional/microbatch/test_microbatch.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 20605e632b8..4f0a8f68fa8 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,4 +1,4 @@ -git+https://github.com/dbt-labs/dbt-adapters.git@main +git+https://github.com/dbt-labs/dbt-adapters.git@fix-microbatch-behavior-flag-conditional git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-tests-adapter git+https://github.com/dbt-labs/dbt-common.git@main git+https://github.com/dbt-labs/dbt-postgres.git@main diff --git a/tests/functional/microbatch/test_microbatch.py b/tests/functional/microbatch/test_microbatch.py index ee1b5ccd762..9667733de64 100644 --- a/tests/functional/microbatch/test_microbatch.py +++ b/tests/functional/microbatch/test_microbatch.py @@ -217,7 +217,6 @@ def test_use_custom_microbatch_strategy_project_flag_true_invalid_incremental_st assert "START batch" in logs -# TODO: fix this test once adapter behaviour flag gating is fixed class TestMicrobatchCustomUserStrategyProjectFlagTrueInvalid(BaseMicrobatchCustomUserStrategy): def test_use_custom_microbatch_strategy_project_flag_true_invalid_incremental_strategy( self, project @@ -229,7 +228,7 @@ def test_use_custom_microbatch_strategy_project_flag_true_invalid_incremental_st # microbatch strategy causes an error to be raised with patch_microbatch_end_time("2020-01-03 13:57:00"): _, logs = run_dbt_and_capture(["run"], expect_pass=False) - assert "'microbatch' is not valid" in logs + assert "'microbatch' is not valid" not in logs class BaseMicrobatchTest: From 33b0183dec5f5cf79962c50630910ccda11d795b Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Tue, 12 Nov 2024 15:14:11 -0500 Subject: [PATCH 3/7] fix TestMicrobatchCustomUserStrategyProjectFlagTrueInvalid --- tests/functional/microbatch/test_microbatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/microbatch/test_microbatch.py b/tests/functional/microbatch/test_microbatch.py index 9667733de64..a465c7c25b7 100644 --- a/tests/functional/microbatch/test_microbatch.py +++ b/tests/functional/microbatch/test_microbatch.py @@ -225,7 +225,7 @@ def test_use_custom_microbatch_strategy_project_flag_true_invalid_incremental_st type(project.adapter), "valid_incremental_strategies", lambda _: [] ): # Run of microbatch model while adapter doesn't have a "valid" - # microbatch strategy causes an error to be raised + # microbatch strategy causes no error raised when behaviour flag set to true with patch_microbatch_end_time("2020-01-03 13:57:00"): _, logs = run_dbt_and_capture(["run"], expect_pass=False) assert "'microbatch' is not valid" not in logs From 0679f3eb6813c3bf95df73db3d094c354a7bd9e9 Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Tue, 12 Nov 2024 20:58:53 -0600 Subject: [PATCH 4/7] Drop breakpoint --- core/dbt/parser/manifest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index a57e1fe6f5c..fbfc83cd6c5 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -665,7 +665,6 @@ def check_for_microbatch_deprecations(self) -> None: if has_microbatch_model and not self.manifest._microbatch_macro_is_core( self.root_project.project_name ): - breakpoint() dbt.deprecations.warn("microbatch-macro-outside-of-batches-deprecation") def load_and_parse_macros(self, project_parser_files): From a188b65a2ade0617edc74c43def4132043cff10b Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Tue, 12 Nov 2024 20:59:56 -0600 Subject: [PATCH 5/7] Fix TestMicrobatchCustomUserStrategyProjectFlagTrueInvalid --- tests/functional/microbatch/test_microbatch.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/functional/microbatch/test_microbatch.py b/tests/functional/microbatch/test_microbatch.py index a465c7c25b7..3de48225f44 100644 --- a/tests/functional/microbatch/test_microbatch.py +++ b/tests/functional/microbatch/test_microbatch.py @@ -217,7 +217,9 @@ def test_use_custom_microbatch_strategy_project_flag_true_invalid_incremental_st assert "START batch" in logs -class TestMicrobatchCustomUserStrategyProjectFlagTrueInvalid(BaseMicrobatchCustomUserStrategy): +class TestMicrobatchCustomUserStrategyProjectFlagTrueNoValidBuiltin( + BaseMicrobatchCustomUserStrategy +): def test_use_custom_microbatch_strategy_project_flag_true_invalid_incremental_strategy( self, project ): @@ -225,10 +227,15 @@ def test_use_custom_microbatch_strategy_project_flag_true_invalid_incremental_st type(project.adapter), "valid_incremental_strategies", lambda _: [] ): # Run of microbatch model while adapter doesn't have a "valid" - # microbatch strategy causes no error raised when behaviour flag set to true + # microbatch strategy causes no error when behaviour flag set to true + # and there is a custom microbatch macro with patch_microbatch_end_time("2020-01-03 13:57:00"): - _, logs = run_dbt_and_capture(["run"], expect_pass=False) + _, logs = run_dbt_and_capture(["run"]) assert "'microbatch' is not valid" not in logs + assert ( + "The use of a custom microbatch macro outside of batched execution is deprecated" + not in logs + ) class BaseMicrobatchTest: From bd087056c3f43e9ea1af1dfffd913eef91c355dc Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Tue, 12 Nov 2024 21:09:16 -0600 Subject: [PATCH 6/7] Add changie doc --- .changes/unreleased/Fixes-20241112-210839.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Fixes-20241112-210839.yaml diff --git a/.changes/unreleased/Fixes-20241112-210839.yaml b/.changes/unreleased/Fixes-20241112-210839.yaml new file mode 100644 index 00000000000..031e160f092 --- /dev/null +++ b/.changes/unreleased/Fixes-20241112-210839.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Correct when custom microbatch macro deprecation warning is fired +time: 2024-11-12T21:08:39.866837-06:00 +custom: + Author: QMalcolm MichelleArk + Issue: "10994" From 82a093a2a4cd8d1bed265cf451b08b90a0a7e136 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 13 Nov 2024 13:19:06 -0500 Subject: [PATCH 7/7] restore dev-requirements for dbt-adapters@main --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 4f0a8f68fa8..20605e632b8 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,4 +1,4 @@ -git+https://github.com/dbt-labs/dbt-adapters.git@fix-microbatch-behavior-flag-conditional +git+https://github.com/dbt-labs/dbt-adapters.git@main git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-tests-adapter git+https://github.com/dbt-labs/dbt-common.git@main git+https://github.com/dbt-labs/dbt-postgres.git@main