From 165507a4e6a413ca4949bf7221c88418f2a0cb31 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Tue, 24 Sep 2024 17:10:05 +0100 Subject: [PATCH] fix schema.yml config validation for datetime --- core/dbt/artifacts/resources/v1/config.py | 3 +- .../test_microbatch_config_validation.py | 51 ++++++++++++++++--- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/core/dbt/artifacts/resources/v1/config.py b/core/dbt/artifacts/resources/v1/config.py index 5a7977c0958..79529aa9e99 100644 --- a/core/dbt/artifacts/resources/v1/config.py +++ b/core/dbt/artifacts/resources/v1/config.py @@ -1,5 +1,6 @@ import re from dataclasses import dataclass, field +from datetime import datetime from typing import Any, Dict, List, Optional, Union from mashumaro.jsonschema.annotations import Pattern @@ -82,7 +83,7 @@ class NodeConfig(NodeAndTestConfig): incremental_strategy: Optional[str] = None batch_size: Any = None lookback: Any = 0 - begin: Any = None + begin: Union[datetime, Any] = None persist_docs: Dict[str, Any] = field(default_factory=dict) post_hook: List[Hook] = field( default_factory=list, diff --git a/tests/functional/microbatch/test_microbatch_config_validation.py b/tests/functional/microbatch/test_microbatch_config_validation.py index 86114d529b6..6a27eb97fad 100644 --- a/tests/functional/microbatch/test_microbatch_config_validation.py +++ b/tests/functional/microbatch/test_microbatch_config_validation.py @@ -11,6 +11,21 @@ select * from {{ ref('input_model') }} """ +valid_microbatch_model_no_config_sql = """ +select * from {{ ref('input_model') }} +""" + +valid_microbatch_model_config_yml = """ +models: + - name: microbatch + config: + materialized: incremental + incremental_strategy: microbatch + batch_size: day + event_time: event_time + begin: 2020-01-01 +""" + missing_event_time_microbatch_model_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', batch_size='day') }} select * from {{ ref('input_model') }} @@ -55,7 +70,7 @@ """ -class BaseMicrobatchTest: +class BaseMicrobatchTestParseError: @pytest.fixture(scope="class") def models(self): return {} @@ -66,7 +81,17 @@ def test_parsing_error_raised(self, project): run_dbt(["parse"]) -class TestMissingEventTimeMicrobatch(BaseMicrobatchTest): +class BaseMicrobatchTestNoError: + @pytest.fixture(scope="class") + def models(self): + return {} + + @mock.patch.dict(os.environ, {"DBT_EXPERIMENTAL_MICROBATCH": "True"}) + def test_parsing_error_not_raised(self, project): + run_dbt(["parse"]) + + +class TestMissingEventTimeMicrobatch(BaseMicrobatchTestParseError): @pytest.fixture(scope="class") def models(self): return { @@ -75,7 +100,7 @@ def models(self): } -class TestInvalidEventTimeMicrobatch(BaseMicrobatchTest): +class TestInvalidEventTimeMicrobatch(BaseMicrobatchTestParseError): @pytest.fixture(scope="class") def models(self): return { @@ -84,7 +109,7 @@ def models(self): } -class TestMissingBeginMicrobatch(BaseMicrobatchTest): +class TestMissingBeginMicrobatch(BaseMicrobatchTestParseError): @pytest.fixture(scope="class") def models(self): return { @@ -93,7 +118,7 @@ def models(self): } -class TestInvaliBeginMicrobatch(BaseMicrobatchTest): +class TestInvaliBeginMicrobatch(BaseMicrobatchTestParseError): @pytest.fixture(scope="class") def models(self): return { @@ -102,7 +127,7 @@ def models(self): } -class TestMissingBatchSizeMicrobatch(BaseMicrobatchTest): +class TestMissingBatchSizeMicrobatch(BaseMicrobatchTestParseError): @pytest.fixture(scope="class") def models(self): return { @@ -111,7 +136,7 @@ def models(self): } -class TestInvalidBatchSizeMicrobatch(BaseMicrobatchTest): +class TestInvalidBatchSizeMicrobatch(BaseMicrobatchTestParseError): @pytest.fixture(scope="class") def models(self): return { @@ -120,10 +145,20 @@ def models(self): } -class TestInvalidInputEventTimeMicrobatch(BaseMicrobatchTest): +class TestInvalidInputEventTimeMicrobatch(BaseMicrobatchTestParseError): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": invalid_event_time_input_model_sql, "microbatch.sql": valid_microbatch_model_sql, } + + +class TestValidBeginMicrobatch(BaseMicrobatchTestNoError): + @pytest.fixture(scope="class") + def models(self): + return { + "input_model.sql": valid_input_model_sql, + "microbatch.sql": valid_microbatch_model_no_config_sql, + "schema.yml": valid_microbatch_model_config_yml, + }