From 3a9d7ced2ca972891030d7aa8e07fa2d062ed311 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 25 Sep 2024 14:29:37 +0100 Subject: [PATCH] improve partition_by missing error message --- .../macros/materializations/incremental/strategies.sql | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dbt/include/spark/macros/materializations/incremental/strategies.sql b/dbt/include/spark/macros/materializations/incremental/strategies.sql index bbb3a571b..28ba8284e 100644 --- a/dbt/include/spark/macros/materializations/incremental/strategies.sql +++ b/dbt/include/spark/macros/materializations/incremental/strategies.sql @@ -77,8 +77,14 @@ {{ get_insert_overwrite_sql(source, target, existing) }} {%- elif strategy == 'microbatch' -%} {#-- microbatch wraps insert_overwrite, and requires a partition_by config #} + {% set missing_partition_key_microbatch_msg -%} + dbt-spark 'microbatch' incremental strategy requires a `partition_by` config. + Ensure you are using a `partition_by` column that is of grain {{ config.get('batch_size') }} + for microbatch model {{ model.name }}. + {%- endset %} + {%- if not config.get('partition_by') -%} - {{ exceptions.raise_compiler_error("dbt-spark 'microbatch' requires a `partition_by` config") }} + {{ exceptions.raise_compiler_error(missing_partition_key_microbatch_msg) }} {%- endif -%} {{ get_insert_overwrite_sql(source, target, existing) }} {%- elif strategy == 'merge' -%}