From ec11a97f0cdda7bbdf91808844274d82aa6d2dba Mon Sep 17 00:00:00 2001 From: Danny T <55750990+dtger@users.noreply.github.com> Date: Tue, 5 Sep 2023 15:07:55 +0100 Subject: [PATCH 1/3] add row-on-row pct change test --- ...olumn_value_running_abs_change_ratio_to_be | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 macros/schema_tests/column_values_basic/expect_column_value_running_abs_change_ratio_to_be diff --git a/macros/schema_tests/column_values_basic/expect_column_value_running_abs_change_ratio_to_be b/macros/schema_tests/column_values_basic/expect_column_value_running_abs_change_ratio_to_be new file mode 100644 index 0000000..4b42daf --- /dev/null +++ b/macros/schema_tests/column_values_basic/expect_column_value_running_abs_change_ratio_to_be @@ -0,0 +1,63 @@ +{% test expect_column_value_running_abs_change_ratio_to_be(model, column_name,change_ratio, + sort_column=None, + row_condition=None, + strictly=True, + group_by=None) %} + +{%- set sort_column = column_name if not sort_column else sort_column -%} +{%- set operator = "<" if strictly else "<=" %} + +with all_values as ( + + select + {{ sort_column }}, + {%- if group_by -%} + {{ group_by | join(", ") }}, + {%- endif %} + {{ column_name }} + from {{ model }} + {% if row_condition %} + where {{ row_condition }} + {% endif %} + +), +add_lag_values as ( + + select + {{ group_by | join(", ") }} + , {{ sort_column }} as current_{{ sort_column | trim }} + , lag( {{- sort_column -}} ) over + {%- if not group_by -%} + (order by sort_column) + {%- else -%} + (partition by {{ group_by | join(", ") }} order by {{ sort_column }}) + {%- endif %} as previous_{{ sort_column | trim }} + , {{ column_name }} as current_{{ column_name | trim }} + , lag( {{- column_name -}} ) over + {%- if not group_by -%} + (order by sort_column) + {%- else -%} + (partition by {{ group_by | join(", ") }} order by {{ sort_column }}) + {%- endif %} as previous_{{ column_name | trim}} + from + all_values + +), +validation_errors as ( + + select + * + , current_{{ column_name | trim}} / nullif(previous_{{ column_name | trim}}, 0) - 1 as actual_change_ratio + , abs(current_{{ column_name | trim}} / nullif(previous_{{ column_name | trim}}, 0)) - 1 as abs_change_ratio + , '{{ operator }} {{ change_ratio }}' as criteria + from + add_lag_values + where + previous_{{ column_name | trim}} is not null + and + not ((abs(current_{{ column_name | trim}} / previous_{{ column_name | trim}} )-1) {{ operator }} {{ change_ratio }} ) + +) +select * +from validation_errors +{% endtest %} From a934592b5336dfa49706718be3cfd2be596e6d42 Mon Sep 17 00:00:00 2001 From: Danny T <55750990+dtger@users.noreply.github.com> Date: Tue, 5 Sep 2023 15:22:52 +0100 Subject: [PATCH 2/3] clarify change threshold parameter --- .../expect_column_value_running_abs_change_ratio_to_be | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/macros/schema_tests/column_values_basic/expect_column_value_running_abs_change_ratio_to_be b/macros/schema_tests/column_values_basic/expect_column_value_running_abs_change_ratio_to_be index 4b42daf..f843a87 100644 --- a/macros/schema_tests/column_values_basic/expect_column_value_running_abs_change_ratio_to_be +++ b/macros/schema_tests/column_values_basic/expect_column_value_running_abs_change_ratio_to_be @@ -1,4 +1,4 @@ -{% test expect_column_value_running_abs_change_ratio_to_be(model, column_name,change_ratio, +{% test expect_column_value_running_abs_change_ratio_to_be(model, column_name,change_threshold, sort_column=None, row_condition=None, strictly=True, @@ -49,13 +49,13 @@ validation_errors as ( * , current_{{ column_name | trim}} / nullif(previous_{{ column_name | trim}}, 0) - 1 as actual_change_ratio , abs(current_{{ column_name | trim}} / nullif(previous_{{ column_name | trim}}, 0)) - 1 as abs_change_ratio - , '{{ operator }} {{ change_ratio }}' as criteria + , '{{ operator }} {{ change_threshold }}' as criteria from add_lag_values where previous_{{ column_name | trim}} is not null and - not ((abs(current_{{ column_name | trim}} / previous_{{ column_name | trim}} )-1) {{ operator }} {{ change_ratio }} ) + not ((abs(current_{{ column_name | trim}} / previous_{{ column_name | trim}} )-1) {{ operator }} {{ change_threshold }} ) ) select * From c51f086601dd69bbc9788f23383bd68b59e25d82 Mon Sep 17 00:00:00 2001 From: Danny T <55750990+dtger@users.noreply.github.com> Date: Tue, 5 Sep 2023 15:27:07 +0100 Subject: [PATCH 3/3] add nullif to fix potential div0 errors --- .../expect_column_value_running_abs_change_ratio_to_be | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/schema_tests/column_values_basic/expect_column_value_running_abs_change_ratio_to_be b/macros/schema_tests/column_values_basic/expect_column_value_running_abs_change_ratio_to_be index f843a87..360a2d0 100644 --- a/macros/schema_tests/column_values_basic/expect_column_value_running_abs_change_ratio_to_be +++ b/macros/schema_tests/column_values_basic/expect_column_value_running_abs_change_ratio_to_be @@ -55,7 +55,7 @@ validation_errors as ( where previous_{{ column_name | trim}} is not null and - not ((abs(current_{{ column_name | trim}} / previous_{{ column_name | trim}} )-1) {{ operator }} {{ change_threshold }} ) + not ((abs(current_{{ column_name | trim}} / nullif(previous_{{ column_name | trim}}, 0) )-1) {{ operator }} {{ change_threshold }} ) ) select *