From 8efa58b492055f8ca9aee8de440a685691a2098f Mon Sep 17 00:00:00 2001 From: oravi Date: Thu, 10 Mar 2022 19:31:55 +0200 Subject: [PATCH 1/4] Updated packages.yml --- monitor/dbt_project/packages.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitor/dbt_project/packages.yml b/monitor/dbt_project/packages.yml index 1e43973f0..8c4f7d4b3 100644 --- a/monitor/dbt_project/packages.yml +++ b/monitor/dbt_project/packages.yml @@ -1,3 +1,3 @@ packages: - - git: git@github.com:elementary-data/dbt-data-reliability.git - revision: 0.2.0 + - package: elementary-data/elementary_data_reliability + version: 0.2.0 From 328f6b41c9146589cb58ecbde4b47c33a615e314 Mon Sep 17 00:00:00 2001 From: oravi Date: Thu, 10 Mar 2022 19:38:49 +0200 Subject: [PATCH 2/4] Small cosmetics and bumped version --- config/config.py | 12 ++++++++++-- setup.py | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/config/config.py b/config/config.py index c43e589b9..6089252a9 100644 --- a/config/config.py +++ b/config/config.py @@ -49,11 +49,19 @@ def anonymous_tracking_enabled(self) -> bool: @property def slack_notification_webhook(self) -> Union[str, None]: - return self.config_dict.get(self.SLACK).get(self.NOTIFICATION_WEBHOOK) if self.config_dict.get(self.SLACK) else None + slack_config = self.config_dict.get(self.SLACK) + if slack_config is not None: + return slack_config.get(self.NOTIFICATION_WEBHOOK) + return None @property def is_slack_workflow(self) -> bool: - return True if self.config_dict.get(self.SLACK) and self.config_dict.get(self.SLACK).get(self.WORKFLOWS) else False + slack_config = self.config_dict.get(self.SLACK) + if slack_config is not None: + workflows = slack_config.get(self.WORKFLOWS) + if workflows is True: + return True + return False @property def target_dir(self) -> str: diff --git a/setup.py b/setup.py index 68e036251..2f206fbe2 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name='elementary-data', description='Data monitoring and lineage', - version='0.2.0', + version='0.2.1', packages=find_packages(), python_requires='>=3.6.2', entry_points=''' From b6e62928697ec62190ef534323e1d8871c313769 Mon Sep 17 00:00:00 2001 From: oravi Date: Thu, 10 Mar 2022 19:49:17 +0200 Subject: [PATCH 3/4] Fixed tests --- tests/monitor/test_data_monitoring.py | 107 +++++--------------------- 1 file changed, 20 insertions(+), 87 deletions(-) diff --git a/tests/monitor/test_data_monitoring.py b/tests/monitor/test_data_monitoring.py index 0909f5d05..3f68701d8 100644 --- a/tests/monitor/test_data_monitoring.py +++ b/tests/monitor/test_data_monitoring.py @@ -103,19 +103,6 @@ def execute_query_side_effect(*args, **kwargs): return snowflake_data_mon -def assert_configuration_exists(data_monitoring): - assert os.path.exists(data_monitoring.DBT_PROJECT_SEEDS_PATH) - monitoring_config_csv_path = os.path.join(data_monitoring.DBT_PROJECT_SEEDS_PATH, - f'{data_monitoring.MONITORING_CONFIGURATION}.csv') - config_mock = data_monitoring.config - config_mock.monitoring_configuration_in_dbt_sources_to_csv.assert_called_once_with(monitoring_config_csv_path) - - -def delete_configuration(data_monitoring): - if os.path.exists(data_monitoring.DBT_PROJECT_SEEDS_PATH): - shutil.rmtree(data_monitoring.DBT_PROJECT_SEEDS_PATH) - - def delete_dbt_package(data_monitoring): if os.path.exists(data_monitoring.DBT_PROJECT_MODULES_PATH): shutil.rmtree(data_monitoring.DBT_PROJECT_MODULES_PATH) @@ -124,93 +111,39 @@ def delete_dbt_package(data_monitoring): shutil.rmtree(data_monitoring.DBT_PROJECT_PACKAGES_PATH) -@pytest.mark.parametrize("full_refresh, update_dbt_package, reload_config, dbt_package_exists", [ - (True, True, False, False), - (True, False, False, False), - (True, True, True, False), - (True, False, True, False), - (True, True, False, True), - (True, False, False, True), - (True, True, True, True), - (True, False, True, True), - (False, True, False, False), - (False, False, False, False), - (False, True, True, False), - (False, False, True, False), - (False, True, False, True), - (False, False, False, True), - (False, True, True, True), - (False, False, True, True), -]) -def test_data_monitoring_run_config_does_not_exist(full_refresh, update_dbt_package, reload_config, dbt_package_exists, - snowflake_data_monitoring_with_empty_config_in_db): - snowflake_data_monitoring = snowflake_data_monitoring_with_empty_config_in_db - delete_configuration(snowflake_data_monitoring) - delete_dbt_package(snowflake_data_monitoring) - snowflake_data_monitoring._dbt_package_exists = lambda: dbt_package_exists - dbt_runner_mock = snowflake_data_monitoring.dbt_runner - # The test function - snowflake_data_monitoring.run(dbt_full_refresh=full_refresh, force_update_dbt_package=update_dbt_package, - reload_monitoring_configuration=reload_config) - - if update_dbt_package or not dbt_package_exists: - dbt_runner_mock.deps.assert_called() - else: - dbt_runner_mock.deps.assert_not_called() - - # Validate configuration exists in the dbt_project seed path - assert_configuration_exists(snowflake_data_monitoring) - dbt_runner_mock.seed.assert_called() - - # Validate that snapshot and run were called as well - dbt_runner_mock.snapshot.assert_called() - dbt_runner_mock.run.assert_called() - - -@pytest.mark.parametrize("full_refresh, update_dbt_package, reload_config, dbt_package_exists", [ - (True, True, False, False), - (True, False, False, False), - (True, True, True, False), - (True, False, True, False), - (True, True, False, True), - (True, False, False, True), - (True, True, True, True), - (True, False, True, True), - (False, True, False, False), - (False, False, False, False), - (False, True, True, False), - (False, False, True, False), - (False, True, False, True), - (False, False, False, True), - (False, True, True, True), - (False, False, True, True), +@pytest.mark.parametrize("full_refresh, update_dbt_package, dbt_package_exists", [ + (True, True, False), + (True, False, False), + (True, True, False), + (True, False, False), + (True, True, True), + (True, False, True), + (True, True, True), + (True, False, True), + (False, True, False), + (False, False, False), + (False, True, False), + (False, False, False), + (False, True, True), + (False, False, True), + (False, True, True), + (False, False, True), ]) -def test_data_monitoring_run(full_refresh, update_dbt_package, reload_config, dbt_package_exists, - snowflake_data_monitoring): +def test_data_monitoring_run(full_refresh, update_dbt_package, dbt_package_exists, snowflake_data_monitoring): delete_dbt_package(snowflake_data_monitoring) - delete_configuration(snowflake_data_monitoring) snowflake_data_monitoring._dbt_package_exists = lambda: dbt_package_exists dbt_runner_mock = snowflake_data_monitoring.dbt_runner # The test function - snowflake_data_monitoring.run(dbt_full_refresh=full_refresh, force_update_dbt_package=update_dbt_package, - reload_monitoring_configuration=reload_config) + snowflake_data_monitoring.run(dbt_full_refresh=full_refresh, force_update_dbt_package=update_dbt_package) if update_dbt_package or not dbt_package_exists: dbt_runner_mock.deps.assert_called() else: dbt_runner_mock.deps.assert_not_called() - if reload_config: - assert_configuration_exists(snowflake_data_monitoring) - dbt_runner_mock.seed.assert_called() - else: - dbt_runner_mock.seed.assert_not_called() - - # Validate that dbt snapshot and dbt run were called as well - dbt_runner_mock.snapshot.assert_called() - dbt_runner_mock.run.assert_called_with(model=snowflake_data_monitoring.DBT_PACKAGE_NAME, full_refresh=full_refresh) + dbt_runner_mock.run.assert_called_with(select=snowflake_data_monitoring.DBT_PACKAGE_NAME, full_refresh=full_refresh) @mock.patch('requests.post') From 4f9e05e5e3efeceb4dd48cb09e268f3a4a58a706 Mon Sep 17 00:00:00 2001 From: oravi Date: Thu, 10 Mar 2022 20:07:49 +0200 Subject: [PATCH 4/4] Updated package version --- monitor/dbt_project/packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitor/dbt_project/packages.yml b/monitor/dbt_project/packages.yml index 8c4f7d4b3..0a6ecd9fc 100644 --- a/monitor/dbt_project/packages.yml +++ b/monitor/dbt_project/packages.yml @@ -1,3 +1,3 @@ packages: - - package: elementary-data/elementary_data_reliability + - package: elementary-data/elementary version: 0.2.0