From c36005b327db7a085caaf68f7152c4765182310e Mon Sep 17 00:00:00 2001
From: rachelannelise <annelise.lynch@gmail.com>
Date: Thu, 12 Nov 2020 17:05:07 -0700
Subject: [PATCH] small updates and error handling

---
 .gitignore                                   |  3 +++
 dbt/redshift/.gitignore                      |  4 ----
 dbt/redshift/analysis/.gitkeep               |  0
 dbt/redshift/data/.gitkeep                   |  0
 dbt/redshift/dbt_project.yml                 |  4 ++--
 dbt/redshift/macros/.gitkeep                 |  0
 dbt/redshift/redshift_etl.py                 | 23 ++++++++++++++++++--
 dbt/redshift/settings/merge_settings_yaml.py |  5 +++--
 dbt/redshift/tests/.gitkeep                  |  0
 parsely_raw_data/__init__.py                 |  2 +-
 10 files changed, 30 insertions(+), 11 deletions(-)
 delete mode 100644 dbt/redshift/.gitignore
 delete mode 100644 dbt/redshift/analysis/.gitkeep
 delete mode 100644 dbt/redshift/data/.gitkeep
 delete mode 100644 dbt/redshift/macros/.gitkeep
 delete mode 100644 dbt/redshift/tests/.gitkeep

diff --git a/.gitignore b/.gitignore
index 015433b..a8187c3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,6 @@
 build/*
 dist/*
 .idea/*
+target/
+dbt_modules/
+logs/
diff --git a/dbt/redshift/.gitignore b/dbt/redshift/.gitignore
deleted file mode 100644
index dad33a4..0000000
--- a/dbt/redshift/.gitignore
+++ /dev/null
@@ -1,4 +0,0 @@
-
-target/
-dbt_modules/
-logs/
diff --git a/dbt/redshift/analysis/.gitkeep b/dbt/redshift/analysis/.gitkeep
deleted file mode 100644
index e69de29..0000000
diff --git a/dbt/redshift/data/.gitkeep b/dbt/redshift/data/.gitkeep
deleted file mode 100644
index e69de29..0000000
diff --git a/dbt/redshift/dbt_project.yml b/dbt/redshift/dbt_project.yml
index 353bde0..3e801a0 100644
--- a/dbt/redshift/dbt_project.yml
+++ b/dbt/redshift/dbt_project.yml
@@ -18,7 +18,7 @@
 - "{% if var('etl:keep_rawdata') == true %} select 1 {% else %} truncate table {{\
   \ target.schema }}.parsely_rawdata {% endif %}"
 - "truncate table {{var('parsely:events')}}"
-"profile": "parsely_dwh"
+"profile": "parsely-dwh"
 "source-paths":
 - "models"
 "target-path": "target"
@@ -36,6 +36,6 @@
   "custom:videoskimtime": "15"
   "etl:keep_rawdata": "True"
   "parsely:actions": "('pageview', 'heartbeat', 'videostart', 'vheartbeat')"
-  "parsely:events": "demo_rawdata"
+  "parsely:events": "parsely_rawdata"
   "parsely:timezone": "'America/New_York'"
 "version": "0.0.1"
diff --git a/dbt/redshift/macros/.gitkeep b/dbt/redshift/macros/.gitkeep
deleted file mode 100644
index e69de29..0000000
diff --git a/dbt/redshift/redshift_etl.py b/dbt/redshift/redshift_etl.py
index 223fde4..3f0027f 100644
--- a/dbt/redshift/redshift_etl.py
+++ b/dbt/redshift/redshift_etl.py
@@ -7,7 +7,22 @@
 
 from parsely_raw_data import redshift as parsely_redshift
 from parsely_raw_data import utils as parsely_utils
-from dbt.redshift.settings.default import *
+from dbt.redshift.settings.default import (
+    DBT_PROFILE_LOCATION,
+    DBT_PROFILE_TARGET_NAME,
+    ETL_END_DATE,
+    ETL_KEEP_RAW_DATA,
+    ETL_START_DATE,
+    PARSELY_RAW_DATA_TABLE,
+    REDSHIFT_DATABASE,
+    REDSHIFT_HOST,
+    REDSHIFT_PASSWORD,
+    REDSHIFT_PORT,
+    REDSHIFT_USER,
+    S3_AWS_ACCESS_KEY_ID,
+    S3_AWS_SECRET_ACCESS_KEY,
+    S3_NETWORK_NAME,
+)
 from dbt.redshift.settings.merge_settings_yaml import migrate_settings
 
 SETTINGS_ARG_MAPPING = {
@@ -86,6 +101,9 @@ def main():
     settings_migration = migrate_settings()
     if not settings_migration:
         logging.warning("Settings not copied to dbt_profiles.yml successfully.")
+        raise Exception("Settings not copied to dbt_profiles.yml successfully. Please edit default.py or copy the"
+                        "original default.py.schema as default.py and edit carefully. Be mindful of single quotes"
+                        "and double quotes.")
 
     # Handle defaults
     if args.create_table:
@@ -100,7 +118,8 @@ def main():
                 keep_extra_data=get_settings_arg_mapping_value('keep_extra_data', args.keep_extra_data)
             )
         except psycopg2.Error:
-            logging.warning("Table already exists, skipping create table statement.")
+            logging.info(f'Table {get_settings_arg_mapping_value("table_name", args.table_name)} already exists, '
+                         f'skipping create table statement.')
 
     migrate_from_s3_by_day(
         network=get_settings_arg_mapping_value('network', args.network),
diff --git a/dbt/redshift/settings/merge_settings_yaml.py b/dbt/redshift/settings/merge_settings_yaml.py
index 7651182..00096c7 100644
--- a/dbt/redshift/settings/merge_settings_yaml.py
+++ b/dbt/redshift/settings/merge_settings_yaml.py
@@ -32,6 +32,7 @@ def migrate_settings():
         continue
 
     with open(r'dbt/redshift/dbt_project.yml', 'w') as file:
-        document = yaml.dump(dbt_profile, file, default_style='"')
+        yaml.dump(dbt_profile, file, default_style='"')
+        stored_successfully = True
 
-    return document
+    return stored_successfully or False
diff --git a/dbt/redshift/tests/.gitkeep b/dbt/redshift/tests/.gitkeep
deleted file mode 100644
index e69de29..0000000
diff --git a/parsely_raw_data/__init__.py b/parsely_raw_data/__init__.py
index 53da8e9..d428942 100644
--- a/parsely_raw_data/__init__.py
+++ b/parsely_raw_data/__init__.py
@@ -14,7 +14,7 @@
 limitations under the License.
 """
 
-__version__ = "2.3.0"
+__version__ = "2.3.1"
 
 from . import bigquery, docgen, redshift, s3, samples, schema, stream, utils
 from six import iteritems