small updates and error handling

.gitignore

@@ -8,3 +8,6 @@
 build/*
 dist/*
 .idea/*
+target/
+dbt_modules/
+logs/

dbt/redshift/dbt_project.yml

@@ -18,7 +18,7 @@
 - "{% if var('etl:keep_rawdata') == true %} select 1 {% else %} truncate table {{\
   \ target.schema }}.parsely_rawdata {% endif %}"
 - "truncate table {{var('parsely:events')}}"
-"profile": "parsely_dwh"
+"profile": "parsely-dwh"
 "source-paths":
 - "models"
 "target-path": "target"
@@ -36,6 +36,6 @@
   "custom:videoskimtime": "15"
   "etl:keep_rawdata": "True"
   "parsely:actions": "('pageview', 'heartbeat', 'videostart', 'vheartbeat')"
-  "parsely:events": "demo_rawdata"
+  "parsely:events": "parsely_rawdata"
   "parsely:timezone": "'America/New_York'"
 "version": "0.0.1"

dbt/redshift/redshift_etl.py

@@ -7,7 +7,22 @@
 
 from parsely_raw_data import redshift as parsely_redshift
 from parsely_raw_data import utils as parsely_utils
-from dbt.redshift.settings.default import *
+from dbt.redshift.settings.default import (
+    DBT_PROFILE_LOCATION,
+    DBT_PROFILE_TARGET_NAME,
+    ETL_END_DATE,
+    ETL_KEEP_RAW_DATA,
+    ETL_START_DATE,
+    PARSELY_RAW_DATA_TABLE,
+    REDSHIFT_DATABASE,
+    REDSHIFT_HOST,
+    REDSHIFT_PASSWORD,
+    REDSHIFT_PORT,
+    REDSHIFT_USER,
+    S3_AWS_ACCESS_KEY_ID,
+    S3_AWS_SECRET_ACCESS_KEY,
+    S3_NETWORK_NAME,
+)
 from dbt.redshift.settings.merge_settings_yaml import migrate_settings
 
 SETTINGS_ARG_MAPPING = {
@@ -86,6 +101,9 @@ def main():
     settings_migration = migrate_settings()
     if not settings_migration:
         logging.warning("Settings not copied to dbt_profiles.yml successfully.")
+        raise Exception("Settings not copied to dbt_profiles.yml successfully. Please edit default.py or copy the"
+                        "original default.py.schema as default.py and edit carefully. Be mindful of single quotes"
+                        "and double quotes.")
 
     # Handle defaults
     if args.create_table:
@@ -100,7 +118,8 @@ def main():
                 keep_extra_data=get_settings_arg_mapping_value('keep_extra_data', args.keep_extra_data)
             )
         except psycopg2.Error:
-            logging.warning("Table already exists, skipping create table statement.")
+            logging.info(f'Table {get_settings_arg_mapping_value("table_name", args.table_name)} already exists, '
+                         f'skipping create table statement.')
 
     migrate_from_s3_by_day(
         network=get_settings_arg_mapping_value('network', args.network),

dbt/redshift/settings/merge_settings_yaml.py

@@ -32,6 +32,7 @@ def migrate_settings():
         continue
 
     with open(r'dbt/redshift/dbt_project.yml', 'w') as file:
-        document = yaml.dump(dbt_profile, file, default_style='"')
+        yaml.dump(dbt_profile, file, default_style='"')
+        stored_successfully = True
 
-    return document
+    return stored_successfully or False

parsely_raw_data/__init__.py

@@ -14,7 +14,7 @@
 limitations under the License.
 """
 
-__version__ = "2.3.0"
+__version__ = "2.3.1"
 
 from . import bigquery, docgen, redshift, s3, samples, schema, stream, utils
 from six import iteritems