From 395381f91f9002accaeb42d3eac0157ca3cd3fc0 Mon Sep 17 00:00:00 2001 From: Nicholas Large <84149918+nlarge-google@users.noreply.github.com> Date: Wed, 4 Oct 2023 03:15:22 +0000 Subject: [PATCH] Fix: Resolve syntax error no 3 in pipeline.yaml for google political ads. (#673) --- .../pipelines/process_csvs_and_load_to_bq/pipeline.yaml | 8 ++++++-- .../process_csvs_and_load_to_bq_dag.py | 8 ++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/datasets/google_political_ads/pipelines/process_csvs_and_load_to_bq/pipeline.yaml b/datasets/google_political_ads/pipelines/process_csvs_and_load_to_bq/pipeline.yaml index c2524db40..efd414f31 100644 --- a/datasets/google_political_ads/pipelines/process_csvs_and_load_to_bq/pipeline.yaml +++ b/datasets/google_political_ads/pipelines/process_csvs_and_load_to_bq/pipeline.yaml @@ -614,7 +614,7 @@ dag: "Spend_SEK": "spend_sek", "Spend_GBP": "spend_gbp", "Spend_NZD": "spend_nzd", - "Spend_NZD": "spend_ils", + "Spend_ILS": "spend_ils", "Spend_AUD": "spend_aud", "Spend_TWD": "spend_twd", "Spend_BRL": "spend_brl", @@ -884,6 +884,8 @@ dag: "spend_range_max_inr", "spend_range_min_bgn", "spend_range_max_bgn", + "spend_range_min_hrk", + "spend_range_max_hrk", "spend_range_min_czk", "spend_range_max_czk", "spend_range_min_dkk", @@ -937,6 +939,8 @@ dag: "Spend_Range_Max_INR": "spend_range_max_inr", "Spend_Range_Min_BGN": "spend_range_min_bgn", "Spend_Range_Max_BGN": "spend_range_max_bgn", + "Spend_Range_Min_HRK": "spend_range_min_hrk", + "Spend_Range_Max_HRK": "spend_range_max_hrk", "Spend_Range_Min_CZK": "spend_range_min_czk", "Spend_Range_Max_CZK": "spend_range_max_czk", "Spend_Range_Min_DKK": "spend_range_min_dkk", @@ -1284,7 +1288,7 @@ dag: "Spend_BRL": "spend_brl", "Spend_ARS": "spend_ars", "Spend_ZAR": "spend_zar", - "Spend_CLP": "spend_clp", + "Spend_CLP": "spend_clp" } resources: request_memory: "1G" diff --git a/datasets/google_political_ads/pipelines/process_csvs_and_load_to_bq/process_csvs_and_load_to_bq_dag.py b/datasets/google_political_ads/pipelines/process_csvs_and_load_to_bq/process_csvs_and_load_to_bq_dag.py index 3905fa9ac..1b3f70b10 100644 --- a/datasets/google_political_ads/pipelines/process_csvs_and_load_to_bq/process_csvs_and_load_to_bq_dag.py +++ b/datasets/google_political_ads/pipelines/process_csvs_and_load_to_bq/process_csvs_and_load_to_bq_dag.py @@ -520,7 +520,7 @@ "TARGET_GCS_PATH": "data/google_political_ads/advertiser_weekly_spend/data_output.csv", "TABLE_NAME": "advertiser_weekly_spend", "CSV_HEADERS": '[\n "advertiser_id",\n "advertiser_name",\n "election_cycle",\n "week_start_date",\n "spend_usd",\n "spend_eur",\n "spend_inr",\n "spend_bgn",\n "spend_hrk",\n "spend_czk",\n "spend_dkk",\n "spend_huf",\n "spend_pln",\n "spend_ron",\n "spend_sek",\n "spend_gbp",\n "spend_nzd",\n "spend_ils",\n "spend_aud",\n "spend_twd",\n "spend_brl",\n "spend_ars",\n "spend_zar",\n "spend_clp"\n]', - "RENAME_MAPPINGS": '{\n "Advertiser_ID": "advertiser_id",\n "Advertiser_Name": "advertiser_name",\n "Election_Cycle": "election_cycle",\n "Week_Start_Date": "week_start_date",\n "Spend_USD": "spend_usd",\n "Spend_EUR": "spend_eur",\n "Spend_INR": "spend_inr",\n "Spend_BGN": "spend_bgn",\n "Spend_HRK": "spend_hrk",\n "Spend_CZK": "spend_czk",\n "Spend_DKK": "spend_dkk",\n "Spend_HUF": "spend_huf",\n "Spend_PLN": "spend_pln",\n "Spend_RON": "spend_ron",\n "Spend_SEK": "spend_sek",\n "Spend_GBP": "spend_gbp",\n "Spend_NZD": "spend_nzd",\n "Spend_NZD": "spend_ils",\n "Spend_AUD": "spend_aud",\n "Spend_TWD": "spend_twd",\n "Spend_BRL": "spend_brl",\n "Spend_ARS": "spend_ars",\n "Spend_ZAR": "spend_zar",\n "Spend_CLP": "spend_clp"\n}', + "RENAME_MAPPINGS": '{\n "Advertiser_ID": "advertiser_id",\n "Advertiser_Name": "advertiser_name",\n "Election_Cycle": "election_cycle",\n "Week_Start_Date": "week_start_date",\n "Spend_USD": "spend_usd",\n "Spend_EUR": "spend_eur",\n "Spend_INR": "spend_inr",\n "Spend_BGN": "spend_bgn",\n "Spend_HRK": "spend_hrk",\n "Spend_CZK": "spend_czk",\n "Spend_DKK": "spend_dkk",\n "Spend_HUF": "spend_huf",\n "Spend_PLN": "spend_pln",\n "Spend_RON": "spend_ron",\n "Spend_SEK": "spend_sek",\n "Spend_GBP": "spend_gbp",\n "Spend_NZD": "spend_nzd",\n "Spend_ILS": "spend_ils",\n "Spend_AUD": "spend_aud",\n "Spend_TWD": "spend_twd",\n "Spend_BRL": "spend_brl",\n "Spend_ARS": "spend_ars",\n "Spend_ZAR": "spend_zar",\n "Spend_CLP": "spend_clp"\n}', }, resources={"request_memory": "1G", "request_cpu": "200m"}, ) @@ -799,8 +799,8 @@ "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", "TARGET_GCS_PATH": "data/google_political_ads/creative_stats/data_output.csv", "TABLE_NAME": "creative_stats", - "CSV_HEADERS": '[\n "ad_id",\n "ad_url",\n "ad_type",\n "regions",\n "advertiser_id",\n "advertiser_name",\n "ad_campaigns_list",\n "date_range_start",\n "date_range_end",\n "num_of_days",\n "impressions",\n "spend_usd",\n "first_served_timestamp",\n "last_served_timestamp",\n "age_targeting",\n "gender_targeting",\n "geo_targeting_included",\n "geo_targeting_excluded",\n "spend_range_min_usd",\n "spend_range_max_usd",\n "spend_range_min_eur",\n "spend_range_max_eur",\n "spend_range_min_inr",\n "spend_range_max_inr",\n "spend_range_min_bgn",\n "spend_range_max_bgn",\n "spend_range_min_czk",\n "spend_range_max_czk",\n "spend_range_min_dkk",\n "spend_range_max_dkk",\n "spend_range_min_huf",\n "spend_range_max_huf",\n "spend_range_min_pln",\n "spend_range_max_pln",\n "spend_range_min_ron",\n "spend_range_max_ron",\n "spend_range_min_sek",\n "spend_range_max_sek",\n "spend_range_min_gbp",\n "spend_range_max_gbp",\n "spend_range_min_nzd",\n "spend_range_max_nzd",\n "spend_range_min_ils",\n "spend_range_max_ils",\n "spend_range_min_aud",\n "spend_range_max_aud",\n "spend_range_min_twd",\n "spend_range_max_twd",\n "spend_range_min_brl",\n "spend_range_max_brl",\n "spend_range_min_ars",\n "spend_range_max_ars",\n "spend_range_min_zar",\n "spend_range_max_zar",\n "spend_range_min_clp",\n "spend_range_max_clp"\n]', - "RENAME_MAPPINGS": '{\n "Ad_ID": "ad_id",\n "Ad_URL": "ad_url",\n "Ad_Type": "ad_type",\n "Regions": "regions",\n "Advertiser_ID": "advertiser_id",\n "Advertiser_Name": "advertiser_name",\n "Ad_Campaigns_List": "ad_campaigns_list",\n "Date_Range_Start": "date_range_start",\n "Date_Range_End": "date_range_end",\n "Num_of_Days": "num_of_days",\n "Impressions": "impressions",\n "Spend_USD": "spend_usd",\n "Spend_Range_Min_USD": "spend_range_min_usd",\n "Spend_Range_Max_USD": "spend_range_max_usd",\n "Spend_Range_Min_EUR": "spend_range_min_eur",\n "Spend_Range_Max_EUR": "spend_range_max_eur",\n "Spend_Range_Min_INR": "spend_range_min_inr",\n "Spend_Range_Max_INR": "spend_range_max_inr",\n "Spend_Range_Min_BGN": "spend_range_min_bgn",\n "Spend_Range_Max_BGN": "spend_range_max_bgn",\n "Spend_Range_Min_CZK": "spend_range_min_czk",\n "Spend_Range_Max_CZK": "spend_range_max_czk",\n "Spend_Range_Min_DKK": "spend_range_min_dkk",\n "Spend_Range_Max_DKK": "spend_range_max_dkk",\n "Spend_Range_Min_HUF": "spend_range_min_huf",\n "Spend_Range_Max_HUF": "spend_range_max_huf",\n "Spend_Range_Min_PLN": "spend_range_min_pln",\n "Spend_Range_Max_PLN": "spend_range_max_pln",\n "Spend_Range_Min_RON": "spend_range_min_ron",\n "Spend_Range_Max_RON": "spend_range_max_ron",\n "Spend_Range_Min_SEK": "spend_range_min_sek",\n "Spend_Range_Max_SEK": "spend_range_max_sek",\n "Spend_Range_Min_GBP": "spend_range_min_gbp",\n "Spend_Range_Max_GBP": "spend_range_max_gbp",\n "Spend_Range_Min_NZD": "spend_range_min_nzd",\n "Spend_Range_Max_NZD": "spend_range_max_nzd",\n "Spend_Range_Min_ILS": "spend_range_min_ils",\n "Spend_Range_Max_ILS": "spend_range_max_ils",\n "Spend_Range_Min_AUD": "spend_range_min_aud",\n "Spend_Range_Max_AUD": "spend_range_max_aud",\n "Spend_Range_Min_TWD": "spend_range_min_twd",\n "Spend_Range_Max_TWD": "spend_range_max_twd",\n "Spend_Range_Min_BRL": "spend_range_min_brl",\n "Spend_Range_Max_BRL": "spend_range_max_brl",\n "Spend_Range_Min_ARS": "spend_range_min_ars",\n "Spend_Range_Max_ARS": "spend_range_max_ars",\n "Spend_Range_Min_ZAR": "spend_range_min_zar",\n "Spend_Range_Max_ZAR": "spend_range_max_zar",\n "Spend_Range_Min_CLP": "spend_range_min_clp",\n "Spend_Range_Max_CLP": "spend_range_max_clp",\n "Age_Targeting": "age_targeting",\n "Gender_Targeting": "gender_targeting",\n "Geo_Targeting_Included": "geo_targeting_included",\n "Geo_Targeting_Excluded": "geo_targeting_excluded",\n "First_Served_Timestamp": "first_served_timestamp",\n "Last_Served_Timestamp": "last_served_timestamp"\n}', + "CSV_HEADERS": '[\n "ad_id",\n "ad_url",\n "ad_type",\n "regions",\n "advertiser_id",\n "advertiser_name",\n "ad_campaigns_list",\n "date_range_start",\n "date_range_end",\n "num_of_days",\n "impressions",\n "spend_usd",\n "first_served_timestamp",\n "last_served_timestamp",\n "age_targeting",\n "gender_targeting",\n "geo_targeting_included",\n "geo_targeting_excluded",\n "spend_range_min_usd",\n "spend_range_max_usd",\n "spend_range_min_eur",\n "spend_range_max_eur",\n "spend_range_min_inr",\n "spend_range_max_inr",\n "spend_range_min_bgn",\n "spend_range_max_bgn",\n "spend_range_min_hrk",\n "spend_range_max_hrk",\n "spend_range_min_czk",\n "spend_range_max_czk",\n "spend_range_min_dkk",\n "spend_range_max_dkk",\n "spend_range_min_huf",\n "spend_range_max_huf",\n "spend_range_min_pln",\n "spend_range_max_pln",\n "spend_range_min_ron",\n "spend_range_max_ron",\n "spend_range_min_sek",\n "spend_range_max_sek",\n "spend_range_min_gbp",\n "spend_range_max_gbp",\n "spend_range_min_nzd",\n "spend_range_max_nzd",\n "spend_range_min_ils",\n "spend_range_max_ils",\n "spend_range_min_aud",\n "spend_range_max_aud",\n "spend_range_min_twd",\n "spend_range_max_twd",\n "spend_range_min_brl",\n "spend_range_max_brl",\n "spend_range_min_ars",\n "spend_range_max_ars",\n "spend_range_min_zar",\n "spend_range_max_zar",\n "spend_range_min_clp",\n "spend_range_max_clp"\n]', + "RENAME_MAPPINGS": '{\n "Ad_ID": "ad_id",\n "Ad_URL": "ad_url",\n "Ad_Type": "ad_type",\n "Regions": "regions",\n "Advertiser_ID": "advertiser_id",\n "Advertiser_Name": "advertiser_name",\n "Ad_Campaigns_List": "ad_campaigns_list",\n "Date_Range_Start": "date_range_start",\n "Date_Range_End": "date_range_end",\n "Num_of_Days": "num_of_days",\n "Impressions": "impressions",\n "Spend_USD": "spend_usd",\n "Spend_Range_Min_USD": "spend_range_min_usd",\n "Spend_Range_Max_USD": "spend_range_max_usd",\n "Spend_Range_Min_EUR": "spend_range_min_eur",\n "Spend_Range_Max_EUR": "spend_range_max_eur",\n "Spend_Range_Min_INR": "spend_range_min_inr",\n "Spend_Range_Max_INR": "spend_range_max_inr",\n "Spend_Range_Min_BGN": "spend_range_min_bgn",\n "Spend_Range_Max_BGN": "spend_range_max_bgn",\n "Spend_Range_Min_HRK": "spend_range_min_hrk",\n "Spend_Range_Max_HRK": "spend_range_max_hrk",\n "Spend_Range_Min_CZK": "spend_range_min_czk",\n "Spend_Range_Max_CZK": "spend_range_max_czk",\n "Spend_Range_Min_DKK": "spend_range_min_dkk",\n "Spend_Range_Max_DKK": "spend_range_max_dkk",\n "Spend_Range_Min_HUF": "spend_range_min_huf",\n "Spend_Range_Max_HUF": "spend_range_max_huf",\n "Spend_Range_Min_PLN": "spend_range_min_pln",\n "Spend_Range_Max_PLN": "spend_range_max_pln",\n "Spend_Range_Min_RON": "spend_range_min_ron",\n "Spend_Range_Max_RON": "spend_range_max_ron",\n "Spend_Range_Min_SEK": "spend_range_min_sek",\n "Spend_Range_Max_SEK": "spend_range_max_sek",\n "Spend_Range_Min_GBP": "spend_range_min_gbp",\n "Spend_Range_Max_GBP": "spend_range_max_gbp",\n "Spend_Range_Min_NZD": "spend_range_min_nzd",\n "Spend_Range_Max_NZD": "spend_range_max_nzd",\n "Spend_Range_Min_ILS": "spend_range_min_ils",\n "Spend_Range_Max_ILS": "spend_range_max_ils",\n "Spend_Range_Min_AUD": "spend_range_min_aud",\n "Spend_Range_Max_AUD": "spend_range_max_aud",\n "Spend_Range_Min_TWD": "spend_range_min_twd",\n "Spend_Range_Max_TWD": "spend_range_max_twd",\n "Spend_Range_Min_BRL": "spend_range_min_brl",\n "Spend_Range_Max_BRL": "spend_range_max_brl",\n "Spend_Range_Min_ARS": "spend_range_min_ars",\n "Spend_Range_Max_ARS": "spend_range_max_ars",\n "Spend_Range_Min_ZAR": "spend_range_min_zar",\n "Spend_Range_Max_ZAR": "spend_range_max_zar",\n "Spend_Range_Min_CLP": "spend_range_min_clp",\n "Spend_Range_Max_CLP": "spend_range_max_clp",\n "Age_Targeting": "age_targeting",\n "Gender_Targeting": "gender_targeting",\n "Geo_Targeting_Included": "geo_targeting_included",\n "Geo_Targeting_Excluded": "geo_targeting_excluded",\n "First_Served_Timestamp": "first_served_timestamp",\n "Last_Served_Timestamp": "last_served_timestamp"\n}', }, resources={ "request_memory": "16G", @@ -1177,7 +1177,7 @@ "TARGET_GCS_PATH": "data/google_political_ads/geo_spend/data_output.csv", "TABLE_NAME": "geo_spend", "CSV_HEADERS": '[\n "country",\n "country_subdivision_primary",\n "country_subdivision_secondary",\n "spend_usd",\n "spend_eur",\n "spend_inr",\n "spend_bgn",\n "spend_hrk",\n "spend_czk",\n "spend_dkk",\n "spend_huf",\n "spend_pln",\n "spend_ron",\n "spend_sek",\n "spend_gbp",\n "spend_nzd",\n "spend_ils",\n "spend_aud",\n "spend_twd",\n "spend_brl",\n "spend_ars",\n "spend_zar",\n "spend_clp"\n]', - "RENAME_MAPPINGS": '{\n "Country": "country",\n "Country_Subdivision_Primary": "country_subdivision_primary",\n "Country_Subdivision_Secondary": "country_subdivision_secondary",\n "Spend_USD": "spend_usd",\n "Spend_EUR": "spend_eur",\n "Spend_INR": "spend_inr",\n "Spend_BGN": "spend_bgn",\n "Spend_HRK": "spend_hrk",\n "Spend_CZK": "spend_czk",\n "Spend_DKK": "spend_dkk",\n "Spend_HUF": "spend_huf",\n "Spend_PLN": "spend_pln",\n "Spend_RON": "spend_ron",\n "Spend_SEK": "spend_sek",\n "Spend_GBP": "spend_gbp",\n "Spend_NZD": "spend_nzd",\n "Spend_ILS": "spend_ils",\n "Spend_TWD": "spend_twd",\n "Spend_BRL": "spend_brl",\n "Spend_ARS": "spend_ars",\n "Spend_ZAR": "spend_zar",\n "Spend_CLP": "spend_clp",\n}', + "RENAME_MAPPINGS": '{\n "Country": "country",\n "Country_Subdivision_Primary": "country_subdivision_primary",\n "Country_Subdivision_Secondary": "country_subdivision_secondary",\n "Spend_USD": "spend_usd",\n "Spend_EUR": "spend_eur",\n "Spend_INR": "spend_inr",\n "Spend_BGN": "spend_bgn",\n "Spend_HRK": "spend_hrk",\n "Spend_CZK": "spend_czk",\n "Spend_DKK": "spend_dkk",\n "Spend_HUF": "spend_huf",\n "Spend_PLN": "spend_pln",\n "Spend_RON": "spend_ron",\n "Spend_SEK": "spend_sek",\n "Spend_GBP": "spend_gbp",\n "Spend_NZD": "spend_nzd",\n "Spend_ILS": "spend_ils",\n "Spend_TWD": "spend_twd",\n "Spend_BRL": "spend_brl",\n "Spend_ARS": "spend_ars",\n "Spend_ZAR": "spend_zar",\n "Spend_CLP": "spend_clp"\n}', }, resources={"request_memory": "1G", "request_cpu": "200m"}, )