From 318d9c248f088d346b55f3fe10111cee4dd79de8 Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Thu, 6 Jun 2019 15:28:49 -0600 Subject: [PATCH 1/3] Adding fields to schema and dev version release --- .gitignore | 1 + parsely_raw_data/__init__.py | 2 +- parsely_raw_data/schema.py | 4 ++++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 73adb37..0695426 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ .cache build/* dist/* +.idea/* \ No newline at end of file diff --git a/parsely_raw_data/__init__.py b/parsely_raw_data/__init__.py index 09761e3..5597b55 100644 --- a/parsely_raw_data/__init__.py +++ b/parsely_raw_data/__init__.py @@ -14,7 +14,7 @@ limitations under the License. """ -__version__ = '2.2.5' +__version__ = '2.3.0.dev0' from . import bigquery, docgen, redshift, s3, samples, schema, stream, utils diff --git a/parsely_raw_data/schema.py b/parsely_raw_data/schema.py index cb003b4..1205052 100644 --- a/parsely_raw_data/schema.py +++ b/parsely_raw_data/schema.py @@ -26,6 +26,7 @@ {"key": "action", "ex": "pageview", "type": str, "size": 256, "req": True}, {"key": "apikey", "ex": "mashable.com", "type": str, "size": 256, "req": True}, {"key": "campaign_id", "ex": "facebook_campaign", "type": str, "size": 256, "available_with_field": "url_query"}, + {"key": "channel", "ex": "AMP", "type": str, "size":64}, {"key": "display", "ex": True, "type": bool}, {"key": "display_avail_height", "ex": 735, "type": int, "available_with_field": "display"}, {"key": "display_avail_width", "ex": 1280, "type": int, "available_with_field": "display"}, @@ -65,6 +66,8 @@ {"key": "metadata_thumb_url", "ex": "https://images.parsely.com/xY9xNBMulGDKRMzfKaUQzs7A9PA=/160x160/smart/http%3A//a.amz.mshcdn.com/media/ZgkyMDE2LzA5LzA3LzU2L0NyeFhpNjNYRUFBSnZwRS5lNDAyMy5qcGcKcAl0aHVtYgkxMjAweDYzMAplCWpwZw/156d0173/3ae/CrxXi63XEAAJvpE.jpg", "type": str, "available_with_field": "metadata"}, {"key": "metadata_title", "ex": "Everyone has the same fear about Apple's new earbuds", "type": str, "available_with_field": "metadata"}, {"key": "metadata_urls", "ex": ["http://mashable.com/2016/09/07/airpods-jokes/"], "type": list, "available_with_field": "metadata"}, + {"key": "pageload_id", "ex": "66554433", "type": int}, + {"key": "pageview_id", "ex": "66554433", "type": int}, {"key": "ref_category", "ex": "internal", "type": str, "size": 64}, {"key": "ref_clean", "ex": "http://mashable.com/", "type": str}, {"key": "ref_domain", "ex": "mashable.com", "type": str, "size": 256}, @@ -135,6 +138,7 @@ {"key": "utm_term", "ex": "8098", "type": str, "size": 64, "available_with_field": "url_query"}, {"key": "utm_content", "ex": "sports", "type": str, "size": 256, "available_with_field": "url_query"}, {"key": "user_agent", "ex": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/601.7.7 (KHTML, like Gecko) Version/9.1.2 Safari/601.7.7", "type": str}, + {"key": "videostart_id", "ex": "66554433", "type": int}, {"key": "version", "ex": 1, "type": int}, {"key": "visitor", "ex": True, "type": bool}, {"key": "visitor_ip", "ex": "108.225.131.20", "type": str, "size": 256, "available_with_field": "visitor"}, From d87787c1eee57194c3941c3c72497f93e76018c5 Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Thu, 6 Jun 2019 15:36:13 -0600 Subject: [PATCH 2/3] Formatting fix --- parsely_raw_data/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsely_raw_data/schema.py b/parsely_raw_data/schema.py index 1205052..d806165 100644 --- a/parsely_raw_data/schema.py +++ b/parsely_raw_data/schema.py @@ -26,7 +26,7 @@ {"key": "action", "ex": "pageview", "type": str, "size": 256, "req": True}, {"key": "apikey", "ex": "mashable.com", "type": str, "size": 256, "req": True}, {"key": "campaign_id", "ex": "facebook_campaign", "type": str, "size": 256, "available_with_field": "url_query"}, - {"key": "channel", "ex": "AMP", "type": str, "size":64}, + {"key": "channel", "ex": "AMP", "type": str, "size": 64}, {"key": "display", "ex": True, "type": bool}, {"key": "display_avail_height", "ex": 735, "type": int, "available_with_field": "display"}, {"key": "display_avail_width", "ex": 1280, "type": int, "available_with_field": "display"}, From bf6fc56a2c114bc2a1de0120ddf9914d2928bcbb Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Thu, 6 Jun 2019 16:46:06 -0600 Subject: [PATCH 3/3] Fixing examples --- parsely_raw_data/schema.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parsely_raw_data/schema.py b/parsely_raw_data/schema.py index d806165..e2610c4 100644 --- a/parsely_raw_data/schema.py +++ b/parsely_raw_data/schema.py @@ -26,7 +26,7 @@ {"key": "action", "ex": "pageview", "type": str, "size": 256, "req": True}, {"key": "apikey", "ex": "mashable.com", "type": str, "size": 256, "req": True}, {"key": "campaign_id", "ex": "facebook_campaign", "type": str, "size": 256, "available_with_field": "url_query"}, - {"key": "channel", "ex": "AMP", "type": str, "size": 64}, + {"key": "channel", "ex": "amp", "type": str, "size": 64}, {"key": "display", "ex": True, "type": bool}, {"key": "display_avail_height", "ex": 735, "type": int, "available_with_field": "display"}, {"key": "display_avail_width", "ex": 1280, "type": int, "available_with_field": "display"}, @@ -66,8 +66,8 @@ {"key": "metadata_thumb_url", "ex": "https://images.parsely.com/xY9xNBMulGDKRMzfKaUQzs7A9PA=/160x160/smart/http%3A//a.amz.mshcdn.com/media/ZgkyMDE2LzA5LzA3LzU2L0NyeFhpNjNYRUFBSnZwRS5lNDAyMy5qcGcKcAl0aHVtYgkxMjAweDYzMAplCWpwZw/156d0173/3ae/CrxXi63XEAAJvpE.jpg", "type": str, "available_with_field": "metadata"}, {"key": "metadata_title", "ex": "Everyone has the same fear about Apple's new earbuds", "type": str, "available_with_field": "metadata"}, {"key": "metadata_urls", "ex": ["http://mashable.com/2016/09/07/airpods-jokes/"], "type": list, "available_with_field": "metadata"}, - {"key": "pageload_id", "ex": "66554433", "type": int}, - {"key": "pageview_id", "ex": "66554433", "type": int}, + {"key": "pageload_id", "ex": "11223344", "type": int}, + {"key": "pageview_id", "ex": "55667788", "type": int}, {"key": "ref_category", "ex": "internal", "type": str, "size": 64}, {"key": "ref_clean", "ex": "http://mashable.com/", "type": str}, {"key": "ref_domain", "ex": "mashable.com", "type": str, "size": 256}, @@ -138,7 +138,7 @@ {"key": "utm_term", "ex": "8098", "type": str, "size": 64, "available_with_field": "url_query"}, {"key": "utm_content", "ex": "sports", "type": str, "size": 256, "available_with_field": "url_query"}, {"key": "user_agent", "ex": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/601.7.7 (KHTML, like Gecko) Version/9.1.2 Safari/601.7.7", "type": str}, - {"key": "videostart_id", "ex": "66554433", "type": int}, + {"key": "videostart_id", "ex": "99887766", "type": int}, {"key": "version", "ex": 1, "type": int}, {"key": "visitor", "ex": True, "type": bool}, {"key": "visitor_ip", "ex": "108.225.131.20", "type": str, "size": 256, "available_with_field": "visitor"},