From 8c768a532135fc19b1a7cdd1f6b18adf47132b2c Mon Sep 17 00:00:00 2001
From: Aarthi <aarthi.sagi@gmail.com>
Date: Thu, 19 May 2022 11:58:59 -0400
Subject: [PATCH 1/3] updated serde details to ensure governed tables are in
 parquet

---
 .DS_Store                                        | Bin 0 -> 6148 bytes
 samples/.DS_Store                                | Bin 0 -> 6148 bytes
 .../standard_table_to_governed.py                |  12 ++++++++++--
 3 files changed, 10 insertions(+), 2 deletions(-)
 create mode 100644 .DS_Store
 create mode 100644 samples/.DS_Store

diff --git a/.DS_Store b/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..df1b7d1b114f59d764c78b65d0fcc5bc61487b09
GIT binary patch
literal 6148
zcmeHK%}T>S5T0$TX(&Pu3gT(OYr$4gD_%maFJMFuDm9@+gK4%jtvQrJ9zb8n2l08F
z+1-HE9z2QI8QA@1=Vv$bLG}j#L}%J>0n`DYij7c^vP8(d)K#*<ghI~|!vNASOGjb6
zWTL-lqHlK~g#d;yf=}PCLiD_aqc}<@&E~r(l`CuOj^jEt=gxmrnLi08)3g(eZ)kL>
zR2&w15MD)ttXJPWQ^_QVlEGLfMEwDV++IgXUu7LNP5P<M^^L=It8TBpGn=)K+YNcJ
z-<&t(tkr5CHDs%KayXy6)vevV(~I6yJWSM!AyXjlS~e^e@QTWpx}N=U6076^Jw;X#
z&BzQe1Iz$3uu2TrQ=nQ~rJ}i3W`G&^5d*Y8NNj|j#oVCYI<TYbGx;loB&gF{f>2uY
zEanE$gCa~SqDd9@i6Kln+NF*2EanDHItaZo&SO_D9xp<#j&`ZTL3jqaX9k#oMFz@d
zSf}&<9DkXmkNm|H9x(&Vz&~R^RJwk*gGJf1^;>y#)=F&m*hnZYCj|xdg-ZYqXdl_u
ePUDxPW1MF(H%POPU8f`RML-k69W(F?4155QuuU8Q

literal 0
HcmV?d00001

diff --git a/samples/.DS_Store b/samples/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..47c981c9acd3409af6490ed3eac011bccef74432
GIT binary patch
literal 6148
zcmeHK%}T>S5T32oHWZ-;1?g$QYr$4gD_%m?7cim+m70*C!I+gKZ4af82hbPtL3|!(
zcDG8YUOY&V8JPWcXJ>Y^--P`E0MQ?YU4SM4)KLj54i?`C^^-10#aal39wP)FG6-P+
z#ez2*mXQI<T?=x|mk%js?kj&j1du?abE&Yr2L&T+XXqC}7>x7hAW;FvzXi`poEM$W
zyQr*GSJ!Gar`D+5dJk&s6@D?y`+jy!tqY}+U{?2o%QzYjnj0r7E&Mo*G93_y5xU%5
z#c8O<eKkzOT*rE*!*S})pxK&CdWXHX+}rI;+j7$F_72*z+d0~wPM!MZ_RjJ7;4yhh
z)r+BA;NPib-Qo;hvGS|ckGw2RRC<r`v$1DmGBN|q05h=s4A`Bk-dO%m<u7Cgn1MfL
zfc6K8O6XcF4eG4}3%Wj1f0d8~>-3f&lonl!r9tdL5hfJTgbMq_5GEY$()zg;OM@mH
zgq|74u`>(%LJ@j)v`g&{!ZpYxGr$bYGq7TsO*;S2et!SY2XTuTU<Upb1EPBBo%V4_
z_H12P9G$fi^$wMU{7QqLAz0BzG5XR`yoIU-?UE{puEo+IT2S~RplRTO8TeBMJ^*5a
BTVVhI

literal 0
HcmV?d00001

diff --git a/samples/standard_table_to_governed/standard_table_to_governed.py b/samples/standard_table_to_governed/standard_table_to_governed.py
index 9955853..02ce290 100644
--- a/samples/standard_table_to_governed/standard_table_to_governed.py
+++ b/samples/standard_table_to_governed/standard_table_to_governed.py
@@ -73,7 +73,15 @@
 }
 
 table_to_create['StorageDescriptor']['Location'] = output_path
-table_to_create['StorageDescriptor']['Parameters']['lakeformation.aso.status'] = 'true'
+table_to_create['StorageDescriptor']['InputFormat'] = 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+table_to_create['StorageDescriptor']['OutputFormat'] = 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+table_to_create['StorageDescriptor']['SerdeInfo']['SerializationLibrary'] = 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+table_to_create['StorageDescriptor']['SerdeInfo']['Parameters'] = {}
+table_to_create['StorageDescriptor']['SerdeInfo']['Parameters']['serialization.format'] = '1'
+table_to_create['Parameters'] = {}
+table_to_create['Parameters']['lakeformation.aso.status'] = 'true'
+table_to_create['Parameters']['classification'] = 'parquet'
+
 if 'Description' in res['Table']:
     table_to_create['Description'] = res['Table']['Description']
 
@@ -98,7 +106,7 @@
     partitionKeys=partition_key_names,
     transactionId=tx_id
 )
-sink.setFormat("glueparquet")
+sink.setFormat("parquet", useGlueParquetWriter=True)
 sink.setCatalogInfo(
     catalogDatabase=output_database,
     catalogTableName=output_table

From fee44f5fd6ff030f010abfd76af3ddd0eedcf005 Mon Sep 17 00:00:00 2001
From: Aarthi <aarthi.sagi@gmail.com>
Date: Thu, 19 May 2022 13:41:05 -0400
Subject: [PATCH 2/3] cleaned up the .DS files

---
 .DS_Store         | Bin 6148 -> 0 bytes
 samples/.DS_Store | Bin 6148 -> 0 bytes
 2 files changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 .DS_Store
 delete mode 100644 samples/.DS_Store

diff --git a/.DS_Store b/.DS_Store
deleted file mode 100644
index df1b7d1b114f59d764c78b65d0fcc5bc61487b09..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 6148
zcmeHK%}T>S5T0$TX(&Pu3gT(OYr$4gD_%maFJMFuDm9@+gK4%jtvQrJ9zb8n2l08F
z+1-HE9z2QI8QA@1=Vv$bLG}j#L}%J>0n`DYij7c^vP8(d)K#*<ghI~|!vNASOGjb6
zWTL-lqHlK~g#d;yf=}PCLiD_aqc}<@&E~r(l`CuOj^jEt=gxmrnLi08)3g(eZ)kL>
zR2&w15MD)ttXJPWQ^_QVlEGLfMEwDV++IgXUu7LNP5P<M^^L=It8TBpGn=)K+YNcJ
z-<&t(tkr5CHDs%KayXy6)vevV(~I6yJWSM!AyXjlS~e^e@QTWpx}N=U6076^Jw;X#
z&BzQe1Iz$3uu2TrQ=nQ~rJ}i3W`G&^5d*Y8NNj|j#oVCYI<TYbGx;loB&gF{f>2uY
zEanE$gCa~SqDd9@i6Kln+NF*2EanDHItaZo&SO_D9xp<#j&`ZTL3jqaX9k#oMFz@d
zSf}&<9DkXmkNm|H9x(&Vz&~R^RJwk*gGJf1^;>y#)=F&m*hnZYCj|xdg-ZYqXdl_u
ePUDxPW1MF(H%POPU8f`RML-k69W(F?4155QuuU8Q

diff --git a/samples/.DS_Store b/samples/.DS_Store
deleted file mode 100644
index 47c981c9acd3409af6490ed3eac011bccef74432..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 6148
zcmeHK%}T>S5T32oHWZ-;1?g$QYr$4gD_%m?7cim+m70*C!I+gKZ4af82hbPtL3|!(
zcDG8YUOY&V8JPWcXJ>Y^--P`E0MQ?YU4SM4)KLj54i?`C^^-10#aal39wP)FG6-P+
z#ez2*mXQI<T?=x|mk%js?kj&j1du?abE&Yr2L&T+XXqC}7>x7hAW;FvzXi`poEM$W
zyQr*GSJ!Gar`D+5dJk&s6@D?y`+jy!tqY}+U{?2o%QzYjnj0r7E&Mo*G93_y5xU%5
z#c8O<eKkzOT*rE*!*S})pxK&CdWXHX+}rI;+j7$F_72*z+d0~wPM!MZ_RjJ7;4yhh
z)r+BA;NPib-Qo;hvGS|ckGw2RRC<r`v$1DmGBN|q05h=s4A`Bk-dO%m<u7Cgn1MfL
zfc6K8O6XcF4eG4}3%Wj1f0d8~>-3f&lonl!r9tdL5hfJTgbMq_5GEY$()zg;OM@mH
zgq|74u`>(%LJ@j)v`g&{!ZpYxGr$bYGq7TsO*;S2et!SY2XTuTU<Upb1EPBBo%V4_
z_H12P9G$fi^$wMU{7QqLAz0BzG5XR`yoIU-?UE{puEo+IT2S~RplRTO8TeBMJ^*5a
BTVVhI


From b9da782785973fd63fc24c819e2fb2e97b1d94dd Mon Sep 17 00:00:00 2001
From: Aarthi <aarthi.sagi@gmail.com>
Date: Wed, 15 Jun 2022 10:06:34 -0400
Subject: [PATCH 3/3] removed glue enableUpdateCatalog flag

---
 samples/standard_table_to_governed/standard_table_to_governed.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/samples/standard_table_to_governed/standard_table_to_governed.py b/samples/standard_table_to_governed/standard_table_to_governed.py
index 02ce290..3880275 100644
--- a/samples/standard_table_to_governed/standard_table_to_governed.py
+++ b/samples/standard_table_to_governed/standard_table_to_governed.py
@@ -102,7 +102,6 @@
 sink = glue_context.getSink(
     connection_type="s3",
     path=output_path,
-    enableUpdateCatalog=True,
     partitionKeys=partition_key_names,
     transactionId=tx_id
 )