diff --git a/metadata-ingestion/tests/unit/glue/glue_deleted_actor_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_deleted_actor_mces_golden.json index 51d5a6974eb0d..725bfdb22b479 100644 --- a/metadata-ingestion/tests/unit/glue/glue_deleted_actor_mces_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_deleted_actor_mces_golden.json @@ -17,7 +17,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -32,7 +33,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -47,7 +49,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -64,7 +67,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -79,13 +83,14 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_jsons_markers,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,awsdatacatalog.test-database.test_jsons_markers,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -220,6 +225,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -231,12 +237,13 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_jsons_markers,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,awsdatacatalog.test-database.test_jsons_markers,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -248,12 +255,13 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_jsons_markers,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,awsdatacatalog.test-database.test_jsons_markers,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -263,12 +271,13 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_jsons_markers,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,awsdatacatalog.test-database.test_jsons_markers,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -283,13 +292,14 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,awsdatacatalog.test-database.test_parquet,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -425,6 +435,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -436,12 +447,13 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,awsdatacatalog.test-database.test_parquet,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -453,12 +465,13 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,awsdatacatalog.test-database.test_parquet,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -468,12 +481,13 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,awsdatacatalog.test-database.test_parquet,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -488,12 +502,13 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,awsdatacatalog.flights-database.avro,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -503,7 +518,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -518,7 +534,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json index a0a4ab00a429e..a12f4120f0109 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json @@ -54,7 +54,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,awsdatacatalog.flights-database.avro,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -230,6 +230,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -254,7 +255,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,awsdatacatalog.flights-database.avro,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -267,7 +268,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,awsdatacatalog.flights-database.avro,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -331,7 +332,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_jsons_markers,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,awsdatacatalog.test-database.test_jsons_markers,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -466,6 +467,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -490,7 +492,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_jsons_markers,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,awsdatacatalog.test-database.test_jsons_markers,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -503,7 +505,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_jsons_markers,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,awsdatacatalog.test-database.test_jsons_markers,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -515,7 +517,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,awsdatacatalog.test-database.test_parquet,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -651,6 +653,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -675,7 +678,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,awsdatacatalog.test-database.test_parquet,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -688,7 +691,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,awsdatacatalog.test-database.test_parquet,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -830,7 +833,7 @@ { "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)" + "urn:li:dataset:(urn:li:dataPlatform:glue,awsdatacatalog.flights-database.avro,PROD)" ], "outputDatasets": [], "inputDatajobs": [] @@ -1019,7 +1022,7 @@ { "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)" + "urn:li:dataset:(urn:li:dataPlatform:glue,awsdatacatalog.test-database.test_parquet,PROD)" ], "outputDatasets": [], "inputDatajobs": [] diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json index 6a5f47ef67cf5..b6f6c23b1b3db 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json @@ -56,7 +56,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.flights-database.avro,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.awsdatacatalog.flights-database.avro,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -233,6 +233,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -257,7 +258,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.flights-database.avro,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.awsdatacatalog.flights-database.avro,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -270,7 +271,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.flights-database.avro,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.awsdatacatalog.flights-database.avro,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -336,7 +337,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_jsons_markers,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.awsdatacatalog.test-database.test_jsons_markers,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -472,6 +473,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -496,7 +498,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_jsons_markers,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.awsdatacatalog.test-database.test_jsons_markers,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -509,7 +511,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_jsons_markers,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.awsdatacatalog.test-database.test_jsons_markers,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -521,7 +523,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_parquet,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.awsdatacatalog.test-database.test_parquet,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -658,6 +660,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -682,7 +685,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_parquet,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.awsdatacatalog.test-database.test_parquet,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -695,7 +698,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_parquet,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.awsdatacatalog.test-database.test_parquet,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -837,7 +840,7 @@ { "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.flights-database.avro,PROD)" + "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.awsdatacatalog.flights-database.avro,PROD)" ], "outputDatasets": [], "inputDatajobs": [] @@ -1026,7 +1029,7 @@ { "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_parquet,PROD)" + "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.awsdatacatalog.test-database.test_parquet,PROD)" ], "outputDatasets": [], "inputDatajobs": [] diff --git a/metadata-ingestion/tests/unit/test_glue_source.py b/metadata-ingestion/tests/unit/test_glue_source.py index 8fb840ee003c7..8b42fba55218d 100644 --- a/metadata-ingestion/tests/unit/test_glue_source.py +++ b/metadata-ingestion/tests/unit/test_glue_source.py @@ -333,6 +333,6 @@ def test_glue_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph): state1.get_urns_not_in(type="*", other_checkpoint_state=state2) ) assert difference_urns == { - "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:glue,awsdatacatalog.flights-database.avro,PROD)", "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", }