chore(ingest): bump acryl-sqlglot (datahub-project#11331)

G-Research · Sep 10, 2024 · f403370 · f403370
1 parent 0d4eda6
commit f403370
Show file tree

Hide file tree

Showing 6 changed files with 12 additions and 9 deletions.
diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
@@ -101,7 +101,7 @@
 sqlglot_lib = {
     # Using an Acryl fork of sqlglot.
     # https://github.com/tobymao/sqlglot/compare/main...hsheth2:sqlglot:main?expand=1
-    "acryl-sqlglot[rs]==25.8.2.dev9",
+    "acryl-sqlglot[rs]==25.20.2.dev5",
 }
 
 classification_lib = {

diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py
@@ -130,7 +130,7 @@ def parse_alter_table_rename(default_schema: str, query: str) -> Tuple[str, str,
     """
 
     parsed_query = parse_statement(query, dialect=get_dialect("redshift"))
-    assert isinstance(parsed_query, sqlglot.exp.AlterTable)
+    assert isinstance(parsed_query, sqlglot.exp.Alter)
     prev_name = parsed_query.this.name
     rename_clause = parsed_query.args["actions"][0]
     assert isinstance(rename_clause, sqlglot.exp.RenameTable)
@@ -875,7 +875,7 @@ def _process_table_renames(
                     default_schema=self.config.default_schema,
                     query=query_text,
                 )
-            except ValueError as e:
+            except Exception as e:
                 logger.info(f"Failed to parse alter table rename: {e}")
                 self.report.num_alter_table_parse_errors += 1
                 continue

diff --git a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py
@@ -198,7 +198,7 @@ def _table_level_lineage(
                 sqlglot.exp.Update,
                 sqlglot.exp.Delete,
                 sqlglot.exp.Merge,
-                sqlglot.exp.AlterTable,
+                sqlglot.exp.Alter,
             )
             # In some cases like "MERGE ... then INSERT (col1, col2) VALUES (col1, col2)",
             # the `this` on the INSERT part isn't a table.
@@ -837,7 +837,10 @@ def _translate_internal_column_lineage(
                 raw_column_lineage.downstream.column_type.sql(dialect=dialect)
                 if raw_column_lineage.downstream.column_type
                 and raw_column_lineage.downstream.column_type.this
-                != sqlglot.exp.DataType.Type.UNKNOWN
+                not in {
+                    sqlglot.exp.DataType.Type.UNKNOWN,
+                    sqlglot.exp.DataType.Type.NULL,
+                }
                 else None
             ),
         ),

diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_view_from_union.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_view_from_union.json
@@ -3,7 +3,7 @@
     "query_type_props": {
         "kind": "VIEW"
     },
-    "query_fingerprint": "e947f7884942bae354b9f90cdb649c01efd85ab9fddf830b0bec194c22652639",
+    "query_fingerprint": "3c811a0b47858c50d1c8f666644986a3d9f4522c3f084249656af345741da690",
     "in_tables": [
         "urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project_2.my_dataset_2.sometable,PROD)",
         "urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project_2.my_dataset_2.sometable2,PROD)"
@@ -59,6 +59,6 @@
     ],
     "debug_info": {
         "confidence": 0.4,
-        "generalized_statement": "CREATE VIEW my_view AS SELECT * FROM my_project_2.my_dataset_2.sometable UNION DISTINCT SELECT * FROM my_project_2.my_dataset_2.sometable2 AS a"
+        "generalized_statement": "CREATE VIEW my_view AS SELECT * FROM my_project_2.my_dataset_2.sometable UNION ALL SELECT * FROM my_project_2.my_dataset_2.sometable2 AS a"
     }
 }
diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
@@ -618,7 +618,7 @@ def test_bigquery_view_from_union():
         """
 CREATE VIEW my_view as
 select * from my_project_2.my_dataset_2.sometable
-union
+union all
 select * from my_project_2.my_dataset_2.sometable2 as a
 """,
         dialect="bigquery",

diff --git a/metadata-ingestion/tests/unit/test_bigquery_sql_lineage.py b/metadata-ingestion/tests/unit/test_bigquery_sql_lineage.py
@@ -289,7 +289,7 @@ def test_bigquery_sql_lineage_create_or_replace_view_name_with_hyphens_is_accept
             CREATE OR REPLACE VIEW test-project.dataset.test_view AS
             SELECT *
             FROM project.dataset.src_table_a
-            UNION
+            UNION ALL
             SELECT * FROM `project.dataset.src_table_b`
         """,
         schema_resolver=SchemaResolver(platform="bigquery"),