feat(bigquery): support config for region qualifiers to fetch jobs (d…

…atahub-project#11728)

metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py

@@ -282,6 +282,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                     include_usage_statistics=self.config.include_usage_statistics,
                     include_operations=self.config.usage.include_operational_stats,
                     top_n_queries=self.config.usage.top_n_queries,
+                    region_qualifiers=self.config.region_qualifiers,
                 ),
                 structured_report=self.report,
                 filters=self.filters,

metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py

@@ -527,6 +527,12 @@ def have_table_data_read_permission(self) -> bool:
         " Set to 1 to disable.",
     )
 
+    region_qualifiers: List[str] = Field(
+        default=["region-us", "region-eu"],
+        description="BigQuery regions to be scanned for bigquery jobs when using `use_queries_v2`. "
+        "See [this](https://cloud.google.com/bigquery/docs/information-schema-jobs#scope_and_syntax) for details.",
+    )
+
     # include_view_lineage and include_view_column_lineage are inherited from SQLCommonConfig
     # but not used in bigquery so we hide them from docs.
     include_view_lineage: bool = Field(default=True, hidden_from_docs=True)