connector with version

lalelisealstad · Sep 29, 2024 · dc6d1d3 · dc6d1d3
1 parent eb6243f
commit dc6d1d3
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/main.py b/main.py
@@ -58,7 +58,7 @@ def transform(df):
 
 def main():
     # Path to the BigQuery connector JAR
-    bigquery_connector_path = 'gs://spark-lib/bigquery/spark-bigquery-latest.jar'
+    bigquery_connector_path = "gs://spark-lib/bigquery/spark-bigquery-with-dependencies_2.12-0.41.0.jar"
 
     # Start a PySpark session with BigQuery connector
     spark = SparkSession.builder \

diff --git a/main.tf b/main.tf
@@ -41,7 +41,7 @@ resource "google_dataproc_workflow_template" "template" {
           }
         }
         software_config {
-          # Default
+          image_version = "2.0-debian10"
         }
       }
     }
@@ -52,7 +52,7 @@ resource "google_dataproc_workflow_template" "template" {
     step_id = "job-a722fda0"
     pyspark_job {
       main_python_file_uri = "gs://${google_storage_bucket.data_bucket.name}/main.py"
-      jar_file_uris = ["gs://spark-lib/bigquery/spark-bigquery-latest.jar"]
+      jar_file_uris = ["gs://spark-lib/bigquery/spark-bigquery-with-dependencies_2.12-0.41.0.jar"]
     }
   }
 }