specify 1.1 in batch runtime config (#532) (#538)

* specify 1.1 in batch runtime config * add changie * fix quoting (cherry picked from commit 919a6ca) Co-authored-by: colin-rogers-dbt <[email protected]>
dbt-labs · Feb 14, 2023 · 9a37eed · 9a37eed
1 parent 8e6558c
commit 9a37eed
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 1 deletion.
diff --git a/.changes/unreleased/Fixes-20230213-203317.yaml b/.changes/unreleased/Fixes-20230213-203317.yaml
@@ -0,0 +1,6 @@
+kind: Fixes
+body: Pin dataproc serverless spark runtime to `1.1`
+time: 2023-02-13T20:33:17.839861-08:00
+custom:
+  Author: colin-rogers-dbt
+  Issue: "531"
diff --git a/dbt/adapters/bigquery/python_submissions.py b/dbt/adapters/bigquery/python_submissions.py
@@ -121,7 +121,8 @@ def _get_job_client(self) -> dataproc_v1.BatchControllerClient:
 
     def _submit_dataproc_job(self) -> dataproc_v1.types.jobs.Job:
         # create the Dataproc Serverless job config
-        batch = dataproc_v1.Batch()
+        # need to pin dataproc version to 1.1 as it now defaults to 2.0
+        batch = dataproc_v1.Batch({"runtime_config": dataproc_v1.RuntimeConfig(version="1.1")})
         batch.pyspark_batch.main_python_file_uri = self.gcs_location
         # how to keep this up to date?
         # we should probably also open this up to be configurable