diff --git a/.changes/unreleased/Fixes-20230213-203317.yaml b/.changes/unreleased/Fixes-20230213-203317.yaml new file mode 100644 index 000000000..ae64c4e2d --- /dev/null +++ b/.changes/unreleased/Fixes-20230213-203317.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Pin dataproc serverless spark runtime to `1.1` +time: 2023-02-13T20:33:17.839861-08:00 +custom: + Author: colin-rogers-dbt + Issue: "531" diff --git a/dbt/adapters/bigquery/python_submissions.py b/dbt/adapters/bigquery/python_submissions.py index 43b8201d1..f82b72d7c 100644 --- a/dbt/adapters/bigquery/python_submissions.py +++ b/dbt/adapters/bigquery/python_submissions.py @@ -121,7 +121,8 @@ def _get_job_client(self) -> dataproc_v1.BatchControllerClient: def _submit_dataproc_job(self) -> dataproc_v1.types.jobs.Job: # create the Dataproc Serverless job config - batch = dataproc_v1.Batch() + # need to pin dataproc version to 1.1 as it now defaults to 2.0 + batch = dataproc_v1.Batch({"runtime_config": dataproc_v1.RuntimeConfig(version="1.1")}) batch.pyspark_batch.main_python_file_uri = self.gcs_location # how to keep this up to date? # we should probably also open this up to be configurable