From 60f87930a70b50f4978dc8abe7dd4724ad9d383e Mon Sep 17 00:00:00 2001 From: liferoad Date: Mon, 25 Nov 2024 09:43:44 -0500 Subject: [PATCH] Use --enable-component-gateway when creating the flink cluster (#33198) * Use --enable-component-gateway when creating the flink cluster * Update flink_cluster.sh --- .test-infra/dataproc/flink_cluster.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.test-infra/dataproc/flink_cluster.sh b/.test-infra/dataproc/flink_cluster.sh index aeb5ec19441f..24d51ac13bde 100755 --- a/.test-infra/dataproc/flink_cluster.sh +++ b/.test-infra/dataproc/flink_cluster.sh @@ -131,8 +131,10 @@ function create_cluster() { # Docker init action restarts yarn so we need to start yarn session after this restart happens. # This is why flink init action is invoked last. - gcloud dataproc clusters create $CLUSTER_NAME --region=$GCLOUD_REGION --num-workers=$FLINK_NUM_WORKERS --public-ip-address \ - --metadata "${metadata}", \ + # TODO(11/22/2024) remove --worker-machine-type and --master-machine-type once N2 CPUs quota relaxed + # Dataproc 2.1 uses n2-standard-2 by default but there is N2 CPUs=24 quota limit for this project + gcloud dataproc clusters create $CLUSTER_NAME --enable-component-gateway --region=$GCLOUD_REGION --num-workers=$FLINK_NUM_WORKERS --public-ip-address \ + --master-machine-type=n1-standard-2 --worker-machine-type=n1-standard-2 --metadata "${metadata}", \ --image-version=$image_version --zone=$GCLOUD_ZONE --optional-components=FLINK,DOCKER --quiet }