From 72e52cb15465d05dea956028370938a19017db39 Mon Sep 17 00:00:00 2001
From: Alexey Romanenko <aromanenko.dev@gmail.com>
Date: Mon, 16 May 2022 15:30:56 +0200
Subject: [PATCH] [BEAM-12918] Add PostCommit_Java_Tpcds_Dataflow job

---
 .../job_PostCommit_Java_Tpcds_Dataflow.groovy | 96 +++++++++++++++++++
 .../apache/beam/sdk/tpcds/TpcdsOptions.java   |  4 +-
 2 files changed, 99 insertions(+), 1 deletion(-)
 create mode 100644 .test-infra/jenkins/job_PostCommit_Java_Tpcds_Dataflow.groovy

diff --git a/.test-infra/jenkins/job_PostCommit_Java_Tpcds_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Java_Tpcds_Dataflow.groovy
new file mode 100644
index 000000000000..0b4f4d586214
--- /dev/null
+++ b/.test-infra/jenkins/job_PostCommit_Java_Tpcds_Dataflow.groovy
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import CommonJobProperties as commonJobProperties
+import CommonTestProperties.Runner
+import CommonTestProperties.SDK
+import CommonTestProperties.TriggeringContext
+import NoPhraseTriggeringPostCommitBuilder
+import PhraseTriggeringPostCommitBuilder
+import InfluxDBCredentialsHelper
+
+
+// This job runs the suite of Tpcds tests against the Dataflow runner.
+NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Tpcds_Dataflow',
+    'Dataflow Runner Tpcds Tests', this) {
+      description('Runs the Tpcds suite on the Dataflow runner.')
+
+      // Set common parameters.
+      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
+      InfluxDBCredentialsHelper.useCredentials(delegate)
+
+      // Gradle goals for this job.
+      steps {
+        shell('echo "*** RUN TPCDS IN BATCH MODE USING DATAFLOW RUNNER ***"')
+        gradle {
+          rootBuildScriptDir(commonJobProperties.checkoutDir)
+          tasks(':sdks:java:testing:tpcds:run')
+          commonJobProperties.setGradleSwitches(delegate)
+          switches('-Ptpcds.runner=":runners:google-cloud-dataflow-java"' +
+              ' -Ptpcds.args="' +
+              [
+                '--runner=DataflowRunner',
+                '--region=us-central1',
+                '--project=apache-beam-testing',
+                '--numWorkers=4',
+                '--maxNumWorkers=4',
+                '--autoscalingAlgorithm=NONE',
+                '--dataSize=1GB',
+                '--sourceType=PARQUET',
+                '--dataDirectory=gs://beam-tpcds/datasets/parquet/partitioned',
+                '--resultsDirectory=gs://beam-tpcds/results/dataflow/',
+                '--tpcParallel=1',
+                '--queries=3'
+              ].join(' '))
+        }
+      }
+    }
+
+PhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Tpcds_Dataflow',
+    'Run Dataflow Runner Tpcds Tests', 'Dataflow Runner Tpcds Tests', this) {
+
+      description('Runs the Tpcds suite on the Dataflow runner against a Pull Request, on demand.')
+
+      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
+
+      // Gradle goals for this job.
+      steps {
+        shell('echo "*** RUN TPCDS IN BATCH MODE USING DATAFLOW RUNNER ***"')
+        gradle {
+          rootBuildScriptDir(commonJobProperties.checkoutDir)
+          tasks(':sdks:java:testing:tpcds:run')
+          commonJobProperties.setGradleSwitches(delegate)
+          switches('-Ptpcds.runner=":runners:google-cloud-dataflow-java"' +
+              ' -Ptpcds.args="' +
+              [
+                '--runner=DataflowRunner',
+                '--region=us-central1',
+                '--project=apache-beam-testing',
+                '--numWorkers=4',
+                '--maxNumWorkers=4',
+                '--autoscalingAlgorithm=NONE',
+                '--dataSize=1GB',
+                '--sourceType=PARQUET',
+                '--dataDirectory=gs://beam-tpcds/datasets/parquet/partitioned',
+                '--resultsDirectory=gs://beam-tpcds/results/dataflow/',
+                '--tpcParallel=1',
+                '--queries=3'
+              ].join(' '))
+        }
+      }
+    }
diff --git a/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TpcdsOptions.java b/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TpcdsOptions.java
index 30159991db7a..3e0e798db835 100644
--- a/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TpcdsOptions.java
+++ b/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TpcdsOptions.java
@@ -17,13 +17,15 @@
  */
 package org.apache.beam.sdk.tpcds;
 
+import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
 import org.apache.beam.sdk.extensions.sql.impl.BeamSqlPipelineOptions;
+import org.apache.beam.sdk.options.ApplicationNameOptions;
 import org.apache.beam.sdk.options.Default;
 import org.apache.beam.sdk.options.Description;
 import org.apache.beam.sdk.options.Validation;
 
 /** Options used to configure TPC-DS test. */
-public interface TpcdsOptions extends BeamSqlPipelineOptions {
+public interface TpcdsOptions extends ApplicationNameOptions, GcpOptions, BeamSqlPipelineOptions {
   @Description(
       "The size of TPC-DS data to run query on, user input should contain the unit, such as '1G', '10G'")
   @Validation.Required