Skip to content

Commit

Permalink
Merge pull request #4714: [BEAM-3456] Re-enable JDBC performance test
Browse files Browse the repository at this point in the history
  • Loading branch information
chamikaramj authored Feb 21, 2018
2 parents ce0c92e + 66c3755 commit 89afba1
Showing 1 changed file with 47 additions and 29 deletions.
76 changes: 47 additions & 29 deletions .test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -19,45 +19,63 @@
import common_job_properties

// This job runs the Beam performance tests on PerfKit Benchmarker.
job('beam_PerformanceTests_JDBC'){
job('beam_PerformanceTests_JDBC') {
// Set default Beam job properties.
common_job_properties.setTopLevelMainJobProperties(delegate)

// Run job in postcommit every 6 hours, don't trigger every push, and
// don't email individual committers.
common_job_properties.setPostCommit(
delegate,
'0 */6 * * *',
false,
'[email protected]',
false)
delegate,
'0 */6 * * *',
false,
'[email protected]',
false)

common_job_properties.enablePhraseTriggeringFromPullRequest(
delegate,
'Java JdbcIO Performance Test',
'Run Java JdbcIO Performance Test')

def pipelineArgs = [
tempRoot: 'gs://temp-storage-for-end-to-end-tests',
project: 'apache-beam-testing',
postgresServerName: '10.36.0.11',
postgresUsername: 'postgres',
postgresDatabaseName: 'postgres',
postgresPassword: 'uuinkks',
postgresSsl: 'false'
tempRoot : 'gs://temp-storage-for-perf-tests',
project : 'apache-beam-testing',
postgresPort : '5432',
numberOfRecords: '5000000'
]
def pipelineArgList = []
pipelineArgs.each({
key, value -> pipelineArgList.add("--$key=$value")
})
def pipelineArgsJoined = pipelineArgList.join(',')

def argMap = [
benchmarks: 'beam_integration_benchmark',
beam_it_module: 'sdks/java/io/jdbc',
beam_it_args: pipelineArgsJoined,
beam_it_class: 'org.apache.beam.sdk.io.jdbc.JdbcIOIT',
// Profile is located in $BEAM_ROOT/sdks/java/io/pom.xml.
beam_it_profile: 'io-it'

def testArgs = [
kubeconfig : '"$HOME/.kube/config"',
beam_it_timeout : '1800',
benchmarks : 'beam_integration_benchmark',
beam_it_profile : 'io-it',
beam_prebuilt : 'true',
beam_sdk : 'java',
beam_it_module : 'sdks/java/io/jdbc',
beam_it_class : 'org.apache.beam.sdk.io.jdbc.JdbcIOIT',
beam_it_options : joinPipelineOptions(pipelineArgs),
beam_kubernetes_scripts : makePathAbsolute('src/.test-infra/kubernetes/postgres/postgres.yml')
+ ',' + makePathAbsolute('src/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml'),
beam_options_config_file: makePathAbsolute('src/.test-infra/kubernetes/postgres/pkb-config-local.yml'),
bigquery_table : 'beam_performance.jdbcioit_pkb_results'
]

common_job_properties.buildPerformanceTest(delegate, argMap)
steps {
// create .kube/config file for perfkit (if not exists)
shell('gcloud container clusters get-credentials io-datastores --zone=us-central1-a --verbosity=debug')
}

// [BEAM-2141] Perf tests do not pass.
disabled()
common_job_properties.buildPerformanceTest(delegate, testArgs)
}

static String joinPipelineOptions(Map pipelineArgs) {
List<String> pipelineArgList = []
pipelineArgs.each({
key, value -> pipelineArgList.add("\"--$key=$value\"")
})
return "[" + pipelineArgList.join(',') + "]"
}

static String makePathAbsolute(String path) {
return '"$WORKSPACE/' + path + '"'
}

0 comments on commit 89afba1

Please sign in to comment.