From fd1b87849c05bb0e4092f80a574cbb7ab3dc667c Mon Sep 17 00:00:00 2001 From: Vitaly Terentyev Date: Sat, 21 Oct 2023 00:21:42 +0400 Subject: [PATCH] Refactor prepare arguments step for Github actions (#29090) * Refactor prepare arguments step * Rename perf/load/ pipeline options folders --- .../beam_CloudML_Benchmarks_Dataflow.yml | 2 +- ...m_Inference_Python_Benchmarks_Dataflow.yml | 10 ++--- .../beam_Java_LoadTests_Combine_Smoke.yml | 6 +-- ...beam_LoadTests_Go_CoGBK_Dataflow_Batch.yml | 8 ++-- .../beam_LoadTests_Go_CoGBK_Flink_batch.yml | 6 +-- ...am_LoadTests_Go_Combine_Dataflow_Batch.yml | 31 +++++++-------- .../beam_LoadTests_Go_Combine_Flink_Batch.yml | 6 +-- .../beam_LoadTests_Go_GBK_Dataflow_Batch.yml | 14 +++---- .../beam_LoadTests_Go_GBK_Flink_Batch.yml | 12 +++--- ...beam_LoadTests_Go_ParDo_Dataflow_Batch.yml | 8 ++-- .../beam_LoadTests_Go_ParDo_Flink_Batch.yml | 8 ++-- ..._LoadTests_Go_SideInput_Dataflow_Batch.yml | 4 +- ...eam_LoadTests_Go_SideInput_Flink_Batch.yml | 4 +- ...am_LoadTests_Java_CoGBK_Dataflow_Batch.yml | 8 ++-- ...oadTests_Java_CoGBK_Dataflow_Streaming.yml | 33 ++++++++-------- ...a_CoGBK_Dataflow_V2_Batch_JavaVersions.yml | 8 ++-- ...GBK_Dataflow_V2_Streaming_JavaVersions.yml | 8 ++-- ...a_CoGBK_SparkStructuredStreaming_Batch.yml | 8 ++-- ..._LoadTests_Java_Combine_Dataflow_Batch.yml | 6 +-- ...dTests_Java_Combine_Dataflow_Streaming.yml | 10 ++--- ...Combine_SparkStructuredStreaming_Batch.yml | 10 ++--- ...beam_LoadTests_Java_GBK_Dataflow_Batch.yml | 14 +++---- ..._LoadTests_Java_GBK_Dataflow_Streaming.yml | 14 +++---- ...ests_Java_GBK_Dataflow_V2_Batch_Java11.yml | 14 +++---- ...ests_Java_GBK_Dataflow_V2_Batch_Java17.yml | 14 +++---- ..._Java_GBK_Dataflow_V2_Streaming_Java11.yml | 14 +++---- ..._Java_GBK_Dataflow_V2_Streaming_Java17.yml | 14 +++---- .../beam_LoadTests_Java_GBK_Smoke.yml | 8 ++-- ...ava_GBK_SparkStructuredStreaming_Batch.yml | 14 +++---- ...am_LoadTests_Java_ParDo_Dataflow_Batch.yml | 13 +++---- ...oadTests_Java_ParDo_Dataflow_Streaming.yml | 13 +++---- ...a_ParDo_Dataflow_V2_Batch_JavaVersions.yml | 8 ++-- ...rDo_Dataflow_V2_Streaming_JavaVersions.yml | 8 ++-- ...a_ParDo_SparkStructuredStreaming_Batch.yml | 13 +++---- ..._LoadTests_Python_CoGBK_Dataflow_Batch.yml | 8 ++-- ...dTests_Python_CoGBK_Dataflow_Streaming.yml | 8 ++-- ...eam_LoadTests_Python_CoGBK_Flink_Batch.yml | 6 +-- ...oadTests_Python_Combine_Dataflow_Batch.yml | 31 +++++++-------- ...ests_Python_Combine_Dataflow_Streaming.yml | 6 +-- ...m_LoadTests_Python_Combine_Flink_Batch.yml | 6 +-- ...adTests_Python_Combine_Flink_Streaming.yml | 4 +- ...ests_Python_FnApiRunner_Microbenchmark.yml | 2 +- ...am_LoadTests_Python_GBK_Dataflow_Batch.yml | 10 ++--- ...oadTests_Python_GBK_Dataflow_Streaming.yml | 2 +- .../beam_LoadTests_Python_GBK_Flink_Batch.yml | 10 ++--- ...ts_Python_GBK_reiterate_Dataflow_Batch.yml | 4 +- ...ython_GBK_reiterate_Dataflow_Streaming.yml | 4 +- ..._LoadTests_Python_ParDo_Dataflow_Batch.yml | 8 ++-- ...dTests_Python_ParDo_Dataflow_Streaming.yml | 8 ++-- ...eam_LoadTests_Python_ParDo_Flink_Batch.yml | 6 +-- ...LoadTests_Python_ParDo_Flink_Streaming.yml | 10 ++--- ...dTests_Python_SideInput_Dataflow_Batch.yml | 20 +++++----- .../workflows/beam_LoadTests_Python_Smoke.yml | 4 +- .../beam_PerformanceTests_AvroIOIT.yml | 4 +- .../beam_PerformanceTests_AvroIOIT_HDFS.yml | 4 +- ...rmanceTests_BigQueryIO_Batch_Java_Avro.yml | 20 +++++----- ...rmanceTests_BigQueryIO_Batch_Java_Json.yml | 20 +++++----- ...ormanceTests_BigQueryIO_Streaming_Java.yml | 20 +++++----- ...erformanceTests_BiqQueryIO_Read_Python.yml | 4 +- ...nceTests_BiqQueryIO_Write_Python_Batch.yml | 4 +- .../workflows/beam_PerformanceTests_Cdap.yml | 4 +- ...m_PerformanceTests_Compressed_TextIOIT.yml | 2 +- ...formanceTests_Compressed_TextIOIT_HDFS.yml | 2 +- .../beam_PerformanceTests_HadoopFormat.yml | 4 +- .../workflows/beam_PerformanceTests_JDBC.yml | 4 +- .../beam_PerformanceTests_Kafka_IO.yml | 7 ++-- ...am_PerformanceTests_ManyFiles_TextIOIT.yml | 4 +- ...rformanceTests_ManyFiles_TextIOIT_HDFS.yml | 4 +- .../beam_PerformanceTests_MongoDBIO_IT.yml | 4 +- .../beam_PerformanceTests_ParquetIOIT.yml | 2 +- ...beam_PerformanceTests_ParquetIOIT_HDFS.yml | 2 +- ...manceTests_PubsubIOIT_Python_Streaming.yml | 4 +- ...formanceTests_SQLBigQueryIO_Batch_Java.yml | 4 +- ...ormanceTests_SpannerIO_Read_2GB_Python.yml | 4 +- ...Tests_SpannerIO_Write_2GB_Python_Batch.yml | 4 +- ...beam_PerformanceTests_SparkReceiver_IO.yml | 4 +- .../beam_PerformanceTests_TFRecordIOIT.yml | 4 +- ...eam_PerformanceTests_TFRecordIOIT_HDFS.yml | 4 +- .../beam_PerformanceTests_TextIOIT.yml | 2 +- .../beam_PerformanceTests_TextIOIT_HDFS.yml | 2 +- .../beam_PerformanceTests_TextIOIT_Python.yml | 2 +- ...rmanceTests_WordCountIT_PythonVersions.yml | 4 +- .../beam_PerformanceTests_XmlIOIT.yml | 4 +- .../beam_PerformanceTests_XmlIOIT_HDFS.yml | 4 +- ..._PerformanceTests_xlang_KafkaIO_Python.yml | 2 +- ..._CloudML_Benchmarks_Dataflow_arguments.txt | 0 ...rch_Imagenet_Classification_Resnet_152.txt | 0 ...Classification_Resnet_152_Tesla_T4_GPU.txt | 0 ...ch_Language_Modeling_Bert_Base_Uncased.txt | 0 ...h_Language_Modeling_Bert_Large_Uncased.txt | 0 ...torch_Vision_Classification_Resnet_101.txt | 0 .../go_CoGBK_Dataflow_Batch_MultipleKey.txt} | 0 ...CoGBK_Dataflow_Batch_Reiteration_10KB.txt} | 0 ..._CoGBK_Dataflow_Batch_Reiteration_2MB.txt} | 0 .../go_CoGBK_Dataflow_Batch_SingleKey.txt} | 0 .../go_CoGBK_Flink_Batch_MultipleKey.txt | 0 .../go_CoGBK_Flink_Batch_Reiteration_10KB.txt | 0 .../go_CoGBK_Flink_Batch_Reiteration_2MB.txt | 0 .../go_Combine_Dataflow_Batch_10b.txt} | 9 +---- .../go_Combine_Dataflow_Batch_Fanout_4.txt} | 9 +---- .../go_Combine_Dataflow_Batch_Fanout_8.txt} | 9 +---- .../go_Combine_Flink_Batch_10b.txt | 0 .../go_Combine_Flink_Batch_Fanout_4.txt | 0 .../go_Combine_Flink_Batch_Fanout_8.txt | 0 .../go_GBK_Dataflow_Batch_100b.txt | 0 .../go_GBK_Dataflow_Batch_100kb.txt | 0 .../go_GBK_Dataflow_Batch_10b.txt | 0 .../go_GBK_Dataflow_Batch_Fanout_4.txt | 0 .../go_GBK_Dataflow_Batch_Fanout_8.txt | 0 ...go_GBK_Dataflow_Batch_Reiteration_10KB.txt | 0 .../go_GBK_Dataflow_Batch_Reiteration_2MB.txt | 0 .../go_GBK_Flink_Batch_100b.txt | 0 .../go_GBK_Flink_Batch_100kb.txt | 0 .../go_GBK_Flink_Batch_10b.txt | 0 .../go_GBK_Flink_Batch_Fanout_4.txt | 0 .../go_GBK_Flink_Batch_Fanout_8.txt | 0 .../go_GBK_Flink_Batch_Reiteration_10KB.txt | 0 .../go_ParDo_Dataflow_Batch_100_Counters.txt} | 0 .../go_ParDo_Dataflow_Batch_10_Counters.txt} | 0 ...go_ParDo_Dataflow_Batch_10_Iterations.txt} | 0 ...o_ParDo_Dataflow_Batch_200_Iterations.txt} | 0 .../go_ParDo_Flink_Batch_100_counters.txt | 0 .../go_ParDo_Flink_Batch_10_counters.txt | 0 .../go_ParDo_Flink_Batch_10_times.txt | 0 .../go_ParDo_Flink_Batch_200_times.txt | 0 ...deInput_Dataflow_Batch_First_Iterable.txt} | 0 .../go_SideInput_Dataflow_Batch_Iterable.txt} | 0 ...o_SideInput_Flink_Batch_First_Iterable.txt | 0 .../go_SideInput_Flink_Batch_Iterable.txt | 0 ...oGBK_Dataflow_Batch_100b_Multiple_Keys.txt | 0 ...a_CoGBK_Dataflow_Batch_100b_Single_Key.txt | 0 .../java_CoGBK_Dataflow_Batch_10kB.txt | 0 .../java_CoGBK_Dataflow_Batch_2MB.txt | 0 ...BK_Dataflow_Streaming_2GB_MultipleKey.txt} | 8 +--- ...taflow_Streaming_2GB_Reiteration_10KB.txt} | 8 +--- ...ataflow_Streaming_2GB_Reiteration_2MB.txt} | 8 +--- ...oGBK_Dataflow_Streaming_2GB_SingleKey.txt} | 8 +--- ...w_V2_Streaming_Java_100b_Multiple_Keys.txt | 0 ...flow_V2_Streaming_Java_100b_Single_Key.txt | 0 ..._CoGBK_Dataflow_V2_Streaming_Java_10kB.txt | 0 ...a_CoGBK_Dataflow_V2_Streaming_Java_2MB.txt | 0 ...uredStreaming_Batch_100b_Multiple_Keys.txt | 0 ...ucturedStreaming_Batch_100b_Single_Key.txt | 0 ...BK_SparkStructuredStreaming_Batch_10kB.txt | 0 ...GBK_SparkStructuredStreaming_Batch_2MB.txt | 0 .../java_Combine_Dataflow_Batch_10b.txt} | 0 .../java_Combine_Dataflow_Batch_Fanout_4.txt} | 0 .../java_Combine_Dataflow_Batch_Fanout_8.txt} | 0 .../java_Combine_Dataflow_Streaming_10b.txt | 0 ...va_Combine_Dataflow_Streaming_Fanout_4.txt | 0 ...va_Combine_Dataflow_Streaming_Fanout_8.txt | 0 ...ine_SparkStructuredStreaming_Batch_10b.txt | 0 ...parkStructuredStreaming_Batch_Fanout_4.txt | 0 ...parkStructuredStreaming_Batch_Fanout_8.txt | 0 ...GBK_Dataflow_Batch_2GB_of_100B_records.txt | 0 ...BK_Dataflow_Batch_2GB_of_100kB_records.txt | 0 ..._GBK_Dataflow_Batch_2GB_of_10B_records.txt | 0 ...4_times_with_2GB_10-byte_records_total.txt | 0 ...8_times_with_2GB_10-byte_records_total.txt | 0 ...ow_Batch_reiterate_4_times_10kB_values.txt | 0 ...low_Batch_reiterate_4_times_2MB_values.txt | 0 ...Dataflow_Streaming_2GB_of_100B_records.txt | 0 ...ataflow_Streaming_2GB_of_100kB_records.txt | 0 ..._Dataflow_Streaming_2GB_of_10B_records.txt | 0 ...4_times_with_2GB_10-byte_records_total.txt | 0 ...8_times_with_2GB_10-byte_records_total.txt | 0 ...treaming_reiterate_4_times_10kB_values.txt | 0 ...Streaming_reiterate_4_times_2MB_values.txt | 0 ...ow_V2_Batch_Java11_2GB_of_100B_records.txt | 0 ...w_V2_Batch_Java11_2GB_of_100kB_records.txt | 0 ...low_V2_Batch_Java11_2GB_of_10B_records.txt | 0 ...4_times_with_2GB_10-byte_records_total.txt | 0 ...8_times_with_2GB_10-byte_records_total.txt | 0 ...h_Java11_reiterate_4_times_10kB_values.txt | 0 ...ch_Java11_reiterate_4_times_2MB_values.txt | 0 ...ow_V2_Batch_Java17_2GB_of_100B_records.txt | 0 ...w_V2_Batch_Java17_2GB_of_100kB_records.txt | 0 ...low_V2_Batch_Java17_2GB_of_10B_records.txt | 0 ...4_times_with_2GB_10-byte_records_total.txt | 0 ...8_times_with_2GB_10-byte_records_total.txt | 0 ...h_Java17_reiterate_4_times_10kB_values.txt | 0 ...ch_Java17_reiterate_4_times_2MB_values.txt | 0 ...2_Streaming_Java11_2GB_of_100B_records.txt | 0 ..._Streaming_Java11_2GB_of_100kB_records.txt | 0 ...V2_Streaming_Java11_2GB_of_10B_records.txt | 0 ...4_times_with_2GB_10-byte_records_total.txt | 0 ...8_times_with_2GB_10-byte_records_total.txt | 0 ...g_Java11_reiterate_4_times_10kB_values.txt | 0 ...ng_Java11_reiterate_4_times_2MB_values.txt | 0 ...2_Streaming_Java17_2GB_of_100B_records.txt | 0 ..._Streaming_Java17_2GB_of_100kB_records.txt | 0 ...V2_Streaming_Java17_2GB_of_10B_records.txt | 0 ...4_times_with_2GB_10-byte_records_total.txt | 0 ...8_times_with_2GB_10-byte_records_total.txt | 0 ...g_Java17_reiterate_4_times_10kB_values.txt | 0 ...ng_Java17_reiterate_4_times_2MB_values.txt | 0 ...edStreaming_Batch_2GB_of_100B_records.txt} | 0 ...dStreaming_Batch_2GB_of_100kB_records.txt} | 0 ...redStreaming_Batch_2GB_of_10B_records.txt} | 0 ..._times_with_2GB_10-byte_records_total.txt} | 0 ..._times_with_2GB_10-byte_records_total.txt} | 0 ...g_Batch_reiterate_4_times_10kB_values.txt} | 0 ...ng_Batch_reiterate_4_times_2MB_values.txt} | 0 ...e_CombineLoadTest_load_test_Dataflow-1.txt | 0 ...e_CombineLoadTest_load_test_Dataflow-2.txt | 0 ...e_CombineLoadTest_load_test_Dataflow-3.txt | 0 ...java_ParDo_Dataflow_Batch_100_counters.txt | 0 .../java_ParDo_Dataflow_Batch_10_counters.txt | 0 .../java_ParDo_Dataflow_Batch_10_times.txt | 0 .../java_ParDo_Dataflow_Batch_200_times.txt | 0 ..._ParDo_Dataflow_Streaming_100_counters.txt | 0 ...a_ParDo_Dataflow_Streaming_10_counters.txt | 0 ...java_ParDo_Dataflow_Streaming_10_times.txt | 0 ...ava_ParDo_Dataflow_Streaming_200_times.txt | 0 ...low_V2_Batch_JavaVersions_100_counters.txt | 0 ...flow_V2_Batch_JavaVersions_10_counters.txt | 0 ...ataflow_V2_Batch_JavaVersions_10_times.txt | 0 ...taflow_V2_Batch_JavaVersions_200_times.txt | 0 ...V2_Streaming_JavaVersions_100_counters.txt | 0 ..._V2_Streaming_JavaVersions_10_counters.txt | 0 ...low_V2_Streaming_JavaVersions_10_times.txt | 0 ...ow_V2_Streaming_JavaVersions_200_times.txt | 0 ...StructuredStreaming_Batch_100_counters.txt | 0 ...kStructuredStreaming_Batch_10_counters.txt | 0 ...parkStructuredStreaming_Batch_10_times.txt | 0 ...arkStructuredStreaming_Batch_200_times.txt | 0 .../java_Smoke_GroupByKey_Dataflow.txt | 0 .../java_Smoke_GroupByKey_Direct.txt | 0 .../java_Smoke_GroupByKey_Flink.txt | 0 .../java_Smoke_GroupByKey_Spark.txt | 0 ...oGBK_Dataflow_Batch_100b_Multiple_Keys.txt | 0 ...n_CoGBK_Dataflow_Batch_100b_Single_Key.txt | 0 .../python_CoGBK_Dataflow_Batch_10kB.txt | 0 .../python_CoGBK_Dataflow_Batch_2MB.txt | 0 ...ataflow_Flink_Batch_100b_Multiple_Keys.txt | 0 ...K_Dataflow_Flink_Batch_100b_Single_Key.txt | 0 ...python_CoGBK_Dataflow_Flink_Batch_10kB.txt | 0 ..._Dataflow_Streaming_100b_Multiple_Keys.txt | 0 ...GBK_Dataflow_Streaming_100b_Single_Key.txt | 0 .../python_CoGBK_Dataflow_Streaming_10kB.txt | 0 .../python_CoGBK_Dataflow_Streaming_2MB.txt | 0 ...python_Combine_Dataflow_Batch_2GB_10b.txt} | 9 +---- ...n_Combine_Dataflow_Batch_2GB_Fanout_4.txt} | 7 +--- ...n_Combine_Dataflow_Batch_2GB_Fanout_8.txt} | 7 +--- ...Dataflow_Streaming_2GB_10_byte_records.txt | 0 ...ombine_Dataflow_Streaming_2GB_Fanout_4.txt | 0 ...ombine_Dataflow_Streaming_2GB_Fanout_8.txt | 0 ...ombine_Flink_Batch_2GB_10_byte_records.txt | 0 ...ython_Combine_Flink_Batch_2GB_Fanout_4.txt | 0 ...ython_Combine_Flink_Batch_2GB_Fanout_8.txt | 0 ...ne_Flink_Streaming_2GB_10_byte_records.txt | 0 ...n_Combine_Flink_Streaming_2GB_Fanout_4.txt | 0 ...n_Combine_Flink_Streaming_2GB_Fanout_8.txt | 0 .../python_FnApiRunner_Microbenchmark.txt} | 0 ...GBK_Dataflow_Batch_2GB_of_100B_records.txt | 0 ...BK_Dataflow_Batch_2GB_of_100kB_records.txt | 0 ..._GBK_Dataflow_Batch_2GB_of_10B_records.txt | 0 ...4_times_with_2GB_10-byte_records_total.txt | 0 ...8_times_with_2GB_10-byte_records_total.txt | 0 ...Dataflow_Streaming_2GB_of_100B_records.txt | 0 ...ataflow_Streaming_2GB_of_100kB_records.txt | 0 ..._Dataflow_Streaming_2GB_of_10B_records.txt | 0 ...4_times_with_2GB_10-byte_records_total.txt | 0 ...8_times_with_2GB_10-byte_records_total.txt | 0 ...on_GBK_Flink_Batch_2GB_of_100B_records.txt | 0 ...hon_GBK_Flink_Batch_2GB_of_10B_records.txt | 0 ...4_times_with_2GB_10-byte_records_total.txt | 0 ...8_times_with_2GB_10-byte_records_total.txt | 0 ...nk_Batch_reiterate_4_times_10kB_values.txt | 0 ...ow_Batch_reiterate_4_times_10kB_values.txt | 0 ...low_Batch_reiterate_4_times_2MB_values.txt | 0 ...treaming_reiterate_4_times_10kB_values.txt | 0 ...Streaming_reiterate_4_times_2MB_values.txt | 0 ...thon_ParDo_Dataflow_Batch_100_Counters.txt | 0 ...ython_ParDo_Dataflow_Batch_10_Counters.txt | 0 ...hon_ParDo_Dataflow_Batch_10_Iterations.txt | 0 ...on_ParDo_Dataflow_Batch_200_Iterations.txt | 0 ..._ParDo_Dataflow_Streaming_100_Counters.txt | 0 ...n_ParDo_Dataflow_Streaming_10_Counters.txt | 0 ...ParDo_Dataflow_Streaming_10_Iterations.txt | 0 ...arDo_Dataflow_Streaming_200_Iterations.txt | 0 .../python_ParDo_Flink_Batch_10_Counters.txt | 0 ...python_ParDo_Flink_Batch_10_Iterations.txt | 0 ...ython_ParDo_Flink_Batch_200_Iterations.txt | 0 ...hon_ParDo_Flink_Streaming_100_Counters.txt | 0 ...thon_ParDo_Flink_Streaming_10_Counters.txt | 0 ...on_ParDo_Flink_Streaming_10_Iterations.txt | 0 ...n_ParDo_Flink_Streaming_200_Iterations.txt | 0 ...hon_ParDo_Flink_Streaming_5_Iterations.txt | 0 ...w_Batch_10gb_1000window_first_iterable.txt | 0 ...ataflow_Batch_10gb_1000window_iterable.txt | 0 ...flow_Batch_10gb_1window_first_iterable.txt | 0 ...t_Dataflow_Batch_10gb_1window_iterable.txt | 0 ...Batch_1gb_1000window_1key_percent_dict.txt | 0 ...atch_1gb_1000window_99key_percent_dict.txt | 0 ...ow_Batch_1gb_1window_1key_percent_dict.txt | 0 ...w_Batch_1gb_1window_99key_percent_dict.txt | 0 ..._Dataflow_Batch_1gb_1window_first_list.txt | 0 ...eInput_Dataflow_Batch_1gb_1window_list.txt | 0 .../python_Smoke_GroupByKey_Dataflow.txt | 0 .../python_Smoke_GroupByKey_Direct.txt | 0 .../config_BigQueryIO_Batch_Java_Avro.txt | 38 ------------------ .../config_BigQueryIO_Batch_Java_Json.txt | 38 ------------------ .../config_BigQueryIO_Streaming_Java.txt | 39 ------------------- .../JDBC.txt | 0 .../SQLBigQueryIO_Batch_Java.txt | 0 .../TFRecordIOIT.txt} | 0 .../TFRecordIOIT_HDFS.txt | 0 .../avroIOIT.txt} | 0 .../avroIOIT_HDFS.txt} | 0 .../bigQueryIO_Batch_Java_Avro.txt | 29 ++++++++++++++ .../bigQueryIO_Batch_Java_Json.txt | 29 ++++++++++++++ .../bigQueryIO_Streaming_Java.txt | 30 ++++++++++++++ .../biqQueryIO_Read_Python.txt | 0 .../biqQueryIO_Write_Python_Batch.txt | 0 .../cdap.txt | 0 .../compressed_TextIOIT.txt} | 0 .../compressed_TextIOIT_HDFS.txt} | 0 .../hadoopFormat.txt | 0 .../kafka_IO_Batch.txt | 0 .../kafka_IO_Streaming.txt | 0 .../manyFiles_TextIOIT.txt} | 0 .../manyFiles_TextIOIT_HDFS.txt} | 0 .../mongoDBIO_IT.txt | 0 .../parquetIOIT.txt} | 0 .../parquetIOIT_HDFS.txt} | 0 .../pubsubIOIT_Python_Streaming.txt | 0 .../spannerIO_Read_2GB_Python.txt | 0 .../spannerIO_Write_2GB_Python.txt | 0 .../sparkReceiver_IO.txt | 0 .../textIOIT.txt | 0 .../textIOIT_HDFS.txt | 0 .../textIOIT_Python.txt | 0 .../wordCountIT_Python.txt | 0 .../xlang_KafkaIO_Python.txt | 0 .../xmlIOIT.txt} | 0 .../xmlIOIT_HDFS.txt} | 0 337 files changed, 446 insertions(+), 530 deletions(-) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/beam_CloudML_Benchmarks_Dataflow_arguments.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152_Tesla_T4_GPU.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Base_Uncased.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Large_Uncased.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Vision_Classification_Resnet_101.txt (100%) rename .github/workflows/{load-tests-job-configs/config_CoGBK_Go_Batch_MultipleKey.txt => load-tests-pipeline-options/go_CoGBK_Dataflow_Batch_MultipleKey.txt} (100%) rename .github/workflows/{load-tests-job-configs/config_CoGBK_Go_Batch_Reiteration_10KB.txt => load-tests-pipeline-options/go_CoGBK_Dataflow_Batch_Reiteration_10KB.txt} (100%) rename .github/workflows/{load-tests-job-configs/config_CoGBK_Go_Batch_Reiteration_2MB.txt => load-tests-pipeline-options/go_CoGBK_Dataflow_Batch_Reiteration_2MB.txt} (100%) rename .github/workflows/{load-tests-job-configs/config_CoGBK_Go_Batch_SingleKey.txt => load-tests-pipeline-options/go_CoGBK_Dataflow_Batch_SingleKey.txt} (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_CoGBK_Flink_Batch_MultipleKey.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_CoGBK_Flink_Batch_Reiteration_10KB.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_CoGBK_Flink_Batch_Reiteration_2MB.txt (100%) rename .github/workflows/{load-tests-job-configs/config_Combine_Go_Batch_10b.txt => load-tests-pipeline-options/go_Combine_Dataflow_Batch_10b.txt} (78%) rename .github/workflows/{load-tests-job-configs/config_Combine_Go_Batch_Fanout_4.txt => load-tests-pipeline-options/go_Combine_Dataflow_Batch_Fanout_4.txt} (78%) rename .github/workflows/{load-tests-job-configs/config_Combine_Go_Batch_Fanout_8.txt => load-tests-pipeline-options/go_Combine_Dataflow_Batch_Fanout_8.txt} (78%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_Combine_Flink_Batch_10b.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_Combine_Flink_Batch_Fanout_4.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_Combine_Flink_Batch_Fanout_8.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_GBK_Dataflow_Batch_100b.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_GBK_Dataflow_Batch_100kb.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_GBK_Dataflow_Batch_10b.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_GBK_Dataflow_Batch_Fanout_4.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_GBK_Dataflow_Batch_Fanout_8.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_GBK_Dataflow_Batch_Reiteration_10KB.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_GBK_Dataflow_Batch_Reiteration_2MB.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_GBK_Flink_Batch_100b.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_GBK_Flink_Batch_100kb.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_GBK_Flink_Batch_10b.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_GBK_Flink_Batch_Fanout_4.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_GBK_Flink_Batch_Fanout_8.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_GBK_Flink_Batch_Reiteration_10KB.txt (100%) rename .github/workflows/{load-tests-job-configs/config_ParDo_Go_Batch_100_Counters.txt => load-tests-pipeline-options/go_ParDo_Dataflow_Batch_100_Counters.txt} (100%) rename .github/workflows/{load-tests-job-configs/config_ParDo_Go_Batch_10_Counters.txt => load-tests-pipeline-options/go_ParDo_Dataflow_Batch_10_Counters.txt} (100%) rename .github/workflows/{load-tests-job-configs/config_ParDo_Go_Batch_10_Iterations.txt => load-tests-pipeline-options/go_ParDo_Dataflow_Batch_10_Iterations.txt} (100%) rename .github/workflows/{load-tests-job-configs/config_ParDo_Go_Batch_200_Iterations.txt => load-tests-pipeline-options/go_ParDo_Dataflow_Batch_200_Iterations.txt} (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_ParDo_Flink_Batch_100_counters.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_ParDo_Flink_Batch_10_counters.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_ParDo_Flink_Batch_10_times.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_ParDo_Flink_Batch_200_times.txt (100%) rename .github/workflows/{load-tests-job-configs/config_SideInput_Go_Batch_Dataflow_First_Iterable.txt => load-tests-pipeline-options/go_SideInput_Dataflow_Batch_First_Iterable.txt} (100%) rename .github/workflows/{load-tests-job-configs/config_SideInput_Go_Batch_Dataflow_Iterable.txt => load-tests-pipeline-options/go_SideInput_Dataflow_Batch_Iterable.txt} (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_SideInput_Flink_Batch_First_Iterable.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/go_SideInput_Flink_Batch_Iterable.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_CoGBK_Dataflow_Batch_100b_Single_Key.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_CoGBK_Dataflow_Batch_10kB.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_CoGBK_Dataflow_Batch_2MB.txt (100%) rename .github/workflows/{load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_MultipleKey.txt => load-tests-pipeline-options/java_CoGBK_Dataflow_Streaming_2GB_MultipleKey.txt} (81%) rename .github/workflows/{load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_Reiteration_10KB.txt => load-tests-pipeline-options/java_CoGBK_Dataflow_Streaming_2GB_Reiteration_10KB.txt} (81%) rename .github/workflows/{load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_Reiteration_2MB.txt => load-tests-pipeline-options/java_CoGBK_Dataflow_Streaming_2GB_Reiteration_2MB.txt} (81%) rename .github/workflows/{load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_SingleKey.txt => load-tests-pipeline-options/java_CoGBK_Dataflow_Streaming_2GB_SingleKey.txt} (81%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_CoGBK_Dataflow_V2_Streaming_Java_100b_Multiple_Keys.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_CoGBK_Dataflow_V2_Streaming_Java_100b_Single_Key.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_CoGBK_Dataflow_V2_Streaming_Java_10kB.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_CoGBK_Dataflow_V2_Streaming_Java_2MB.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_CoGBK_SparkStructuredStreaming_Batch_100b_Multiple_Keys.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_CoGBK_SparkStructuredStreaming_Batch_100b_Single_Key.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_CoGBK_SparkStructuredStreaming_Batch_10kB.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_CoGBK_SparkStructuredStreaming_Batch_2MB.txt (100%) rename .github/workflows/{load-tests-job-configs/config_Combine_Java_Dataflow_Batch_10b.txt => load-tests-pipeline-options/java_Combine_Dataflow_Batch_10b.txt} (100%) rename .github/workflows/{load-tests-job-configs/config_Combine_Java_Dataflow_Batch_Fanout_4.txt => load-tests-pipeline-options/java_Combine_Dataflow_Batch_Fanout_4.txt} (100%) rename .github/workflows/{load-tests-job-configs/config_Combine_Java_Dataflow_Batch_Fanout_8.txt => load-tests-pipeline-options/java_Combine_Dataflow_Batch_Fanout_8.txt} (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_Combine_Dataflow_Streaming_10b.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_Combine_Dataflow_Streaming_Fanout_4.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_Combine_Dataflow_Streaming_Fanout_8.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_Combine_SparkStructuredStreaming_Batch_10b.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_Combine_SparkStructuredStreaming_Batch_Fanout_4.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_Combine_SparkStructuredStreaming_Batch_Fanout_8.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_Batch_2GB_of_100B_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_Batch_2GB_of_100kB_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_Batch_2GB_of_10B_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_Batch_reiterate_4_times_10kB_values.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_Batch_reiterate_4_times_2MB_values.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_Streaming_2GB_of_100B_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_Streaming_2GB_of_10B_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_Streaming_reiterate_4_times_10kB_values.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_Streaming_reiterate_4_times_2MB_values.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100B_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100kB_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_10B_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Batch_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Batch_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_10kB_values.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_2MB_values.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100B_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100kB_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_10B_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Batch_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Batch_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_10kB_values.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_2MB_values.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100B_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100kB_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_10B_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Streaming_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Streaming_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_10kB_values.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_2MB_values.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100B_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100kB_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_10B_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Streaming_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Streaming_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_10kB_values.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_2MB_values.txt (100%) rename .github/workflows/{load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_100B_records.txt => load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_2GB_of_100B_records.txt} (100%) rename .github/workflows/{load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_100kB_records.txt => load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_2GB_of_100kB_records.txt} (100%) rename .github/workflows/{load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_10B_records.txt => load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_2GB_of_10B_records.txt} (100%) rename .github/workflows/{load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt => load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt} (100%) rename .github/workflows/{load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt => load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt} (100%) rename .github/workflows/{load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_10kB_values.txt => load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_10kB_values.txt} (100%) rename .github/workflows/{load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_2MB_values.txt => load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_2MB_values.txt} (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-1.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-2.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-3.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_ParDo_Dataflow_Batch_100_counters.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_ParDo_Dataflow_Batch_10_counters.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_ParDo_Dataflow_Batch_10_times.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_ParDo_Dataflow_Batch_200_times.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_ParDo_Dataflow_Streaming_100_counters.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_ParDo_Dataflow_Streaming_10_counters.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_ParDo_Dataflow_Streaming_10_times.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_ParDo_Dataflow_Streaming_200_times.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_ParDo_Dataflow_V2_Batch_JavaVersions_100_counters.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_counters.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_times.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_ParDo_Dataflow_V2_Batch_JavaVersions_200_times.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_ParDo_Dataflow_V2_Streaming_JavaVersions_100_counters.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_counters.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_times.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_ParDo_Dataflow_V2_Streaming_JavaVersions_200_times.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_ParDo_SparkStructuredStreaming_Batch_100_counters.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_ParDo_SparkStructuredStreaming_Batch_10_counters.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_ParDo_SparkStructuredStreaming_Batch_10_times.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_ParDo_SparkStructuredStreaming_Batch_200_times.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_Smoke_GroupByKey_Dataflow.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_Smoke_GroupByKey_Direct.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_Smoke_GroupByKey_Flink.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/java_Smoke_GroupByKey_Spark.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_CoGBK_Dataflow_Batch_100b_Single_Key.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_CoGBK_Dataflow_Batch_10kB.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_CoGBK_Dataflow_Batch_2MB.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_CoGBK_Dataflow_Flink_Batch_100b_Multiple_Keys.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_CoGBK_Dataflow_Flink_Batch_100b_Single_Key.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_CoGBK_Dataflow_Flink_Batch_10kB.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_CoGBK_Dataflow_Streaming_100b_Multiple_Keys.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_CoGBK_Dataflow_Streaming_100b_Single_Key.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_CoGBK_Dataflow_Streaming_10kB.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_CoGBK_Dataflow_Streaming_2MB.txt (100%) rename .github/workflows/{load-tests-job-configs/config_Combine_Python_Batch_2GB_10b.txt => load-tests-pipeline-options/python_Combine_Dataflow_Batch_2GB_10b.txt} (77%) rename .github/workflows/{load-tests-job-configs/config_Combine_Python_Batch_2GB_Fanout_4.txt => load-tests-pipeline-options/python_Combine_Dataflow_Batch_2GB_Fanout_4.txt} (79%) rename .github/workflows/{load-tests-job-configs/config_Combine_Python_Batch_2GB_Fanout_8.txt => load-tests-pipeline-options/python_Combine_Dataflow_Batch_2GB_Fanout_8.txt} (79%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_Combine_Dataflow_Streaming_2GB_10_byte_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_Combine_Dataflow_Streaming_2GB_Fanout_4.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_Combine_Dataflow_Streaming_2GB_Fanout_8.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_Combine_Flink_Batch_2GB_10_byte_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_Combine_Flink_Batch_2GB_Fanout_4.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_Combine_Flink_Batch_2GB_Fanout_8.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_Combine_Flink_Streaming_2GB_10_byte_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_Combine_Flink_Streaming_2GB_Fanout_4.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_Combine_Flink_Streaming_2GB_Fanout_8.txt (100%) rename .github/workflows/{load-tests-job-configs/config_FnApiRunner_Python_Microbenchmark.txt => load-tests-pipeline-options/python_FnApiRunner_Microbenchmark.txt} (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_GBK_Dataflow_Batch_2GB_of_100B_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_GBK_Dataflow_Batch_2GB_of_100kB_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_GBK_Dataflow_Batch_2GB_of_10B_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_GBK_Dataflow_Streaming_2GB_of_100B_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_GBK_Dataflow_Streaming_2GB_of_10B_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_GBK_Flink_Batch_2GB_of_100B_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_GBK_Flink_Batch_2GB_of_10B_records.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_10kB_values.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_2MB_values.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_10kB_values.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_2MB_values.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_ParDo_Dataflow_Batch_100_Counters.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_ParDo_Dataflow_Batch_10_Counters.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_ParDo_Dataflow_Batch_10_Iterations.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_ParDo_Dataflow_Batch_200_Iterations.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_ParDo_Dataflow_Streaming_100_Counters.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_ParDo_Dataflow_Streaming_10_Counters.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_ParDo_Dataflow_Streaming_10_Iterations.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_ParDo_Dataflow_Streaming_200_Iterations.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_ParDo_Flink_Batch_10_Counters.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_ParDo_Flink_Batch_10_Iterations.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_ParDo_Flink_Batch_200_Iterations.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_ParDo_Flink_Streaming_100_Counters.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_ParDo_Flink_Streaming_10_Counters.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_ParDo_Flink_Streaming_10_Iterations.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_ParDo_Flink_Streaming_200_Iterations.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_ParDo_Flink_Streaming_5_Iterations.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_SideInput_Dataflow_Batch_10gb_1000window_first_iterable.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_SideInput_Dataflow_Batch_10gb_1000window_iterable.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_SideInput_Dataflow_Batch_10gb_1window_first_iterable.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_SideInput_Dataflow_Batch_10gb_1window_iterable.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_SideInput_Dataflow_Batch_1gb_1000window_1key_percent_dict.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_SideInput_Dataflow_Batch_1gb_1000window_99key_percent_dict.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_SideInput_Dataflow_Batch_1gb_1window_1key_percent_dict.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_SideInput_Dataflow_Batch_1gb_1window_99key_percent_dict.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_SideInput_Dataflow_Batch_1gb_1window_first_list.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_SideInput_Dataflow_Batch_1gb_1window_list.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_Smoke_GroupByKey_Dataflow.txt (100%) rename .github/workflows/{load-tests-job-configs => load-tests-pipeline-options}/python_Smoke_GroupByKey_Direct.txt (100%) delete mode 100644 .github/workflows/performance-tests-job-configs/config_BigQueryIO_Batch_Java_Avro.txt delete mode 100644 .github/workflows/performance-tests-job-configs/config_BigQueryIO_Batch_Java_Json.txt delete mode 100644 .github/workflows/performance-tests-job-configs/config_BigQueryIO_Streaming_Java.txt rename .github/workflows/{performance-tests-job-configs => performance-tests-pipeline-options}/JDBC.txt (100%) rename .github/workflows/{performance-tests-job-configs => performance-tests-pipeline-options}/SQLBigQueryIO_Batch_Java.txt (100%) rename .github/workflows/{performance-tests-job-configs/config_TFRecordIOIT.txt => performance-tests-pipeline-options/TFRecordIOIT.txt} (100%) rename .github/workflows/{performance-tests-job-configs => performance-tests-pipeline-options}/TFRecordIOIT_HDFS.txt (100%) rename .github/workflows/{performance-tests-job-configs/config_AvroIOIT.txt => performance-tests-pipeline-options/avroIOIT.txt} (100%) rename .github/workflows/{performance-tests-job-configs/config_AvroIOIT_HDFS.txt => performance-tests-pipeline-options/avroIOIT_HDFS.txt} (100%) create mode 100644 .github/workflows/performance-tests-pipeline-options/bigQueryIO_Batch_Java_Avro.txt create mode 100644 .github/workflows/performance-tests-pipeline-options/bigQueryIO_Batch_Java_Json.txt create mode 100644 .github/workflows/performance-tests-pipeline-options/bigQueryIO_Streaming_Java.txt rename .github/workflows/{performance-tests-job-configs => performance-tests-pipeline-options}/biqQueryIO_Read_Python.txt (100%) rename .github/workflows/{performance-tests-job-configs => performance-tests-pipeline-options}/biqQueryIO_Write_Python_Batch.txt (100%) rename .github/workflows/{performance-tests-job-configs => performance-tests-pipeline-options}/cdap.txt (100%) rename .github/workflows/{performance-tests-job-configs/config_Compressed_TextIOIT.txt => performance-tests-pipeline-options/compressed_TextIOIT.txt} (100%) rename .github/workflows/{performance-tests-job-configs/config_Compressed_TextIOIT_HDFS.txt => performance-tests-pipeline-options/compressed_TextIOIT_HDFS.txt} (100%) rename .github/workflows/{performance-tests-job-configs => performance-tests-pipeline-options}/hadoopFormat.txt (100%) rename .github/workflows/{performance-tests-job-configs => performance-tests-pipeline-options}/kafka_IO_Batch.txt (100%) rename .github/workflows/{performance-tests-job-configs => performance-tests-pipeline-options}/kafka_IO_Streaming.txt (100%) rename .github/workflows/{performance-tests-job-configs/config_ManyFiles_TextIOIT.txt => performance-tests-pipeline-options/manyFiles_TextIOIT.txt} (100%) rename .github/workflows/{performance-tests-job-configs/config_ManyFiles_TextIOIT_HDFS.txt => performance-tests-pipeline-options/manyFiles_TextIOIT_HDFS.txt} (100%) rename .github/workflows/{performance-tests-job-configs => performance-tests-pipeline-options}/mongoDBIO_IT.txt (100%) rename .github/workflows/{performance-tests-job-configs/config_ParquetIOIT.txt => performance-tests-pipeline-options/parquetIOIT.txt} (100%) rename .github/workflows/{performance-tests-job-configs/config_ParquetIOIT_HDFS.txt => performance-tests-pipeline-options/parquetIOIT_HDFS.txt} (100%) rename .github/workflows/{performance-tests-job-configs => performance-tests-pipeline-options}/pubsubIOIT_Python_Streaming.txt (100%) rename .github/workflows/{performance-tests-job-configs => performance-tests-pipeline-options}/spannerIO_Read_2GB_Python.txt (100%) rename .github/workflows/{performance-tests-job-configs => performance-tests-pipeline-options}/spannerIO_Write_2GB_Python.txt (100%) rename .github/workflows/{performance-tests-job-configs => performance-tests-pipeline-options}/sparkReceiver_IO.txt (100%) rename .github/workflows/{performance-tests-job-configs => performance-tests-pipeline-options}/textIOIT.txt (100%) rename .github/workflows/{performance-tests-job-configs => performance-tests-pipeline-options}/textIOIT_HDFS.txt (100%) rename .github/workflows/{performance-tests-job-configs => performance-tests-pipeline-options}/textIOIT_Python.txt (100%) rename .github/workflows/{performance-tests-job-configs => performance-tests-pipeline-options}/wordCountIT_Python.txt (100%) rename .github/workflows/{performance-tests-job-configs => performance-tests-pipeline-options}/xlang_KafkaIO_Python.txt (100%) rename .github/workflows/{performance-tests-job-configs/config_XmlIOIT.txt => performance-tests-pipeline-options/xmlIOIT.txt} (100%) rename .github/workflows/{performance-tests-job-configs/config_XmlIOIT_HDFS.txt => performance-tests-pipeline-options/xmlIOIT_HDFS.txt} (100%) diff --git a/.github/workflows/beam_CloudML_Benchmarks_Dataflow.yml b/.github/workflows/beam_CloudML_Benchmarks_Dataflow.yml index df26f1a5cfc2..fceae349a08d 100644 --- a/.github/workflows/beam_CloudML_Benchmarks_Dataflow.yml +++ b/.github/workflows/beam_CloudML_Benchmarks_Dataflow.yml @@ -83,7 +83,7 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/beam_CloudML_Benchmarks_Dataflow_arguments.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_CloudML_Benchmarks_Dataflow_arguments.txt # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run TFT Criteo Benchmarks uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml b/.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml index 97099ec525f1..16ef684b0589 100644 --- a/.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml +++ b/.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml @@ -81,11 +81,11 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Vision_Classification_Resnet_101.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Base_Uncased.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Large_Uncased.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152_Tesla_T4_GPU.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Vision_Classification_Resnet_101.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Base_Uncased.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Large_Uncased.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152_Tesla_T4_GPU.txt # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: get current time run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV diff --git a/.github/workflows/beam_Java_LoadTests_Combine_Smoke.yml b/.github/workflows/beam_Java_LoadTests_Combine_Smoke.yml index 2eb7f061ce99..19d5b0f7174f 100644 --- a/.github/workflows/beam_Java_LoadTests_Combine_Smoke.yml +++ b/.github/workflows/beam_Java_LoadTests_Combine_Smoke.yml @@ -75,9 +75,9 @@ jobs: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-1.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-2.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-3.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-1.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-2.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-3.txt # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run CombineLoadTest load test Dataflow-1 uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_LoadTests_Go_CoGBK_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Go_CoGBK_Dataflow_Batch.yml index 4a7b6949ee44..dac3733da4a9 100644 --- a/.github/workflows/beam_LoadTests_Go_CoGBK_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Go_CoGBK_Dataflow_Batch.yml @@ -75,10 +75,10 @@ jobs: test-type: load test-language: go argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_SingleKey.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_MultipleKey.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_Reiteration_10KB.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_Reiteration_2MB.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_CoGBK_Dataflow_Batch_SingleKey.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_CoGBK_Dataflow_Batch_MultipleKey.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_CoGBK_Dataflow_Batch_Reiteration_10KB.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_CoGBK_Dataflow_Batch_Reiteration_2MB.txt arguments: | --job_name=load-tests-go-dataflow-batch-cogbk-$(date '+%m%d%H%M%S' --utc) - name: run CoGBK Dataflow Batch Go Load Test 1 (single key) diff --git a/.github/workflows/beam_LoadTests_Go_CoGBK_Flink_batch.yml b/.github/workflows/beam_LoadTests_Go_CoGBK_Flink_batch.yml index f3a4cba5fce1..05b61f285cd7 100644 --- a/.github/workflows/beam_LoadTests_Go_CoGBK_Flink_batch.yml +++ b/.github/workflows/beam_LoadTests_Go_CoGBK_Flink_batch.yml @@ -85,9 +85,9 @@ jobs: test-type: load test-language: go argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_MultipleKey.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_Reiteration_10KB.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_Reiteration_2MB.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_MultipleKey.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_Reiteration_10KB.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_Reiteration_2MB.txt arguments: | --job_name=load-tests-go-flink-batch-cogbk-$(date '+%m%d%H%M%S' --utc) - name: Start Flink with parallelism 5 diff --git a/.github/workflows/beam_LoadTests_Go_Combine_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Go_Combine_Dataflow_Batch.yml index 9568d0cfcff6..cb198bbc233f 100644 --- a/.github/workflows/beam_LoadTests_Go_Combine_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Go_Combine_Dataflow_Batch.yml @@ -69,19 +69,18 @@ jobs: comment_phrase: ${{ matrix.job_phrase }} github_token: ${{ secrets.GITHUB_TOKEN }} github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - - name: Prepare configs - #Reads config files, excludes comments, appends current date to the job_name parameter - id: set_configs - shell: bash - run: | - CURDATE=$(date '+%m%d%H%M%S' --utc) - CONFIG_ARR=('config_Combine_Go_Batch_10b.txt' 'config_Combine_Go_Batch_Fanout_4.txt' 'config_Combine_Go_Batch_Fanout_8.txt') - for INDEX in ${!CONFIG_ARR[@]} - do - CURCONFIG=$(grep -v "^#.*" ./.github/workflows/load-tests-job-configs/${CONFIG_ARR[INDEX]} | tr '\n' ' ') - CURCONFIG=$(echo "${CURCONFIG/load-tests-go-dataflow-batch-combine-$((INDEX + 1))-/load-tests-go-dataflow-batch-combine-$((INDEX + 1))-$CURDATE}") - echo "prepared_config_$((INDEX + 1))=$CURCONFIG" >> $GITHUB_OUTPUT - done + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: go + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_Combine_Dataflow_Batch_10b.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_Combine_Dataflow_Batch_Fanout_4.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_Combine_Dataflow_Batch_Fanout_8.txt + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: get current time + run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV - name: run Combine Dataflow Batch Go Load Test 1 (single key) uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -89,7 +88,7 @@ jobs: arguments: | -PloadTest.mainClass=combine \ -Prunner=DataflowRunner \ - '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_1 }}' \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_Combine_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-go-dataflow-batch-combine-1-${{env.NOW_UTC}}' \ - name: run Combine Dataflow Batch Go Load Test 2 (multiple keys) uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -97,7 +96,7 @@ jobs: arguments: | -PloadTest.mainClass=combine \ -Prunner=DataflowRunner \ - '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_2 }}' \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_Combine_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-go-dataflow-batch-combine-2-${{env.NOW_UTC}}' \ - name: run Combine Dataflow Batch Go Load Test 3 (reiterate 10KB) uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -105,4 +104,4 @@ jobs: arguments: | -PloadTest.mainClass=combine \ -Prunner=DataflowRunner \ - '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_3 }}' \ No newline at end of file + '-PloadTest.args=${{ env.beam_LoadTests_Go_Combine_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-go-dataflow-batch-combine-3-${{env.NOW_UTC}}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Go_Combine_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Go_Combine_Flink_Batch.yml index f368707f1bd9..f5c9c97652ee 100644 --- a/.github/workflows/beam_LoadTests_Go_Combine_Flink_Batch.yml +++ b/.github/workflows/beam_LoadTests_Go_Combine_Flink_Batch.yml @@ -85,9 +85,9 @@ jobs: test-type: load test-language: go argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_Combine_Flink_Batch_10b.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_Combine_Flink_Batch_Fanout_4.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_Combine_Flink_Batch_Fanout_8.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_Combine_Flink_Batch_10b.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_Combine_Flink_Batch_Fanout_4.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_Combine_Flink_Batch_Fanout_8.txt arguments: | --job_name=load-tests-go-flink-batch-combine-$(date '+%m%d%H%M%S' --utc) - name: Start Flink with parallelism 5 diff --git a/.github/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml index 01493d467b35..9778b542bacb 100644 --- a/.github/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml @@ -75,13 +75,13 @@ jobs: test-type: load test-language: go argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_10b.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_100b.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_100kb.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Fanout_4.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Fanout_8.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Reiteration_10KB.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Reiteration_2MB.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_GBK_Dataflow_Batch_10b.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_GBK_Dataflow_Batch_100b.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_GBK_Dataflow_Batch_100kb.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_GBK_Dataflow_Batch_Fanout_4.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_GBK_Dataflow_Batch_Fanout_8.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_GBK_Dataflow_Batch_Reiteration_10KB.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_GBK_Dataflow_Batch_Reiteration_2MB.txt - name: get current time run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV # The env variables are created and populated in the test-arguments-action as "_test_arguments_" diff --git a/.github/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml index 53982b4175f4..37ae1e419c59 100644 --- a/.github/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml +++ b/.github/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml @@ -85,12 +85,12 @@ jobs: test-type: load test-language: go argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_10b.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_100b.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_100kb.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_Fanout_4.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_Fanout_8.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_Reiteration_10KB.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_10b.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_100b.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_100kb.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Fanout_4.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Fanout_8.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Reiteration_10KB.txt arguments: | --job_name=load-tests-go-flink-batch-gbk-$(date '+%m%d%H%M%S' --utc) - name: Start Flink with parallelism 5 diff --git a/.github/workflows/beam_LoadTests_Go_ParDo_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Go_ParDo_Dataflow_Batch.yml index 73e21c4d6c56..99292c88757a 100644 --- a/.github/workflows/beam_LoadTests_Go_ParDo_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Go_ParDo_Dataflow_Batch.yml @@ -75,10 +75,10 @@ jobs: test-type: load test-language: go argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_10_Iterations.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_200_Iterations.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_10_Counters.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_100_Counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_ParDo_Dataflow_Batch_10_Iterations.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_ParDo_Dataflow_Batch_200_Iterations.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_ParDo_Dataflow_Batch_10_Counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_ParDo_Dataflow_Batch_100_Counters.txt arguments: | --job_name=load-tests-go-dataflow-batch-pardo-$(date '+%m%d%H%M%S' --utc) - name: run ParDo Dataflow Batch Go Load Test 1 (10 iterations) diff --git a/.github/workflows/beam_LoadTests_Go_ParDo_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Go_ParDo_Flink_Batch.yml index 9fe817f01566..666799bfc655 100644 --- a/.github/workflows/beam_LoadTests_Go_ParDo_Flink_Batch.yml +++ b/.github/workflows/beam_LoadTests_Go_ParDo_Flink_Batch.yml @@ -85,10 +85,10 @@ jobs: test-type: load test-language: go argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_10_times.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_200_times.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_10_counters.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_100_counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_ParDo_Flink_Batch_10_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_ParDo_Flink_Batch_200_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_ParDo_Flink_Batch_10_counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_ParDo_Flink_Batch_100_counters.txt arguments: | --job_name=load-tests-go-flink-batch-pardo-$(date '+%m%d%H%M%S' --utc) - name: Start Flink with parallelism 5 diff --git a/.github/workflows/beam_LoadTests_Go_SideInput_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Go_SideInput_Dataflow_Batch.yml index 4aa8d7f7c3de..123e839d46bb 100644 --- a/.github/workflows/beam_LoadTests_Go_SideInput_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Go_SideInput_Dataflow_Batch.yml @@ -75,8 +75,8 @@ jobs: test-type: load test-language: go argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_SideInput_Go_Batch_Dataflow_First_Iterable.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_SideInput_Go_Batch_Dataflow_Iterable.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_SideInput_Dataflow_Batch_First_Iterable.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_SideInput_Dataflow_Batch_Iterable.txt arguments: | --job_name=load-tests-go-dataflow-batch-sideinput-$(date '+%m%d%H%M%S' --utc) - name: run SideInput Dataflow Batch Go Load Test 1 (first iterable) diff --git a/.github/workflows/beam_LoadTests_Go_SideInput_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Go_SideInput_Flink_Batch.yml index 53c100fff2c8..592af7e86676 100644 --- a/.github/workflows/beam_LoadTests_Go_SideInput_Flink_Batch.yml +++ b/.github/workflows/beam_LoadTests_Go_SideInput_Flink_Batch.yml @@ -85,8 +85,8 @@ jobs: test-type: load test-language: go argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_SideInput_Flink_Batch_First_Iterable.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_SideInput_Flink_Batch_Iterable.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_SideInput_Flink_Batch_First_Iterable.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/go_SideInput_Flink_Batch_Iterable.txt arguments: | --job_name=load-tests-go-flink-batch-sideinput-$(date '+%m%d%H%M%S' --utc) - name: Start Flink with parallelism 10 diff --git a/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Batch.yml index 705d8a85fa65..8677f6f81f71 100644 --- a/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Batch.yml @@ -75,10 +75,10 @@ jobs: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_100b_Single_Key.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_10kB.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_2MB.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Batch_100b_Single_Key.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Batch_10kB.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Batch_2MB.txt # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run CoGBK 2GB 100 byte records - single key uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml index 4ecca6231401..bdd0adadb11d 100644 --- a/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml @@ -59,7 +59,7 @@ jobs: name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) strategy: matrix: - job_name: [ "beam_LoadTests_Java_CoGBK_Dataflow_Streaming" ] + job_name: ["beam_LoadTests_Java_CoGBK_Dataflow_Streaming"] job_phrase: ["Run Load Tests Java CoGBK Dataflow Streaming"] steps: - uses: actions/checkout@v4 @@ -69,18 +69,17 @@ jobs: comment_phrase: ${{ matrix.job_phrase }} github_token: ${{ secrets.GITHUB_TOKEN }} github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - - name: Prepare configs - id: set_configs - shell: bash - run: | - CURCONFIG=$(grep -v "^#.*" ./.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_SingleKey.txt | tr '\n' ' ') - echo "prepared_config_1=$CURCONFIG" >> $GITHUB_OUTPUT - CURCONFIG=$(grep -v "^#.*" ./.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_MultipleKey.txt | tr '\n' ' ') - echo "prepared_config_2=$CURCONFIG" >> $GITHUB_OUTPUT - CURCONFIG=$(grep -v "^#.*" ./.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_Reiteration_10KB.txt | tr '\n' ' ') - echo "prepared_config_3=$CURCONFIG" >> $GITHUB_OUTPUT - CURCONFIG=$(grep -v "^#.*" ./.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_Reiteration_2MB.txt | tr '\n' ' ') - echo "prepared_config_4=$CURCONFIG" >> $GITHUB_OUTPUT + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Streaming_2GB_SingleKey.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Streaming_2GB_MultipleKey.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Streaming_2GB_Reiteration_10KB.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Streaming_2GB_Reiteration_2MB.txt + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run CoGBK Dataflow Streaming Java Load Test 1 (single key) uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -88,7 +87,7 @@ jobs: arguments: | -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ - '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_1 }}' \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_Streaming_test_arguments_1 }}' \ - name: run CoGBK Dataflow Streaming Java Load Test 2 (multiple key) uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -96,7 +95,7 @@ jobs: arguments: | -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ - '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_2 }}' \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_Streaming_test_arguments_2 }}' \ - name: run CoGBK Dataflow Streaming Java Load Test 3 (reiteration 10KB value) uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -104,7 +103,7 @@ jobs: arguments: | -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ - '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_3 }}' \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_Streaming_test_arguments_3 }}' \ - name: run CoGBK Dataflow Streaming Java Load Test 4 (reiteration 2MB value) uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -112,7 +111,7 @@ jobs: arguments: | -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ - '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_4 }}' \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_Streaming_test_arguments_4 }}' \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v3 if: failure() diff --git a/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml index 327e007d5437..8c785595ffec 100644 --- a/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml +++ b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml @@ -83,10 +83,10 @@ jobs: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_100b_Single_Key.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_10kB.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_2MB.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Batch_100b_Single_Key.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Batch_10kB.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Batch_2MB.txt arguments: | --influxTags={\"runnerVersion\":\"v2\",\"jdk\":\"java${{ matrix.java_version }}\"} # The env variables are created and populated in the test-arguments-action as "_test_arguments_" diff --git a/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml index 0abe99ce83d2..0c620f02750d 100644 --- a/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml +++ b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml @@ -83,10 +83,10 @@ jobs: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_100b_Single_Key.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_100b_Multiple_Keys.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_10kB.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_2MB.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_V2_Streaming_Java_100b_Single_Key.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_V2_Streaming_Java_100b_Multiple_Keys.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_V2_Streaming_Java_10kB.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_V2_Streaming_Java_2MB.txt arguments: | --influxTags={\"runnerVersion\":\"v2\",\"jdk\":\"java${{ matrix.java_version }}\"} # The env variables are created and populated in the test-arguments-action as "_test_arguments_" diff --git a/.github/workflows/beam_LoadTests_Java_CoGBK_SparkStructuredStreaming_Batch.yml b/.github/workflows/beam_LoadTests_Java_CoGBK_SparkStructuredStreaming_Batch.yml index f4b4b6d9946e..7f1935267e95 100644 --- a/.github/workflows/beam_LoadTests_Java_CoGBK_SparkStructuredStreaming_Batch.yml +++ b/.github/workflows/beam_LoadTests_Java_CoGBK_SparkStructuredStreaming_Batch.yml @@ -75,10 +75,10 @@ jobs: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_100b_Single_Key.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_100b_Multiple_Keys.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_10kB.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_2MB.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_CoGBK_SparkStructuredStreaming_Batch_100b_Single_Key.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_CoGBK_SparkStructuredStreaming_Batch_100b_Multiple_Keys.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_CoGBK_SparkStructuredStreaming_Batch_10kB.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_CoGBK_SparkStructuredStreaming_Batch_2MB.txt # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run CoGBK 2GB 100 byte records - single key uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_LoadTests_Java_Combine_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Java_Combine_Dataflow_Batch.yml index 1702d6c75184..02f5da33afaf 100644 --- a/.github/workflows/beam_LoadTests_Java_Combine_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Java_Combine_Dataflow_Batch.yml @@ -75,9 +75,9 @@ jobs: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_Combine_Java_Dataflow_Batch_10b.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_Combine_Java_Dataflow_Batch_Fanout_4.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_Combine_Java_Dataflow_Batch_Fanout_8.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_Combine_Dataflow_Batch_10b.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_Combine_Dataflow_Batch_Fanout_4.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_Combine_Dataflow_Batch_Fanout_8.txt - name: run Combine Dataflow Batch Java Load Test 1 (10 b records) uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_LoadTests_Java_Combine_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Java_Combine_Dataflow_Streaming.yml index e0e033939d3f..d6520646fcdd 100644 --- a/.github/workflows/beam_LoadTests_Java_Combine_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Java_Combine_Dataflow_Streaming.yml @@ -47,9 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_LoadTests_Java_Combine_Dataflow_Streaming_test_arguments_1: '' - beam_LoadTests_Java_Combine_Dataflow_Streaming_test_arguments_2: '' - beam_LoadTests_Java_Combine_Dataflow_Streaming_test_arguments_3: '' jobs: beam_LoadTests_Java_Combine_Dataflow_Streaming: @@ -78,9 +75,10 @@ jobs: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_Combine_Dataflow_Streaming_10b.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_Combine_Dataflow_Streaming_Fanout_4.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_Combine_Dataflow_Streaming_Fanout_8.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_Combine_Dataflow_Streaming_10b.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_Combine_Dataflow_Streaming_Fanout_4.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_Combine_Dataflow_Streaming_Fanout_8.txt + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run Combine Dataflow Streaming Java Load Test 1 (10 b records) uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch.yml b/.github/workflows/beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch.yml index 26c1a4975a7c..d4862ad45c07 100644 --- a/.github/workflows/beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch.yml +++ b/.github/workflows/beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch.yml @@ -47,9 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch_test_arguments_1: '' - beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch_test_arguments_2: '' - beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch_test_arguments_3: '' jobs: beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch: @@ -78,9 +75,10 @@ jobs: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_Combine_SparkStructuredStreaming_Batch_10b.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_Combine_SparkStructuredStreaming_Batch_Fanout_4.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_Combine_SparkStructuredStreaming_Batch_Fanout_8.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_Combine_SparkStructuredStreaming_Batch_10b.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_Combine_SparkStructuredStreaming_Batch_Fanout_4.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_Combine_SparkStructuredStreaming_Batch_Fanout_8.txt + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run Combine SparkStructuredStreaming Batch Java Load Test 1 (10b records) uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_Batch.yml index fd86521e592d..403deba2f0a5 100644 --- a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_Batch.yml @@ -75,13 +75,13 @@ jobs: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_10B_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_100B_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_100kB_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_reiterate_4_times_10kB_values.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_reiterate_4_times_2MB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Batch_2GB_of_10B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Batch_2GB_of_100B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Batch_2GB_of_100kB_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Batch_reiterate_4_times_10kB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Batch_reiterate_4_times_2MB_values.txt # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run Load test 2GB of 10B records uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_Streaming.yml index 63bc91bb1599..483c06cf821a 100644 --- a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_Streaming.yml @@ -75,13 +75,13 @@ jobs: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_10B_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_100B_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_reiterate_4_times_10kB_values.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_reiterate_4_times_2MB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Streaming_2GB_of_10B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Streaming_2GB_of_100B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Streaming_reiterate_4_times_10kB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Streaming_reiterate_4_times_2MB_values.txt # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run Load test 2GB of 10B records uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml index 39490d59fc22..fc5d072c3711 100644 --- a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml +++ b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml @@ -79,13 +79,13 @@ jobs: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_10B_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100B_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100kB_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_10kB_values.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_2MB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_10B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100kB_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_10kB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_2MB_values.txt # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run Load test 2GB of 10B records uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml index bb4e039ec8c2..fb249d2ade50 100644 --- a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml +++ b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml @@ -81,13 +81,13 @@ jobs: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_10B_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100B_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100kB_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_10kB_values.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_2MB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_10B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100kB_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_10kB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_2MB_values.txt # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run Load test 2GB of 10B records uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml index 60cf635457de..1ce6c4d7d378 100644 --- a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml +++ b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml @@ -79,13 +79,13 @@ jobs: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_10B_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100B_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100kB_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_10kB_values.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_2MB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_10B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100kB_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_10kB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_2MB_values.txt # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run Load test 2GB of 10B records uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml index 24d218c152ed..ecffc9923f02 100644 --- a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml +++ b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml @@ -81,13 +81,13 @@ jobs: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_10B_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100B_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100kB_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_10kB_values.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_2MB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_10B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100kB_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_10kB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_2MB_values.txt # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run Load test 2GB of 10B records uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_LoadTests_Java_GBK_Smoke.yml b/.github/workflows/beam_LoadTests_Java_GBK_Smoke.yml index d41ee990a6ae..251291163efe 100644 --- a/.github/workflows/beam_LoadTests_Java_GBK_Smoke.yml +++ b/.github/workflows/beam_LoadTests_Java_GBK_Smoke.yml @@ -72,10 +72,10 @@ jobs: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Direct.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Dataflow.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Flink.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Spark.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_Smoke_GroupByKey_Direct.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_Smoke_GroupByKey_Dataflow.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_Smoke_GroupByKey_Flink.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_Smoke_GroupByKey_Spark.txt - name: Set current datetime id: datetime run: | diff --git a/.github/workflows/beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch.yml b/.github/workflows/beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch.yml index 9f057bba3d3e..de66760e5298 100644 --- a/.github/workflows/beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch.yml +++ b/.github/workflows/beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch.yml @@ -75,13 +75,13 @@ jobs: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_10B_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_100B_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_100kB_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_10kB_values.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_2MB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_2GB_of_10B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_2GB_of_100B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_2GB_of_100kB_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_10kB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_2MB_values.txt # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run Load test 2GB of 10B records uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_Batch.yml index 3b19c6a6fa70..ebc30fb0efa4 100644 --- a/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_Batch.yml @@ -47,10 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_LoadTests_Java_ParDo_Dataflow_Batch_test_arguments_1: '' - beam_LoadTests_Java_ParDo_Dataflow_Batch_test_arguments_2: '' - beam_LoadTests_Java_ParDo_Dataflow_Batch_test_arguments_3: '' - beam_LoadTests_Java_ParDo_Dataflow_Batch_test_arguments_4: '' jobs: beam_LoadTests_Java_ParDo_Dataflow_Batch: @@ -79,10 +75,11 @@ jobs: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_10_times.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_200_times.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_10_counters.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_100_counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Batch_10_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Batch_200_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Batch_10_counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Batch_100_counters.txt + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run ParDo Dataflow Batch Java Load Test 1 (10 times) uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_Streaming.yml index 1f91207fee55..6dec9beca4bf 100644 --- a/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_Streaming.yml @@ -47,10 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_LoadTests_Java_ParDo_Dataflow_Streaming_test_arguments_1: '' - beam_LoadTests_Java_ParDo_Dataflow_Streaming_test_arguments_2: '' - beam_LoadTests_Java_ParDo_Dataflow_Streaming_test_arguments_3: '' - beam_LoadTests_Java_ParDo_Dataflow_Streaming_test_arguments_4: '' jobs: beam_LoadTests_Java_ParDo_Dataflow_Streaming: @@ -79,10 +75,11 @@ jobs: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_10_times.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_200_times.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_10_counters.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_100_counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Streaming_10_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Streaming_200_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Streaming_10_counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Streaming_100_counters.txt + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run ParDo Dataflow Streaming Java Load Test 1 (10 times) uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml index 66b5b5a8837f..812541087ea7 100644 --- a/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml +++ b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml @@ -83,10 +83,10 @@ jobs: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_times.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_200_times.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_counters.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_100_counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Batch_JavaVersions_200_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Batch_JavaVersions_100_counters.txt arguments: | --influxTags={\"runnerVersion\":\"v2\",\"jdk\":\"java${{ matrix.java_version }}\"} # The env variables are created and populated in the test-arguments-action as "_test_arguments_" diff --git a/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml index bd8987d4ee0d..64013955ad22 100644 --- a/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml +++ b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml @@ -83,10 +83,10 @@ jobs: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_times.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_200_times.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_counters.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_100_counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Streaming_JavaVersions_200_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Streaming_JavaVersions_100_counters.txt arguments: | --influxTags={\"runnerVersion\":\"v2\",\"jdk\":\"java${{ matrix.java_version }}\"} # The env variables are created and populated in the test-arguments-action as "_test_arguments_" diff --git a/.github/workflows/beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch.yml b/.github/workflows/beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch.yml index 45700ca56937..26c1c88dd893 100644 --- a/.github/workflows/beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch.yml +++ b/.github/workflows/beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch.yml @@ -47,10 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch_test_arguments_1: '' - beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch_test_arguments_2: '' - beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch_test_arguments_3: '' - beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch_test_arguments_4: '' jobs: beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch: @@ -79,10 +75,11 @@ jobs: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_10_times.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_200_times.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_10_counters.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_100_counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_ParDo_SparkStructuredStreaming_Batch_10_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_ParDo_SparkStructuredStreaming_Batch_200_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_ParDo_SparkStructuredStreaming_Batch_10_counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_ParDo_SparkStructuredStreaming_Batch_100_counters.txt + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run ParDo SparkStructuredStreaming Batch Java Load Test 1 (10 times) uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml index bc1b41461bcc..504d512802aa 100644 --- a/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml @@ -79,10 +79,10 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Batch_100b_Single_Key.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Batch_10kB.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Batch_2MB.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Batch_100b_Single_Key.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Batch_10kB.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Batch_2MB.txt - name: Set current datetime id: datetime run: | diff --git a/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml index 15d36839de1f..ccd8c371e583 100644 --- a/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml @@ -79,10 +79,10 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Streaming_100b_Single_Key.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Streaming_100b_Multiple_Keys.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Streaming_10kB.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Streaming_2MB.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python-cogbk/python_CoGBK_Dataflow_Streaming_100b_Single_Key.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python-cogbk/python_CoGBK_Dataflow_Streaming_100b_Multiple_Keys.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python-cogbk/python_CoGBK_Dataflow_Streaming_10kB.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python-cogbk/python_CoGBK_Dataflow_Streaming_2MB.txt - name: Set current datetime id: datetime run: | diff --git a/.github/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml index e06ed4f94e4f..cdf0c0953165 100644 --- a/.github/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml @@ -89,9 +89,9 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Flink_Batch_100b_Single_Key.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Flink_Batch_100b_Multiple_Keys.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Flink_Batch_10kB.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python-cogbk/python_CoGBK_Dataflow_Flink_Batch_100b_Single_Key.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python-cogbk/python_CoGBK_Dataflow_Flink_Batch_100b_Multiple_Keys.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python-cogbk/python_CoGBK_Dataflow_Flink_Batch_10kB.txt - name: Start Flink with parallelism 5 env: FLINK_NUM_WORKERS: 5 diff --git a/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml index c042cafab864..2a1220c3b885 100644 --- a/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml @@ -73,19 +73,18 @@ jobs: uses: actions/setup-python@v4 with: python-version: '3.8' - - name: Prepare configs - #Reads config files, excludes comments, appends current date to the job_name parameter - id: set_configs - shell: bash - run: | - CURDATE=$(date '+%m%d%H%M%S' --utc) - CONFIG_ARR=('config_Combine_Python_Batch_2GB_10b.txt' 'config_Combine_Python_Batch_2GB_Fanout_4.txt' 'config_Combine_Python_Batch_2GB_Fanout_8.txt') - for INDEX in ${!CONFIG_ARR[@]} - do - CURCONFIG=$(grep -v "^#.*" ./.github/workflows/load-tests-job-configs/${CONFIG_ARR[INDEX]} | tr '\n' ' ') - CURCONFIG=$(echo "${CURCONFIG/load-tests-python-dataflow-batch-combine-$((INDEX + 1))-/load-tests-python-dataflow-batch-combine-$((INDEX + 1))-$CURDATE}") - echo "prepared_config_$((INDEX + 1))=$CURCONFIG" >> $GITHUB_OUTPUT - done + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Batch_2GB_10b.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Batch_2GB_Fanout_4.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Batch_2GB_Fanout_8.txt + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: get current time + run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV - name: run Combine Dataflow Batch Python Load Test 1 (10 bytes records) uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -94,7 +93,7 @@ jobs: -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=DataflowRunner \ -PpythonVersion=3.8 \ - '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_1 }}' \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-combine-1-${{env.NOW_UTC}}' \ - name: run Combine Dataflow Batch Python Load Test 2 (fanout 4) uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -103,7 +102,7 @@ jobs: -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=DataflowRunner \ -PpythonVersion=3.8 \ - '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_2 }}' \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-combine-2-${{env.NOW_UTC}}' \ - name: run Combine Dataflow Batch Python Load Test 3 (fanout 8) uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -112,4 +111,4 @@ jobs: -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=DataflowRunner \ -PpythonVersion=3.8 \ - '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_3 }}' \ No newline at end of file + '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-combine-3-${{env.NOW_UTC}}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml index c9f05083cfa8..1dabfa77673c 100644 --- a/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml @@ -79,9 +79,9 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_10_byte_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_Fanout_4.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_Fanout_8.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Streaming_2GB_10_byte_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Streaming_2GB_Fanout_4.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Streaming_2GB_Fanout_8.txt - name: get current time run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV # The env variables are created and populated in the test-arguments-action as "_test_arguments_" diff --git a/.github/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml index 3ab3e410faef..358a9a5378d9 100644 --- a/.github/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml @@ -89,9 +89,9 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_Combine_Flink_Batch_2GB_10_byte_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_Combine_Flink_Batch_2GB_Fanout_4.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_Combine_Flink_Batch_2GB_Fanout_8.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_10_byte_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_4.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_8.txt - name: Start Flink with parallelism 5 env: FLINK_NUM_WORKERS: 5 diff --git a/.github/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml b/.github/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml index d90862ddef3c..8da2bd586045 100644 --- a/.github/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml @@ -89,8 +89,8 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_Combine_Flink_Streaming_2GB_Fanout_4.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_Combine_Flink_Streaming_2GB_Fanout_8.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_4.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_8.txt - name: Start Flink with parallelism 16 env: FLINK_NUM_WORKERS: 16 diff --git a/.github/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml b/.github/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml index 1483f59472ec..674f37fd3481 100644 --- a/.github/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml +++ b/.github/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml @@ -79,7 +79,7 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_FnApiRunner_Python_Microbenchmark.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_FnApiRunner_Microbenchmark.txt - name: run FnApiRunner Microbenchmark Python Load Test uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml index 92a9d82b79aa..d3af9666a782 100644 --- a/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml @@ -79,11 +79,11 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_2GB_of_10B_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_2GB_of_100B_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_2GB_of_100kB_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Batch_2GB_of_10B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Batch_2GB_of_100B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Batch_2GB_of_100kB_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt - name: get current time run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV # The env variables are created and populated in the test-arguments-action as "_test_arguments_" diff --git a/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml index 9a123f7b4aac..ff796c2db1ef 100644 --- a/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml @@ -79,7 +79,7 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt - name: get current time run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV # The env variable is created and populated in the test-arguments-action as "_test_arguments_" diff --git a/.github/workflows/beam_LoadTests_Python_GBK_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Python_GBK_Flink_Batch.yml index 4074a0a7417c..4674e4bb01ed 100644 --- a/.github/workflows/beam_LoadTests_Python_GBK_Flink_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_GBK_Flink_Batch.yml @@ -89,11 +89,11 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_2GB_of_10B_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_2GB_of_100B_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_10B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_100B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt - name: get current time run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV - name: Start Flink with parallelism 5 diff --git a/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml index 556e1746f3ee..b5062369c4d9 100644 --- a/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml @@ -79,8 +79,8 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_10kB_values.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_2MB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_10kB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_2MB_values.txt - name: get current time run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV # The env variables are created and populated in the test-arguments-action as "_test_arguments_" diff --git a/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml index a1f37bae5133..25fb9fdd27ed 100644 --- a/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml @@ -79,8 +79,8 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_10kB_values.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_2MB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_10kB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_2MB_values.txt - name: get current time run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV # The env variables are created and populated in the test-arguments-action as "_test_arguments_" diff --git a/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml index 472d494cd87b..303780ac8088 100644 --- a/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml @@ -79,10 +79,10 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_10_Iterations.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_200_Iterations.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_10_Counters.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_100_Counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Batch_10_Iterations.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Batch_200_Iterations.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Batch_10_Counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Batch_100_Counters.txt - name: Set current datetime id: datetime run: | diff --git a/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml index cbcd28149aea..c6c7bc322e93 100644 --- a/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml @@ -79,10 +79,10 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_10_Iterations.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_200_Iterations.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_10_Counters.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_100_Counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_10_Iterations.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_200_Iterations.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_10_Counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_100_Counters.txt - name: Set current datetime id: datetime run: | diff --git a/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml index d78fe22b2858..9c8d86289e6e 100644 --- a/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml @@ -89,9 +89,9 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Batch_10_Iterations.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Batch_200_Iterations.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Batch_10_Counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Iterations.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_200_Iterations.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Counters.txt - name: Start Flink with parallelism 5 env: FLINK_NUM_WORKERS: 5 diff --git a/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml b/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml index e30ab4d699cc..067cdd5cd525 100644 --- a/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml @@ -89,11 +89,11 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_10_Iterations.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_200_Iterations.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_10_Counters.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_100_Counters.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_5_Iterations.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Iterations.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_200_Iterations.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_100_Counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_5_Iterations.txt - name: Start Flink with parallelism 5 env: FLINK_NUM_WORKERS: 5 diff --git a/.github/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml index f62f72343357..3be652ab8891 100644 --- a/.github/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml @@ -79,16 +79,16 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_1key_percent_dict.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_99key_percent_dict.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1window_first_iterable.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1window_iterable.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_first_list.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_list.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1000window_1key_percent_dict.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1000window_99key_percent_dict.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1000window_first_iterable.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1000window_iterable.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_1key_percent_dict.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_99key_percent_dict.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1window_first_iterable.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1window_iterable.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_first_list.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_list.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1000window_1key_percent_dict.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1000window_99key_percent_dict.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1000window_first_iterable.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1000window_iterable.txt - name: Set current datetime id: datetime run: | diff --git a/.github/workflows/beam_LoadTests_Python_Smoke.yml b/.github/workflows/beam_LoadTests_Python_Smoke.yml index 77d4d65bfe9a..8150956fe8db 100644 --- a/.github/workflows/beam_LoadTests_Python_Smoke.yml +++ b/.github/workflows/beam_LoadTests_Python_Smoke.yml @@ -76,8 +76,8 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_Smoke_GroupByKey_Direct.txt - ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_Smoke_GroupByKey_Dataflow.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_Smoke_GroupByKey_Direct.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_Smoke_GroupByKey_Dataflow.txt - name: Set current datetime id: datetime run: | diff --git a/.github/workflows/beam_PerformanceTests_AvroIOIT.yml b/.github/workflows/beam_PerformanceTests_AvroIOIT.yml index 67b00a98ce8e..011f8a740fbb 100644 --- a/.github/workflows/beam_PerformanceTests_AvroIOIT.yml +++ b/.github/workflows/beam_PerformanceTests_AvroIOIT.yml @@ -47,7 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_PerformanceTests_AvroIOIT_test_arguments_1: '' jobs: beam_PerformanceTests_AvroIOIT: @@ -76,9 +75,10 @@ jobs: test-type: performance test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/config_AvroIOIT.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/avroIOIT.txt arguments: | --filenamePrefix=gs://temp-storage-for-perf-tests/${{ matrix.job_name }}/${{github.run_id}}/ + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run integrationTest uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PerformanceTests_AvroIOIT_HDFS.yml b/.github/workflows/beam_PerformanceTests_AvroIOIT_HDFS.yml index 868d1b283636..c85c8d481434 100644 --- a/.github/workflows/beam_PerformanceTests_AvroIOIT_HDFS.yml +++ b/.github/workflows/beam_PerformanceTests_AvroIOIT_HDFS.yml @@ -47,7 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_PerformanceTests_AvroIOIT_HDFS_test_arguments_1: '' jobs: beam_PerformanceTests_AvroIOIT_HDFS: @@ -94,10 +93,11 @@ jobs: test-type: performance test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/config_AvroIOIT_HDFS.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/avroIOIT_HDFS.txt arguments: | --filenamePrefix=hdfs://${{ steps.install_hadoop.outputs.hadoop_IP }}:9000/TEXTIO_IT_ --hdfsConfiguration=[{\\\"fs.defaultFS\\\":\\\"hdfs:${{ steps.install_hadoop.outputs.hadoop_IP }}:9000\\\",\\\"dfs.replication\\\":1}] + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run integrationTest uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml b/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml index cd56c66c5a04..0f5f69dcd320 100644 --- a/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml +++ b/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml @@ -69,14 +69,16 @@ jobs: comment_phrase: ${{ matrix.job_phrase }} github_token: ${{ secrets.GITHUB_TOKEN }} github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - - name: Prepare config - id: set_config - shell: bash - run: | - CURDATE=$(date '+%m%d%H%M%S' --utc) - CURCONFIG=$(grep -v "^#.*" ./.github/workflows/performance-tests-job-configs/config_BigQueryIO_Batch_Java_Avro.txt | tr '\n' ' ') - CONFIGWITHDATE=$(echo "${CURCONFIG/bqio_write_10GB_java_avro_/bqio_write_10GB_java_avro_$CURDATE}") - echo "prepared_config=$CONFIGWITHDATE" >> $GITHUB_OUTPUT + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/bigQueryIO_Batch_Java_Avro.txt + arguments: | + --testBigQueryTable=bqio_write_10GB_java_avro_$(date '+%m%d%H%M%S' --utc) + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run Java BigQueryIO Batch Avro Performance Test uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -85,7 +87,7 @@ jobs: --tests org.apache.beam.sdk.bigqueryioperftests.BigQueryIOIT \ --info \ -DintegrationTestRunner=dataflow \ - -DintegrationTestPipelineOptions=${{ steps.set_config.outputs.prepared_config }} \ + -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_BigQueryIO_Batch_Java_Avro_test_arguments_1 }}]' \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v3 if: failure() diff --git a/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml b/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml index 2e2e00d40ab4..df2f406c4294 100644 --- a/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml +++ b/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml @@ -69,14 +69,16 @@ jobs: comment_phrase: ${{ matrix.job_phrase }} github_token: ${{ secrets.GITHUB_TOKEN }} github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - - name: Prepare config - id: set_config - shell: bash - run: | - CURDATE=$(date '+%m%d%H%M%S' --utc) - CURCONFIG=$(grep -v "^#.*" ./.github/workflows/performance-tests-job-configs/config_BigQueryIO_Batch_Java_Json.txt | tr '\n' ' ') - CONFIGWITHDATE=$(echo "${CURCONFIG/bqio_write_10GB_java_json_/bqio_write_10GB_java_json_$CURDATE}") - echo "prepared_config=$CONFIGWITHDATE" >> $GITHUB_OUTPUT + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/bigQueryIO_Batch_Java_Json.txt + arguments: | + --testBigQueryTable=bqio_write_10GB_java_json_$(date '+%m%d%H%M%S' --utc) + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run Java BigQueryIO Batch Json Performance Test uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -85,7 +87,7 @@ jobs: --tests org.apache.beam.sdk.bigqueryioperftests.BigQueryIOIT \ --info \ -DintegrationTestRunner=dataflow \ - -DintegrationTestPipelineOptions=${{ steps.set_config.outputs.prepared_config }} \ + -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_BigQueryIO_Batch_Java_Json_test_arguments_1 }}]' \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v3 if: failure() diff --git a/.github/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml b/.github/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml index 95c4fa86b4e0..cc628012f1a4 100644 --- a/.github/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml +++ b/.github/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml @@ -69,14 +69,16 @@ jobs: comment_phrase: ${{ matrix.job_phrase }} github_token: ${{ secrets.GITHUB_TOKEN }} github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - - name: Prepare config - id: set_config - shell: bash - run: | - CURDATE=$(date '+%m%d%H%M%S' --utc) - CURCONFIG=$(grep -v "^#.*" ./.github/workflows/performance-tests-job-configs/config_BigQueryIO_Streaming_Java.txt | tr '\n' ' ') - CONFIGWITHDATE=$(echo "${CURCONFIG/bqio_write_10GB_java_stream_/bqio_write_10GB_java_stream_$CURDATE}") - echo "prepared_config=$CONFIGWITHDATE" >> $GITHUB_OUTPUT + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/bigQueryIO_Streaming_Java.txt + arguments: | + --testBigQueryTable=bqio_write_10GB_java_stream_$(date '+%m%d%H%M%S' --utc) + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run Java BigQueryIO Streaming Performance Test uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -85,7 +87,7 @@ jobs: --tests org.apache.beam.sdk.bigqueryioperftests.BigQueryIOIT \ --info \ -DintegrationTestRunner=dataflow \ - -DintegrationTestPipelineOptions=${{ steps.set_config.outputs.prepared_config }} \ + -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_BigQueryIO_Streaming_Java_test_arguments_1 }}]' \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v3 if: failure() diff --git a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml index a574cdc4bdb5..dd9477f091af 100644 --- a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml +++ b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml @@ -47,7 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_PerformanceTests_BiqQueryIO_Read_Python_test_arguments_1: '' jobs: beam_PerformanceTests_BiqQueryIO_Read_Python: @@ -80,9 +79,10 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/biqQueryIO_Read_Python.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/biqQueryIO_Read_Python.txt arguments: | --job_name=performance-tests-bqio-read-python-10gb$(date '+%m%d%H%M%S' --utc) + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: Run BigQueryIO Read Performance Test Python uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml index c906c6efc575..32eb2c9301ca 100644 --- a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml +++ b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml @@ -47,7 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_PerformanceTests_BiqQueryIO_Write_Python_Batch_test_arguments_1: '' jobs: beam_PerformanceTests_BiqQueryIO_Write_Python_Batch: @@ -80,9 +79,10 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/biqQueryIO_Write_Python_Batch.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/biqQueryIO_Write_Python_Batch.txt arguments: | --job_name=performance-tests-bqio-write-python-batch-10gb$(date '+%m%d%H%M%S' --utc) + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run BigQueryIO Write Batch Python Performance Test uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PerformanceTests_Cdap.yml b/.github/workflows/beam_PerformanceTests_Cdap.yml index 804ea0a8f454..49184abfd4f3 100644 --- a/.github/workflows/beam_PerformanceTests_Cdap.yml +++ b/.github/workflows/beam_PerformanceTests_Cdap.yml @@ -47,7 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_PerformanceTests_Cdap_test_arguments_1: '' jobs: beam_PerformanceTests_Cdap: @@ -94,9 +93,10 @@ jobs: test-type: performance test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/cdap.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/cdap.txt arguments: | --postgresServerName=${{ steps.install_postgres.outputs.postgres_IP }} + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run integrationTest uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PerformanceTests_Compressed_TextIOIT.yml b/.github/workflows/beam_PerformanceTests_Compressed_TextIOIT.yml index cd1f2433284b..7ea42f197708 100644 --- a/.github/workflows/beam_PerformanceTests_Compressed_TextIOIT.yml +++ b/.github/workflows/beam_PerformanceTests_Compressed_TextIOIT.yml @@ -76,7 +76,7 @@ jobs: test-type: performance test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/config_Compressed_TextIOIT.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/compressed_TextIOIT.txt arguments: | --filenamePrefix=gs://temp-storage-for-perf-tests/${{ matrix.job_name }}/${{github.run_id}}/ - name: run integrationTest diff --git a/.github/workflows/beam_PerformanceTests_Compressed_TextIOIT_HDFS.yml b/.github/workflows/beam_PerformanceTests_Compressed_TextIOIT_HDFS.yml index 885d6c793c72..38b28ea084ae 100644 --- a/.github/workflows/beam_PerformanceTests_Compressed_TextIOIT_HDFS.yml +++ b/.github/workflows/beam_PerformanceTests_Compressed_TextIOIT_HDFS.yml @@ -93,7 +93,7 @@ jobs: test-type: performance test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/config_Compressed_TextIOIT_HDFS.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/compressed_TextIOIT_HDFS.txt arguments: | --filenamePrefix=hdfs://${{ steps.install_hadoop.outputs.hadoop_IP }}:9000/TEXTIO_IT_ --hdfsConfiguration=[{\\\"fs.defaultFS\\\":\\\"hdfs:${{ steps.install_hadoop.outputs.hadoop_IP }}:9000\\\",\\\"dfs.replication\\\":1}] diff --git a/.github/workflows/beam_PerformanceTests_HadoopFormat.yml b/.github/workflows/beam_PerformanceTests_HadoopFormat.yml index 060bb24811c3..9e6160007e9d 100644 --- a/.github/workflows/beam_PerformanceTests_HadoopFormat.yml +++ b/.github/workflows/beam_PerformanceTests_HadoopFormat.yml @@ -47,7 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_PerformanceTests_HadoopFormat_test_arguments_1: '' jobs: beam_PerformanceTests_HadoopFormat: @@ -94,9 +93,10 @@ jobs: test-type: performance test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/hadoopFormat.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/hadoopFormat.txt arguments: | --postgresServerName=${{ steps.install_postgres.outputs.postgres_IP }} + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run integrationTest uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PerformanceTests_JDBC.yml b/.github/workflows/beam_PerformanceTests_JDBC.yml index 8d7dba3eb05e..206f7cd199b6 100644 --- a/.github/workflows/beam_PerformanceTests_JDBC.yml +++ b/.github/workflows/beam_PerformanceTests_JDBC.yml @@ -47,7 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_PerformanceTests_JDBC_test_arguments_1: '' jobs: beam_PerformanceTests_JDBC: @@ -94,9 +93,10 @@ jobs: test-type: performance test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/JDBC.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/JDBC.txt arguments: | --postgresServerName=${{ steps.postgres_for_dev.outputs.pfd_ip }} + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: Run Java JdbcIO Performance Test uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PerformanceTests_Kafka_IO.yml b/.github/workflows/beam_PerformanceTests_Kafka_IO.yml index a69f3400f80a..47cafaac9f81 100644 --- a/.github/workflows/beam_PerformanceTests_Kafka_IO.yml +++ b/.github/workflows/beam_PerformanceTests_Kafka_IO.yml @@ -47,8 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_PerformanceTests_Kafka_IO_test_arguments_1: '' - beam_PerformanceTests_Kafka_IO_test_arguments_2: '' jobs: beam_PerformanceTests_Kafka_IO: @@ -98,10 +96,11 @@ jobs: test-type: performance test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/kafka_IO_Streaming.txt - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/kafka_IO_Batch.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/kafka_IO_Streaming.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/kafka_IO_Batch.txt arguments: | --kafkaBootstrapServerAddresses=${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_0 }}:${{ env.KAFKA_SERVICE_PORT }},${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_1 }}:${{ env.KAFKA_SERVICE_PORT }},${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_2 }}:${{ env.KAFKA_SERVICE_PORT }} + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: Run Java KafkaIO Performance Streaming Test uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PerformanceTests_ManyFiles_TextIOIT.yml b/.github/workflows/beam_PerformanceTests_ManyFiles_TextIOIT.yml index 0559195a5a7d..a804f66740cf 100644 --- a/.github/workflows/beam_PerformanceTests_ManyFiles_TextIOIT.yml +++ b/.github/workflows/beam_PerformanceTests_ManyFiles_TextIOIT.yml @@ -47,7 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_PerformanceTests_ManyFiles_TextIOIT_test_arguments_1: '' jobs: beam_PerformanceTests_ManyFiles_TextIOIT: @@ -76,9 +75,10 @@ jobs: test-type: performance test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/config_ManyFiles_TextIOIT.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/manyFiles_TextIOIT.txt arguments: | --filenamePrefix=gs://temp-storage-for-perf-tests/${{ matrix.job_name }}/${{github.run_id}}/ + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run integrationTest uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PerformanceTests_ManyFiles_TextIOIT_HDFS.yml b/.github/workflows/beam_PerformanceTests_ManyFiles_TextIOIT_HDFS.yml index 33db6119c066..9510a98e1c19 100644 --- a/.github/workflows/beam_PerformanceTests_ManyFiles_TextIOIT_HDFS.yml +++ b/.github/workflows/beam_PerformanceTests_ManyFiles_TextIOIT_HDFS.yml @@ -47,7 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_PerformanceTests_ManyFiles_TextIOIT_HDFS_test_arguments_1: '' jobs: beam_PerformanceTests_ManyFiles_TextIOIT_HDFS: @@ -93,10 +92,11 @@ jobs: test-type: performance test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/config_ManyFiles_TextIOIT_HDFS.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/manyFiles_TextIOIT_HDFS.txt arguments: | --filenamePrefix=hdfs://${{ steps.install_hadoop.outputs.hadoop_IP }}:9000/TEXTIO_IT_ --hdfsConfiguration=[{\\\"fs.defaultFS\\\":\\\"hdfs:${{ steps.install_hadoop.outputs.hadoop_IP }}:9000\\\",\\\"dfs.replication\\\":1}] + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run integrationTest uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PerformanceTests_MongoDBIO_IT.yml b/.github/workflows/beam_PerformanceTests_MongoDBIO_IT.yml index 3b874d819d54..4f3d15f29a17 100644 --- a/.github/workflows/beam_PerformanceTests_MongoDBIO_IT.yml +++ b/.github/workflows/beam_PerformanceTests_MongoDBIO_IT.yml @@ -47,7 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_PerformanceTests_MongoDBIO_IT_test_arguments_1: '' jobs: beam_PerformanceTests_MongoDBIO_IT: @@ -94,9 +93,10 @@ jobs: test-type: performance test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/mongoDBIO_IT.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/mongoDBIO_IT.txt arguments: | --mongoDBHostName=${{ steps.install_mongo.outputs.mongo_IP }} + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run integrationTest uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PerformanceTests_ParquetIOIT.yml b/.github/workflows/beam_PerformanceTests_ParquetIOIT.yml index 2c822bb3a12f..879d622c70d6 100644 --- a/.github/workflows/beam_PerformanceTests_ParquetIOIT.yml +++ b/.github/workflows/beam_PerformanceTests_ParquetIOIT.yml @@ -76,7 +76,7 @@ jobs: test-type: performance test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/config_ParquetIOIT.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/parquetIOIT.txt arguments: | --filenamePrefix=gs://temp-storage-for-perf-tests/${{ matrix.job_name }}/${{github.run_id}}/ - name: run integrationTest diff --git a/.github/workflows/beam_PerformanceTests_ParquetIOIT_HDFS.yml b/.github/workflows/beam_PerformanceTests_ParquetIOIT_HDFS.yml index ac3e194efd46..e9bc1f3b5db5 100644 --- a/.github/workflows/beam_PerformanceTests_ParquetIOIT_HDFS.yml +++ b/.github/workflows/beam_PerformanceTests_ParquetIOIT_HDFS.yml @@ -93,7 +93,7 @@ jobs: test-type: performance test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/config_ParquetIOIT_HDFS.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/parquetIOIT_HDFS.txt arguments: | --filenamePrefix=hdfs://${{ steps.install_hadoop.outputs.hadoop_IP }}:9000/TEXTIO_IT_ --hdfsConfiguration=[{\\\"fs.defaultFS\\\":\\\"hdfs:${{ steps.install_hadoop.outputs.hadoop_IP }}:9000\\\",\\\"dfs.replication\\\":1}] diff --git a/.github/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml b/.github/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml index 31e0c995aded..17bb5ce81986 100644 --- a/.github/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml +++ b/.github/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml @@ -47,7 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_PerformanceTests_PubsubIOIT_Python_Streaming_test_arguments_1: '' jobs: beam_PerformanceTests_PubsubIOIT_Python_Streaming: @@ -80,9 +79,10 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/pubsubIOIT_Python_Streaming.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/pubsubIOIT_Python_Streaming.txt arguments: | --job_name=performance-tests-psio-python-2gb$(date '+%m%d%H%M%S' --utc) + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: Run test uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PerformanceTests_SQLBigQueryIO_Batch_Java.yml b/.github/workflows/beam_PerformanceTests_SQLBigQueryIO_Batch_Java.yml index 8eab9e68a3e1..32297e41b74e 100644 --- a/.github/workflows/beam_PerformanceTests_SQLBigQueryIO_Batch_Java.yml +++ b/.github/workflows/beam_PerformanceTests_SQLBigQueryIO_Batch_Java.yml @@ -47,7 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_PerformanceTests_SQLBigQueryIO_Batch_Java_test_arguments_1: '' jobs: beam_PerformanceTests_SQLBigQueryIO_Batch_Java: @@ -80,7 +79,8 @@ jobs: test-type: performance test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/SQLBigQueryIO_Batch_Java.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/SQLBigQueryIO_Batch_Java.txt + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: Run SQLBigQueryIO Batch Performance Test Java uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml b/.github/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml index ddea3f6a92ee..4f1b8357e9e3 100644 --- a/.github/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml +++ b/.github/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml @@ -47,7 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_PerformanceTests_SpannerIO_Read_2GB_Python_test_arguments_1: '' jobs: beam_PerformanceTests_SpannerIO_Read_2GB_Python: @@ -80,9 +79,10 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/spannerIO_Read_2GB_Python.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/spannerIO_Read_2GB_Python.txt arguments: | --job_name=performance-tests-spanner-read-python-2gb$(date '+%m%d%H%M%S' --utc) + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run Performance SpannerIO Read 2GB Test Python uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml b/.github/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml index 34066b890eeb..07b336e53348 100644 --- a/.github/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml +++ b/.github/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml @@ -47,7 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch_test_arguments_1: '' jobs: beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch: @@ -80,9 +79,10 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/spannerIO_Write_2GB_Python.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/spannerIO_Write_2GB_Python.txt arguments: | --job_name=performance-tests-spannerio-write-python-batch-2gb$(date '+%m%d%H%M%S' --utc) + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run Performance SpannerIO Write 2GB Test Python uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PerformanceTests_SparkReceiver_IO.yml b/.github/workflows/beam_PerformanceTests_SparkReceiver_IO.yml index daab6a20a546..bf750acbd31c 100644 --- a/.github/workflows/beam_PerformanceTests_SparkReceiver_IO.yml +++ b/.github/workflows/beam_PerformanceTests_SparkReceiver_IO.yml @@ -47,7 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_PerformanceTests_SparkReceiver_IO_test_arguments_1: '' jobs: beam_PerformanceTests_SparkReceiver_IO: @@ -94,9 +93,10 @@ jobs: test-type: performance test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/sparkReceiver_IO.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/sparkReceiver_IO.txt arguments: | --rabbitMqBootstrapServerAddress=amqp://guest:guest@${{ steps.install_rabbitmq.outputs.rabbitmq_IP }}:5672 + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run integrationTest uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PerformanceTests_TFRecordIOIT.yml b/.github/workflows/beam_PerformanceTests_TFRecordIOIT.yml index 7c2a1011694b..7088dab537d9 100644 --- a/.github/workflows/beam_PerformanceTests_TFRecordIOIT.yml +++ b/.github/workflows/beam_PerformanceTests_TFRecordIOIT.yml @@ -47,7 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_PerformanceTests_TFRecordIOIT_test_arguments_1: '' jobs: beam_PerformanceTests_TFRecordIOIT: @@ -76,9 +75,10 @@ jobs: test-type: performance test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/config_TFRecordIOIT.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/TFRecordIOIT.txt arguments: | --filenamePrefix=gs://temp-storage-for-perf-tests/${{ matrix.job_name }}/${{github.run_id}}/ + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run integrationTest uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PerformanceTests_TFRecordIOIT_HDFS.yml b/.github/workflows/beam_PerformanceTests_TFRecordIOIT_HDFS.yml index 32f31a29567b..f49be834154b 100644 --- a/.github/workflows/beam_PerformanceTests_TFRecordIOIT_HDFS.yml +++ b/.github/workflows/beam_PerformanceTests_TFRecordIOIT_HDFS.yml @@ -49,7 +49,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_PerformanceTests_TFRecordIOIT_HDFS_test_arguments_1: '' jobs: beam_PerformanceTests_TFRecordIOIT_HDFS: @@ -96,10 +95,11 @@ jobs: test-type: performance test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/TFRecordIOIT_HDFS.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/TFRecordIOIT_HDFS.txt arguments: | --filenamePrefix=hdfs://${{ steps.install_hadoop.outputs.hadoop_IP }}:9000/TEXTIO_IT_ --hdfsConfiguration=[{\\\"fs.defaultFS\\\":\\\"hdfs:${{ steps.install_hadoop.outputs.hadoop_IP }}:9000\\\",\\\"dfs.replication\\\":1}] + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run integrationTest uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PerformanceTests_TextIOIT.yml b/.github/workflows/beam_PerformanceTests_TextIOIT.yml index 477a8a08cac7..8be4370b3e4f 100644 --- a/.github/workflows/beam_PerformanceTests_TextIOIT.yml +++ b/.github/workflows/beam_PerformanceTests_TextIOIT.yml @@ -75,7 +75,7 @@ jobs: test-type: performance test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/textIOIT.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/textIOIT.txt arguments: | --filenamePrefix=gs://temp-storage-for-perf-tests/${{ matrix.job_name }}/${{github.run_id}} # The env variable is created and populated in the test-arguments-action as "beam_PerformanceTests_TextIOIT_test_arguments_1" diff --git a/.github/workflows/beam_PerformanceTests_TextIOIT_HDFS.yml b/.github/workflows/beam_PerformanceTests_TextIOIT_HDFS.yml index 8d1683cd1e35..e0f4b0365b6e 100644 --- a/.github/workflows/beam_PerformanceTests_TextIOIT_HDFS.yml +++ b/.github/workflows/beam_PerformanceTests_TextIOIT_HDFS.yml @@ -87,7 +87,7 @@ jobs: test-type: performance test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/textIOIT_HDFS.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/textIOIT_HDFS.txt arguments: | --filenamePrefix=hdfs://${{ steps.install_hadoop.outputs.hadoop_IP }}:9000/TEXTIO_IT_ --hdfsConfiguration=[{\\\"fs.defaultFS\\\":\\\"hdfs:${{ steps.install_hadoop.outputs.hadoop_IP }}:9000\\\",\\\"dfs.replication\\\":1}] diff --git a/.github/workflows/beam_PerformanceTests_TextIOIT_Python.yml b/.github/workflows/beam_PerformanceTests_TextIOIT_Python.yml index 64d4cad53327..b5aecb97889b 100644 --- a/.github/workflows/beam_PerformanceTests_TextIOIT_Python.yml +++ b/.github/workflows/beam_PerformanceTests_TextIOIT_Python.yml @@ -79,7 +79,7 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/textIOIT_Python.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/textIOIT_Python.txt arguments: | --filename_prefix=gs://temp-storage-for-perf-tests/${{ matrix.job_name }}/${{github.run_id}} # The env variable is created and populated in the test-arguments-action as "beam_PerformanceTests_TextIOIT_Python_test_arguments_1" diff --git a/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml b/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml index dee49eb0b5e3..6df9ef42d60a 100644 --- a/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml +++ b/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml @@ -47,7 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_PerformanceTests_WordCountIT_PythonVersions_test_arguments_1: '' jobs: beam_PerformanceTests_WordCountIT_PythonVersions: @@ -90,11 +89,12 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/wordCountIT_Python.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/wordCountIT_Python.txt arguments: | --job_name=performance-tests-wordcount-python${{steps.set_py_ver_clean.outputs.py_ver_clean}}-batch-1gb$(date '+%m%d%H%M%S' --utc) --metrics_table=wordcount_py${{steps.set_py_ver_clean.outputs.py_ver_clean}}_pkb_results --influx_measurement=wordcount_py${{steps.set_py_ver_clean.outputs.py_ver_clean}}_results + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: Run test uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PerformanceTests_XmlIOIT.yml b/.github/workflows/beam_PerformanceTests_XmlIOIT.yml index 33849aecf994..d610d362cdac 100644 --- a/.github/workflows/beam_PerformanceTests_XmlIOIT.yml +++ b/.github/workflows/beam_PerformanceTests_XmlIOIT.yml @@ -47,7 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_PerformanceTests_XmlIOIT_test_arguments_1: '' jobs: beam_PerformanceTests_XmlIOIT: @@ -76,9 +75,10 @@ jobs: test-type: performance test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/config_XmlIOIT.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/xmlIOIT.txt arguments: | --filenamePrefix=gs://temp-storage-for-perf-tests/${{ matrix.job_name }}/${{github.run_id}}/ + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run integrationTest uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PerformanceTests_XmlIOIT_HDFS.yml b/.github/workflows/beam_PerformanceTests_XmlIOIT_HDFS.yml index ff130dc4d53d..6ceb38ca87a4 100644 --- a/.github/workflows/beam_PerformanceTests_XmlIOIT_HDFS.yml +++ b/.github/workflows/beam_PerformanceTests_XmlIOIT_HDFS.yml @@ -47,7 +47,6 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - beam_PerformanceTests_XmlIOIT_HDFS_test_arguments_1: '' jobs: beam_PerformanceTests_XmlIOIT_HDFS: @@ -93,10 +92,11 @@ jobs: test-type: performance test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/config_XmlIOIT_HDFS.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/xmlIOIT_HDFS.txt arguments: | --filenamePrefix=hdfs://${{ steps.install_hadoop.outputs.hadoop_IP }}:9000/TEXTIO_IT_ --hdfsConfiguration=[{\\\"fs.defaultFS\\\":\\\"hdfs:${{ steps.install_hadoop.outputs.hadoop_IP }}:9000\\\",\\\"dfs.replication\\\":1}] + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run integrationTest uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml b/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml index bc62013c649d..92787670d93c 100644 --- a/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml +++ b/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml @@ -101,7 +101,7 @@ jobs: test-type: load test-language: python argument-file-paths: | - ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/xlang_KafkaIO_Python.txt + ${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/xlang_KafkaIO_Python.txt arguments: | --filename_prefix=gs://temp-storage-for-perf-tests/${{ matrix.job_name }}/${{github.run_id}}/ --bootstrap_servers=${{ steps.kafka_ip.outputs.KAFKA_BROKER_0 }}:32400,${{ steps.kafka_ip.outputs.KAFKA_BROKER_1 }}:32400,${{ steps.kafka_ip.outputs.KAFKA_BROKER_2 }}:32400 diff --git a/.github/workflows/load-tests-job-configs/beam_CloudML_Benchmarks_Dataflow_arguments.txt b/.github/workflows/load-tests-pipeline-options/beam_CloudML_Benchmarks_Dataflow_arguments.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/beam_CloudML_Benchmarks_Dataflow_arguments.txt rename to .github/workflows/load-tests-pipeline-options/beam_CloudML_Benchmarks_Dataflow_arguments.txt diff --git a/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152.txt b/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152.txt rename to .github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152.txt diff --git a/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152_Tesla_T4_GPU.txt b/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152_Tesla_T4_GPU.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152_Tesla_T4_GPU.txt rename to .github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152_Tesla_T4_GPU.txt diff --git a/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Base_Uncased.txt b/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Base_Uncased.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Base_Uncased.txt rename to .github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Base_Uncased.txt diff --git a/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Large_Uncased.txt b/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Large_Uncased.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Large_Uncased.txt rename to .github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Large_Uncased.txt diff --git a/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Vision_Classification_Resnet_101.txt b/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Vision_Classification_Resnet_101.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Vision_Classification_Resnet_101.txt rename to .github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Vision_Classification_Resnet_101.txt diff --git a/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_MultipleKey.txt b/.github/workflows/load-tests-pipeline-options/go_CoGBK_Dataflow_Batch_MultipleKey.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_MultipleKey.txt rename to .github/workflows/load-tests-pipeline-options/go_CoGBK_Dataflow_Batch_MultipleKey.txt diff --git a/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_Reiteration_10KB.txt b/.github/workflows/load-tests-pipeline-options/go_CoGBK_Dataflow_Batch_Reiteration_10KB.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_Reiteration_10KB.txt rename to .github/workflows/load-tests-pipeline-options/go_CoGBK_Dataflow_Batch_Reiteration_10KB.txt diff --git a/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_Reiteration_2MB.txt b/.github/workflows/load-tests-pipeline-options/go_CoGBK_Dataflow_Batch_Reiteration_2MB.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_Reiteration_2MB.txt rename to .github/workflows/load-tests-pipeline-options/go_CoGBK_Dataflow_Batch_Reiteration_2MB.txt diff --git a/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_SingleKey.txt b/.github/workflows/load-tests-pipeline-options/go_CoGBK_Dataflow_Batch_SingleKey.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_SingleKey.txt rename to .github/workflows/load-tests-pipeline-options/go_CoGBK_Dataflow_Batch_SingleKey.txt diff --git a/.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_MultipleKey.txt b/.github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_MultipleKey.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_MultipleKey.txt rename to .github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_MultipleKey.txt diff --git a/.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_Reiteration_10KB.txt b/.github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_Reiteration_10KB.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_Reiteration_10KB.txt rename to .github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_Reiteration_10KB.txt diff --git a/.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_Reiteration_2MB.txt b/.github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_Reiteration_2MB.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_Reiteration_2MB.txt rename to .github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_Reiteration_2MB.txt diff --git a/.github/workflows/load-tests-job-configs/config_Combine_Go_Batch_10b.txt b/.github/workflows/load-tests-pipeline-options/go_Combine_Dataflow_Batch_10b.txt similarity index 78% rename from .github/workflows/load-tests-job-configs/config_Combine_Go_Batch_10b.txt rename to .github/workflows/load-tests-pipeline-options/go_Combine_Dataflow_Batch_10b.txt index b9ad28105903..d97a3731bb97 100644 --- a/.github/workflows/load-tests-job-configs/config_Combine_Go_Batch_10b.txt +++ b/.github/workflows/load-tests-pipeline-options/go_Combine_Dataflow_Batch_10b.txt @@ -1,4 +1,3 @@ -############################################################################### # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -13,10 +12,8 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. -############################################################################### ---job_name=load-tests-go-dataflow-batch-combine-1- ---project=apache-beam-testing +# limitations under the License. + --region=us-central1 --temp_location=gs://temp-storage-for-perf-tests/loadtests --staging_location=gs://temp-storage-for-perf-tests/loadtests @@ -29,6 +26,4 @@ --autoscaling_algorithm=NONE --environment_type=DOCKER --environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest ---influx_db_name=beam_test_metrics ---influx_hostname=http://10.128.0.96:8086 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_Combine_Go_Batch_Fanout_4.txt b/.github/workflows/load-tests-pipeline-options/go_Combine_Dataflow_Batch_Fanout_4.txt similarity index 78% rename from .github/workflows/load-tests-job-configs/config_Combine_Go_Batch_Fanout_4.txt rename to .github/workflows/load-tests-pipeline-options/go_Combine_Dataflow_Batch_Fanout_4.txt index 5f3a18583270..a64b0c9e7c2c 100644 --- a/.github/workflows/load-tests-job-configs/config_Combine_Go_Batch_Fanout_4.txt +++ b/.github/workflows/load-tests-pipeline-options/go_Combine_Dataflow_Batch_Fanout_4.txt @@ -1,4 +1,3 @@ -############################################################################### # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -13,10 +12,8 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. -############################################################################### ---job_name=load-tests-go-dataflow-batch-combine-2- ---project=apache-beam-testing +# limitations under the License. + --region=us-central1 --temp_location=gs://temp-storage-for-perf-tests/loadtests --staging_location=gs://temp-storage-for-perf-tests/loadtests @@ -29,6 +26,4 @@ --autoscaling_algorithm=NONE --environment_type=DOCKER --environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest ---influx_db_name=beam_test_metrics ---influx_hostname=http://10.128.0.96:8086 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_Combine_Go_Batch_Fanout_8.txt b/.github/workflows/load-tests-pipeline-options/go_Combine_Dataflow_Batch_Fanout_8.txt similarity index 78% rename from .github/workflows/load-tests-job-configs/config_Combine_Go_Batch_Fanout_8.txt rename to .github/workflows/load-tests-pipeline-options/go_Combine_Dataflow_Batch_Fanout_8.txt index eba65b666a25..23ef62663b1c 100644 --- a/.github/workflows/load-tests-job-configs/config_Combine_Go_Batch_Fanout_8.txt +++ b/.github/workflows/load-tests-pipeline-options/go_Combine_Dataflow_Batch_Fanout_8.txt @@ -1,4 +1,3 @@ -############################################################################### # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -13,10 +12,8 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. -############################################################################### ---job_name=load-tests-go-dataflow-batch-combine-3- ---project=apache-beam-testing +# limitations under the License. + --region=us-central1 --temp_location=gs://temp-storage-for-perf-tests/loadtests --staging_location=gs://temp-storage-for-perf-tests/loadtests @@ -29,6 +26,4 @@ --autoscaling_algorithm=NONE --environment_type=DOCKER --environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest ---influx_db_name=beam_test_metrics ---influx_hostname=http://10.128.0.96:8086 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_Combine_Flink_Batch_10b.txt b/.github/workflows/load-tests-pipeline-options/go_Combine_Flink_Batch_10b.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_Combine_Flink_Batch_10b.txt rename to .github/workflows/load-tests-pipeline-options/go_Combine_Flink_Batch_10b.txt diff --git a/.github/workflows/load-tests-job-configs/go_Combine_Flink_Batch_Fanout_4.txt b/.github/workflows/load-tests-pipeline-options/go_Combine_Flink_Batch_Fanout_4.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_Combine_Flink_Batch_Fanout_4.txt rename to .github/workflows/load-tests-pipeline-options/go_Combine_Flink_Batch_Fanout_4.txt diff --git a/.github/workflows/load-tests-job-configs/go_Combine_Flink_Batch_Fanout_8.txt b/.github/workflows/load-tests-pipeline-options/go_Combine_Flink_Batch_Fanout_8.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_Combine_Flink_Batch_Fanout_8.txt rename to .github/workflows/load-tests-pipeline-options/go_Combine_Flink_Batch_Fanout_8.txt diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_100b.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Dataflow_Batch_100b.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_100b.txt rename to .github/workflows/load-tests-pipeline-options/go_GBK_Dataflow_Batch_100b.txt diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_100kb.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Dataflow_Batch_100kb.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_100kb.txt rename to .github/workflows/load-tests-pipeline-options/go_GBK_Dataflow_Batch_100kb.txt diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_10b.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Dataflow_Batch_10b.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_10b.txt rename to .github/workflows/load-tests-pipeline-options/go_GBK_Dataflow_Batch_10b.txt diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Fanout_4.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Dataflow_Batch_Fanout_4.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Fanout_4.txt rename to .github/workflows/load-tests-pipeline-options/go_GBK_Dataflow_Batch_Fanout_4.txt diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Fanout_8.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Dataflow_Batch_Fanout_8.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Fanout_8.txt rename to .github/workflows/load-tests-pipeline-options/go_GBK_Dataflow_Batch_Fanout_8.txt diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Reiteration_10KB.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Dataflow_Batch_Reiteration_10KB.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Reiteration_10KB.txt rename to .github/workflows/load-tests-pipeline-options/go_GBK_Dataflow_Batch_Reiteration_10KB.txt diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Reiteration_2MB.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Dataflow_Batch_Reiteration_2MB.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Reiteration_2MB.txt rename to .github/workflows/load-tests-pipeline-options/go_GBK_Dataflow_Batch_Reiteration_2MB.txt diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_100b.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_100b.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_100b.txt rename to .github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_100b.txt diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_100kb.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_100kb.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_100kb.txt rename to .github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_100kb.txt diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_10b.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_10b.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_10b.txt rename to .github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_10b.txt diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_Fanout_4.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Fanout_4.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_Fanout_4.txt rename to .github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Fanout_4.txt diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_Fanout_8.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Fanout_8.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_Fanout_8.txt rename to .github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Fanout_8.txt diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_Reiteration_10KB.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Reiteration_10KB.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_Reiteration_10KB.txt rename to .github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Reiteration_10KB.txt diff --git a/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_100_Counters.txt b/.github/workflows/load-tests-pipeline-options/go_ParDo_Dataflow_Batch_100_Counters.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_100_Counters.txt rename to .github/workflows/load-tests-pipeline-options/go_ParDo_Dataflow_Batch_100_Counters.txt diff --git a/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_10_Counters.txt b/.github/workflows/load-tests-pipeline-options/go_ParDo_Dataflow_Batch_10_Counters.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_10_Counters.txt rename to .github/workflows/load-tests-pipeline-options/go_ParDo_Dataflow_Batch_10_Counters.txt diff --git a/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_10_Iterations.txt b/.github/workflows/load-tests-pipeline-options/go_ParDo_Dataflow_Batch_10_Iterations.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_10_Iterations.txt rename to .github/workflows/load-tests-pipeline-options/go_ParDo_Dataflow_Batch_10_Iterations.txt diff --git a/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_200_Iterations.txt b/.github/workflows/load-tests-pipeline-options/go_ParDo_Dataflow_Batch_200_Iterations.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_200_Iterations.txt rename to .github/workflows/load-tests-pipeline-options/go_ParDo_Dataflow_Batch_200_Iterations.txt diff --git a/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_100_counters.txt b/.github/workflows/load-tests-pipeline-options/go_ParDo_Flink_Batch_100_counters.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_100_counters.txt rename to .github/workflows/load-tests-pipeline-options/go_ParDo_Flink_Batch_100_counters.txt diff --git a/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_10_counters.txt b/.github/workflows/load-tests-pipeline-options/go_ParDo_Flink_Batch_10_counters.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_10_counters.txt rename to .github/workflows/load-tests-pipeline-options/go_ParDo_Flink_Batch_10_counters.txt diff --git a/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_10_times.txt b/.github/workflows/load-tests-pipeline-options/go_ParDo_Flink_Batch_10_times.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_10_times.txt rename to .github/workflows/load-tests-pipeline-options/go_ParDo_Flink_Batch_10_times.txt diff --git a/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_200_times.txt b/.github/workflows/load-tests-pipeline-options/go_ParDo_Flink_Batch_200_times.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_200_times.txt rename to .github/workflows/load-tests-pipeline-options/go_ParDo_Flink_Batch_200_times.txt diff --git a/.github/workflows/load-tests-job-configs/config_SideInput_Go_Batch_Dataflow_First_Iterable.txt b/.github/workflows/load-tests-pipeline-options/go_SideInput_Dataflow_Batch_First_Iterable.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/config_SideInput_Go_Batch_Dataflow_First_Iterable.txt rename to .github/workflows/load-tests-pipeline-options/go_SideInput_Dataflow_Batch_First_Iterable.txt diff --git a/.github/workflows/load-tests-job-configs/config_SideInput_Go_Batch_Dataflow_Iterable.txt b/.github/workflows/load-tests-pipeline-options/go_SideInput_Dataflow_Batch_Iterable.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/config_SideInput_Go_Batch_Dataflow_Iterable.txt rename to .github/workflows/load-tests-pipeline-options/go_SideInput_Dataflow_Batch_Iterable.txt diff --git a/.github/workflows/load-tests-job-configs/go_SideInput_Flink_Batch_First_Iterable.txt b/.github/workflows/load-tests-pipeline-options/go_SideInput_Flink_Batch_First_Iterable.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_SideInput_Flink_Batch_First_Iterable.txt rename to .github/workflows/load-tests-pipeline-options/go_SideInput_Flink_Batch_First_Iterable.txt diff --git a/.github/workflows/load-tests-job-configs/go_SideInput_Flink_Batch_Iterable.txt b/.github/workflows/load-tests-pipeline-options/go_SideInput_Flink_Batch_Iterable.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/go_SideInput_Flink_Batch_Iterable.txt rename to .github/workflows/load-tests-pipeline-options/go_SideInput_Flink_Batch_Iterable.txt diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt b/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt rename to .github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_100b_Single_Key.txt b/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Batch_100b_Single_Key.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_100b_Single_Key.txt rename to .github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Batch_100b_Single_Key.txt diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_10kB.txt b/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Batch_10kB.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_10kB.txt rename to .github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Batch_10kB.txt diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_2MB.txt b/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Batch_2MB.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_2MB.txt rename to .github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Batch_2MB.txt diff --git a/.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_MultipleKey.txt b/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Streaming_2GB_MultipleKey.txt similarity index 81% rename from .github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_MultipleKey.txt rename to .github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Streaming_2GB_MultipleKey.txt index 5fd9518bc8d0..94cbd383a736 100644 --- a/.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_MultipleKey.txt +++ b/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Streaming_2GB_MultipleKey.txt @@ -1,4 +1,3 @@ -############################################################################### # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -13,9 +12,8 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. -############################################################################### ---project=apache-beam-testing +# limitations under the License. + --region=us-central1 --appName=load_tests_Java_Dataflow_streaming_CoGBK_2 --tempLocation=gs://temp-storage-for-perf-tests/loadtests @@ -29,6 +27,4 @@ --streaming=true --inputWindowDurationSec=1200 --coInputWindowDurationSec=1200 ---influxDatabase=beam_test_metrics ---influxHost=http://10.128.0.96:8086 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_Reiteration_10KB.txt b/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Streaming_2GB_Reiteration_10KB.txt similarity index 81% rename from .github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_Reiteration_10KB.txt rename to .github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Streaming_2GB_Reiteration_10KB.txt index 2840fe75d5af..6b16e4546ff6 100644 --- a/.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_Reiteration_10KB.txt +++ b/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Streaming_2GB_Reiteration_10KB.txt @@ -1,4 +1,3 @@ -############################################################################### # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -13,9 +12,8 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. -############################################################################### ---project=apache-beam-testing +# limitations under the License. + --region=us-central1 --appName=load_tests_Java_Dataflow_streaming_CoGBK_3 --tempLocation=gs://temp-storage-for-perf-tests/loadtests @@ -29,6 +27,4 @@ --streaming=true --inputWindowDurationSec=1200 --coInputWindowDurationSec=1200 ---influxDatabase=beam_test_metrics ---influxHost=http://10.128.0.96:8086 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_Reiteration_2MB.txt b/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Streaming_2GB_Reiteration_2MB.txt similarity index 81% rename from .github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_Reiteration_2MB.txt rename to .github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Streaming_2GB_Reiteration_2MB.txt index bcc8a36cf31f..d643e21d18cb 100644 --- a/.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_Reiteration_2MB.txt +++ b/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Streaming_2GB_Reiteration_2MB.txt @@ -1,4 +1,3 @@ -############################################################################### # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -13,9 +12,8 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. -############################################################################### ---project=apache-beam-testing +# limitations under the License. + --region=us-central1 --appName=load_tests_Java_Dataflow_streaming_CoGBK_4 --tempLocation=gs://temp-storage-for-perf-tests/loadtests @@ -29,6 +27,4 @@ --streaming=true --inputWindowDurationSec=1200 --coInputWindowDurationSec=1200 ---influxDatabase=beam_test_metrics ---influxHost=http://10.128.0.96:8086 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_SingleKey.txt b/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Streaming_2GB_SingleKey.txt similarity index 81% rename from .github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_SingleKey.txt rename to .github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Streaming_2GB_SingleKey.txt index afae1a1bd6bf..82c6612e920a 100644 --- a/.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_SingleKey.txt +++ b/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_Streaming_2GB_SingleKey.txt @@ -1,4 +1,3 @@ -############################################################################### # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -13,9 +12,8 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. -############################################################################### ---project=apache-beam-testing +# limitations under the License. + --region=us-central1 --appName=load_tests_Java_Dataflow_streaming_CoGBK_1 --tempLocation=gs://temp-storage-for-perf-tests/loadtests @@ -29,6 +27,4 @@ --streaming=true --inputWindowDurationSec=1200 --coInputWindowDurationSec=1200 ---influxDatabase=beam_test_metrics ---influxHost=http://10.128.0.96:8086 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_100b_Multiple_Keys.txt b/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_V2_Streaming_Java_100b_Multiple_Keys.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_100b_Multiple_Keys.txt rename to .github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_V2_Streaming_Java_100b_Multiple_Keys.txt diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_100b_Single_Key.txt b/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_V2_Streaming_Java_100b_Single_Key.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_100b_Single_Key.txt rename to .github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_V2_Streaming_Java_100b_Single_Key.txt diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_10kB.txt b/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_V2_Streaming_Java_10kB.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_10kB.txt rename to .github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_V2_Streaming_Java_10kB.txt diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_2MB.txt b/.github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_V2_Streaming_Java_2MB.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_2MB.txt rename to .github/workflows/load-tests-pipeline-options/java_CoGBK_Dataflow_V2_Streaming_Java_2MB.txt diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_100b_Multiple_Keys.txt b/.github/workflows/load-tests-pipeline-options/java_CoGBK_SparkStructuredStreaming_Batch_100b_Multiple_Keys.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_100b_Multiple_Keys.txt rename to .github/workflows/load-tests-pipeline-options/java_CoGBK_SparkStructuredStreaming_Batch_100b_Multiple_Keys.txt diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_100b_Single_Key.txt b/.github/workflows/load-tests-pipeline-options/java_CoGBK_SparkStructuredStreaming_Batch_100b_Single_Key.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_100b_Single_Key.txt rename to .github/workflows/load-tests-pipeline-options/java_CoGBK_SparkStructuredStreaming_Batch_100b_Single_Key.txt diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_10kB.txt b/.github/workflows/load-tests-pipeline-options/java_CoGBK_SparkStructuredStreaming_Batch_10kB.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_10kB.txt rename to .github/workflows/load-tests-pipeline-options/java_CoGBK_SparkStructuredStreaming_Batch_10kB.txt diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_2MB.txt b/.github/workflows/load-tests-pipeline-options/java_CoGBK_SparkStructuredStreaming_Batch_2MB.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_2MB.txt rename to .github/workflows/load-tests-pipeline-options/java_CoGBK_SparkStructuredStreaming_Batch_2MB.txt diff --git a/.github/workflows/load-tests-job-configs/config_Combine_Java_Dataflow_Batch_10b.txt b/.github/workflows/load-tests-pipeline-options/java_Combine_Dataflow_Batch_10b.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/config_Combine_Java_Dataflow_Batch_10b.txt rename to .github/workflows/load-tests-pipeline-options/java_Combine_Dataflow_Batch_10b.txt diff --git a/.github/workflows/load-tests-job-configs/config_Combine_Java_Dataflow_Batch_Fanout_4.txt b/.github/workflows/load-tests-pipeline-options/java_Combine_Dataflow_Batch_Fanout_4.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/config_Combine_Java_Dataflow_Batch_Fanout_4.txt rename to .github/workflows/load-tests-pipeline-options/java_Combine_Dataflow_Batch_Fanout_4.txt diff --git a/.github/workflows/load-tests-job-configs/config_Combine_Java_Dataflow_Batch_Fanout_8.txt b/.github/workflows/load-tests-pipeline-options/java_Combine_Dataflow_Batch_Fanout_8.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/config_Combine_Java_Dataflow_Batch_Fanout_8.txt rename to .github/workflows/load-tests-pipeline-options/java_Combine_Dataflow_Batch_Fanout_8.txt diff --git a/.github/workflows/load-tests-job-configs/java_Combine_Dataflow_Streaming_10b.txt b/.github/workflows/load-tests-pipeline-options/java_Combine_Dataflow_Streaming_10b.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_Combine_Dataflow_Streaming_10b.txt rename to .github/workflows/load-tests-pipeline-options/java_Combine_Dataflow_Streaming_10b.txt diff --git a/.github/workflows/load-tests-job-configs/java_Combine_Dataflow_Streaming_Fanout_4.txt b/.github/workflows/load-tests-pipeline-options/java_Combine_Dataflow_Streaming_Fanout_4.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_Combine_Dataflow_Streaming_Fanout_4.txt rename to .github/workflows/load-tests-pipeline-options/java_Combine_Dataflow_Streaming_Fanout_4.txt diff --git a/.github/workflows/load-tests-job-configs/java_Combine_Dataflow_Streaming_Fanout_8.txt b/.github/workflows/load-tests-pipeline-options/java_Combine_Dataflow_Streaming_Fanout_8.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_Combine_Dataflow_Streaming_Fanout_8.txt rename to .github/workflows/load-tests-pipeline-options/java_Combine_Dataflow_Streaming_Fanout_8.txt diff --git a/.github/workflows/load-tests-job-configs/java_Combine_SparkStructuredStreaming_Batch_10b.txt b/.github/workflows/load-tests-pipeline-options/java_Combine_SparkStructuredStreaming_Batch_10b.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_Combine_SparkStructuredStreaming_Batch_10b.txt rename to .github/workflows/load-tests-pipeline-options/java_Combine_SparkStructuredStreaming_Batch_10b.txt diff --git a/.github/workflows/load-tests-job-configs/java_Combine_SparkStructuredStreaming_Batch_Fanout_4.txt b/.github/workflows/load-tests-pipeline-options/java_Combine_SparkStructuredStreaming_Batch_Fanout_4.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_Combine_SparkStructuredStreaming_Batch_Fanout_4.txt rename to .github/workflows/load-tests-pipeline-options/java_Combine_SparkStructuredStreaming_Batch_Fanout_4.txt diff --git a/.github/workflows/load-tests-job-configs/java_Combine_SparkStructuredStreaming_Batch_Fanout_8.txt b/.github/workflows/load-tests-pipeline-options/java_Combine_SparkStructuredStreaming_Batch_Fanout_8.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_Combine_SparkStructuredStreaming_Batch_Fanout_8.txt rename to .github/workflows/load-tests-pipeline-options/java_Combine_SparkStructuredStreaming_Batch_Fanout_8.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_100B_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Batch_2GB_of_100B_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_100B_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Batch_2GB_of_100B_records.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_100kB_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Batch_2GB_of_100kB_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_100kB_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Batch_2GB_of_100kB_records.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_10B_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Batch_2GB_of_10B_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_10B_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Batch_2GB_of_10B_records.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Batch_reiterate_4_times_10kB_values.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_reiterate_4_times_10kB_values.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Batch_reiterate_4_times_10kB_values.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_reiterate_4_times_2MB_values.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Batch_reiterate_4_times_2MB_values.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_reiterate_4_times_2MB_values.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Batch_reiterate_4_times_2MB_values.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_100B_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Streaming_2GB_of_100B_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_100B_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Streaming_2GB_of_100B_records.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_10B_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Streaming_2GB_of_10B_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_10B_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Streaming_2GB_of_10B_records.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Streaming_reiterate_4_times_10kB_values.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_reiterate_4_times_10kB_values.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Streaming_reiterate_4_times_10kB_values.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_reiterate_4_times_2MB_values.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Streaming_reiterate_4_times_2MB_values.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_reiterate_4_times_2MB_values.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_Streaming_reiterate_4_times_2MB_values.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100B_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100B_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100B_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100B_records.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100kB_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100kB_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100kB_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100kB_records.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_10B_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_10B_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_10B_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_10B_records.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_10kB_values.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_10kB_values.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_10kB_values.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_2MB_values.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_2MB_values.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_2MB_values.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_2MB_values.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100B_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100B_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100B_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100B_records.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100kB_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100kB_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100kB_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100kB_records.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_10B_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_10B_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_10B_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_10B_records.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_10kB_values.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_10kB_values.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_10kB_values.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_2MB_values.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_2MB_values.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_2MB_values.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_2MB_values.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100B_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100B_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100B_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100B_records.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100kB_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100kB_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100kB_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100kB_records.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_10B_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_10B_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_10B_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_10B_records.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_10kB_values.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_10kB_values.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_10kB_values.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_2MB_values.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_2MB_values.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_2MB_values.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_2MB_values.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100B_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100B_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100B_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100B_records.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100kB_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100kB_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100kB_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100kB_records.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_10B_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_10B_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_10B_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_10B_records.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_10kB_values.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_10kB_values.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_10kB_values.txt diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_2MB_values.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_2MB_values.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_2MB_values.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_2MB_values.txt diff --git a/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_100B_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_2GB_of_100B_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_100B_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_2GB_of_100B_records.txt diff --git a/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_100kB_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_2GB_of_100kB_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_100kB_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_2GB_of_100kB_records.txt diff --git a/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_10B_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_2GB_of_10B_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_10B_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_2GB_of_10B_records.txt diff --git a/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_10kB_values.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_10kB_values.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_10kB_values.txt diff --git a/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_2MB_values.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_2MB_values.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_2MB_values.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_2MB_values.txt diff --git a/.github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-1.txt b/.github/workflows/load-tests-pipeline-options/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-1.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-1.txt rename to .github/workflows/load-tests-pipeline-options/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-1.txt diff --git a/.github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-2.txt b/.github/workflows/load-tests-pipeline-options/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-2.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-2.txt rename to .github/workflows/load-tests-pipeline-options/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-2.txt diff --git a/.github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-3.txt b/.github/workflows/load-tests-pipeline-options/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-3.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-3.txt rename to .github/workflows/load-tests-pipeline-options/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-3.txt diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_100_counters.txt b/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Batch_100_counters.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_100_counters.txt rename to .github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Batch_100_counters.txt diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_10_counters.txt b/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Batch_10_counters.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_10_counters.txt rename to .github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Batch_10_counters.txt diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_10_times.txt b/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Batch_10_times.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_10_times.txt rename to .github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Batch_10_times.txt diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_200_times.txt b/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Batch_200_times.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_200_times.txt rename to .github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Batch_200_times.txt diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_100_counters.txt b/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Streaming_100_counters.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_100_counters.txt rename to .github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Streaming_100_counters.txt diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_10_counters.txt b/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Streaming_10_counters.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_10_counters.txt rename to .github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Streaming_10_counters.txt diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_10_times.txt b/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Streaming_10_times.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_10_times.txt rename to .github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Streaming_10_times.txt diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_200_times.txt b/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Streaming_200_times.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_200_times.txt rename to .github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_Streaming_200_times.txt diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_100_counters.txt b/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Batch_JavaVersions_100_counters.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_100_counters.txt rename to .github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Batch_JavaVersions_100_counters.txt diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_counters.txt b/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_counters.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_counters.txt rename to .github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_counters.txt diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_times.txt b/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_times.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_times.txt rename to .github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_times.txt diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_200_times.txt b/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Batch_JavaVersions_200_times.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_200_times.txt rename to .github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Batch_JavaVersions_200_times.txt diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_100_counters.txt b/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Streaming_JavaVersions_100_counters.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_100_counters.txt rename to .github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Streaming_JavaVersions_100_counters.txt diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_counters.txt b/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_counters.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_counters.txt rename to .github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_counters.txt diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_times.txt b/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_times.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_times.txt rename to .github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_times.txt diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_200_times.txt b/.github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Streaming_JavaVersions_200_times.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_200_times.txt rename to .github/workflows/load-tests-pipeline-options/java_ParDo_Dataflow_V2_Streaming_JavaVersions_200_times.txt diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_100_counters.txt b/.github/workflows/load-tests-pipeline-options/java_ParDo_SparkStructuredStreaming_Batch_100_counters.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_100_counters.txt rename to .github/workflows/load-tests-pipeline-options/java_ParDo_SparkStructuredStreaming_Batch_100_counters.txt diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_10_counters.txt b/.github/workflows/load-tests-pipeline-options/java_ParDo_SparkStructuredStreaming_Batch_10_counters.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_10_counters.txt rename to .github/workflows/load-tests-pipeline-options/java_ParDo_SparkStructuredStreaming_Batch_10_counters.txt diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_10_times.txt b/.github/workflows/load-tests-pipeline-options/java_ParDo_SparkStructuredStreaming_Batch_10_times.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_10_times.txt rename to .github/workflows/load-tests-pipeline-options/java_ParDo_SparkStructuredStreaming_Batch_10_times.txt diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_200_times.txt b/.github/workflows/load-tests-pipeline-options/java_ParDo_SparkStructuredStreaming_Batch_200_times.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_200_times.txt rename to .github/workflows/load-tests-pipeline-options/java_ParDo_SparkStructuredStreaming_Batch_200_times.txt diff --git a/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Dataflow.txt b/.github/workflows/load-tests-pipeline-options/java_Smoke_GroupByKey_Dataflow.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Dataflow.txt rename to .github/workflows/load-tests-pipeline-options/java_Smoke_GroupByKey_Dataflow.txt diff --git a/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Direct.txt b/.github/workflows/load-tests-pipeline-options/java_Smoke_GroupByKey_Direct.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Direct.txt rename to .github/workflows/load-tests-pipeline-options/java_Smoke_GroupByKey_Direct.txt diff --git a/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Flink.txt b/.github/workflows/load-tests-pipeline-options/java_Smoke_GroupByKey_Flink.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Flink.txt rename to .github/workflows/load-tests-pipeline-options/java_Smoke_GroupByKey_Flink.txt diff --git a/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Spark.txt b/.github/workflows/load-tests-pipeline-options/java_Smoke_GroupByKey_Spark.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Spark.txt rename to .github/workflows/load-tests-pipeline-options/java_Smoke_GroupByKey_Spark.txt diff --git a/.github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt rename to .github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt diff --git a/.github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Batch_100b_Single_Key.txt b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Batch_100b_Single_Key.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Batch_100b_Single_Key.txt rename to .github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Batch_100b_Single_Key.txt diff --git a/.github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Batch_10kB.txt b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Batch_10kB.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Batch_10kB.txt rename to .github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Batch_10kB.txt diff --git a/.github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Batch_2MB.txt b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Batch_2MB.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Batch_2MB.txt rename to .github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Batch_2MB.txt diff --git a/.github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Flink_Batch_100b_Multiple_Keys.txt b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Flink_Batch_100b_Multiple_Keys.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Flink_Batch_100b_Multiple_Keys.txt rename to .github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Flink_Batch_100b_Multiple_Keys.txt diff --git a/.github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Flink_Batch_100b_Single_Key.txt b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Flink_Batch_100b_Single_Key.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Flink_Batch_100b_Single_Key.txt rename to .github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Flink_Batch_100b_Single_Key.txt diff --git a/.github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Flink_Batch_10kB.txt b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Flink_Batch_10kB.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Flink_Batch_10kB.txt rename to .github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Flink_Batch_10kB.txt diff --git a/.github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Streaming_100b_Multiple_Keys.txt b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Streaming_100b_Multiple_Keys.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Streaming_100b_Multiple_Keys.txt rename to .github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Streaming_100b_Multiple_Keys.txt diff --git a/.github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Streaming_100b_Single_Key.txt b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Streaming_100b_Single_Key.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Streaming_100b_Single_Key.txt rename to .github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Streaming_100b_Single_Key.txt diff --git a/.github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Streaming_10kB.txt b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Streaming_10kB.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Streaming_10kB.txt rename to .github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Streaming_10kB.txt diff --git a/.github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Streaming_2MB.txt b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Streaming_2MB.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_CoGBK_Dataflow_Streaming_2MB.txt rename to .github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Streaming_2MB.txt diff --git a/.github/workflows/load-tests-job-configs/config_Combine_Python_Batch_2GB_10b.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Batch_2GB_10b.txt similarity index 77% rename from .github/workflows/load-tests-job-configs/config_Combine_Python_Batch_2GB_10b.txt rename to .github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Batch_2GB_10b.txt index a6dabb5e5086..29b301551283 100644 --- a/.github/workflows/load-tests-job-configs/config_Combine_Python_Batch_2GB_10b.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Batch_2GB_10b.txt @@ -1,4 +1,3 @@ -############################################################################### # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -13,10 +12,8 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. -############################################################################### ---job_name=load-tests-python-dataflow-batch-combine-1- ---project=apache-beam-testing +# limitations under the License. + --region=us-central1 --temp_location=gs://temp-storage-for-perf-tests/smoketests --publish_to_big_query=true @@ -27,6 +24,4 @@ --num_workers=5 --autoscaling_algorithm=NONE --top_count=20 ---influxDatabase=beam_test_metrics ---influxHost=http://10.128.0.96:8086 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_Combine_Python_Batch_2GB_Fanout_4.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Batch_2GB_Fanout_4.txt similarity index 79% rename from .github/workflows/load-tests-job-configs/config_Combine_Python_Batch_2GB_Fanout_4.txt rename to .github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Batch_2GB_Fanout_4.txt index 7639456296b6..7405d6b75a82 100644 --- a/.github/workflows/load-tests-job-configs/config_Combine_Python_Batch_2GB_Fanout_4.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Batch_2GB_Fanout_4.txt @@ -1,4 +1,3 @@ -############################################################################### # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -14,9 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -############################################################################### ---job_name=load-tests-python-dataflow-batch-combine-2- ---project=apache-beam-testing + --region=us-central1 --temp_location=gs://temp-storage-for-perf-tests/smoketests --publish_to_big_query=true @@ -28,6 +25,4 @@ --autoscaling_algorithm=NONE --fanout=4 --top_count=20 ---influxDatabase=beam_test_metrics ---influxHost=http://10.128.0.96:8086 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_Combine_Python_Batch_2GB_Fanout_8.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Batch_2GB_Fanout_8.txt similarity index 79% rename from .github/workflows/load-tests-job-configs/config_Combine_Python_Batch_2GB_Fanout_8.txt rename to .github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Batch_2GB_Fanout_8.txt index e5d46791a83c..b45c4eb1bd4a 100644 --- a/.github/workflows/load-tests-job-configs/config_Combine_Python_Batch_2GB_Fanout_8.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Batch_2GB_Fanout_8.txt @@ -1,4 +1,3 @@ -############################################################################### # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -14,9 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -############################################################################### ---job_name=load-tests-python-dataflow-batch-combine-3- ---project=apache-beam-testing + --region=us-central1 --temp_location=gs://temp-storage-for-perf-tests/smoketests --publish_to_big_query=true @@ -28,6 +25,4 @@ --autoscaling_algorithm=NONE --fanout=8 --top_count=20 ---influxDatabase=beam_test_metrics ---influxHost=http://10.128.0.96:8086 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_10_byte_records.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Streaming_2GB_10_byte_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_10_byte_records.txt rename to .github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Streaming_2GB_10_byte_records.txt diff --git a/.github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_Fanout_4.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Streaming_2GB_Fanout_4.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_Fanout_4.txt rename to .github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Streaming_2GB_Fanout_4.txt diff --git a/.github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_Fanout_8.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Streaming_2GB_Fanout_8.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_Fanout_8.txt rename to .github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Streaming_2GB_Fanout_8.txt diff --git a/.github/workflows/load-tests-job-configs/python_Combine_Flink_Batch_2GB_10_byte_records.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_10_byte_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_Combine_Flink_Batch_2GB_10_byte_records.txt rename to .github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_10_byte_records.txt diff --git a/.github/workflows/load-tests-job-configs/python_Combine_Flink_Batch_2GB_Fanout_4.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_4.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_Combine_Flink_Batch_2GB_Fanout_4.txt rename to .github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_4.txt diff --git a/.github/workflows/load-tests-job-configs/python_Combine_Flink_Batch_2GB_Fanout_8.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_8.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_Combine_Flink_Batch_2GB_Fanout_8.txt rename to .github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_8.txt diff --git a/.github/workflows/load-tests-job-configs/python_Combine_Flink_Streaming_2GB_10_byte_records.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_10_byte_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_Combine_Flink_Streaming_2GB_10_byte_records.txt rename to .github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_10_byte_records.txt diff --git a/.github/workflows/load-tests-job-configs/python_Combine_Flink_Streaming_2GB_Fanout_4.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_4.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_Combine_Flink_Streaming_2GB_Fanout_4.txt rename to .github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_4.txt diff --git a/.github/workflows/load-tests-job-configs/python_Combine_Flink_Streaming_2GB_Fanout_8.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_8.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_Combine_Flink_Streaming_2GB_Fanout_8.txt rename to .github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_8.txt diff --git a/.github/workflows/load-tests-job-configs/config_FnApiRunner_Python_Microbenchmark.txt b/.github/workflows/load-tests-pipeline-options/python_FnApiRunner_Microbenchmark.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/config_FnApiRunner_Python_Microbenchmark.txt rename to .github/workflows/load-tests-pipeline-options/python_FnApiRunner_Microbenchmark.txt diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_2GB_of_100B_records.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Batch_2GB_of_100B_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_2GB_of_100B_records.txt rename to .github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Batch_2GB_of_100B_records.txt diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_2GB_of_100kB_records.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Batch_2GB_of_100kB_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_2GB_of_100kB_records.txt rename to .github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Batch_2GB_of_100kB_records.txt diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_2GB_of_10B_records.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Batch_2GB_of_10B_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_2GB_of_10B_records.txt rename to .github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Batch_2GB_of_10B_records.txt diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_2GB_of_100B_records.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_2GB_of_100B_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_2GB_of_100B_records.txt rename to .github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_2GB_of_100B_records.txt diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt rename to .github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_2GB_of_10B_records.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_2GB_of_10B_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_2GB_of_10B_records.txt rename to .github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_2GB_of_10B_records.txt diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_2GB_of_100B_records.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_100B_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_2GB_of_100B_records.txt rename to .github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_100B_records.txt diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_2GB_of_10B_records.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_10B_records.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_2GB_of_10B_records.txt rename to .github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_10B_records.txt diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt rename to .github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt diff --git a/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_10kB_values.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_10kB_values.txt rename to .github/workflows/load-tests-pipeline-options/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_10kB_values.txt diff --git a/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_2MB_values.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_2MB_values.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_2MB_values.txt rename to .github/workflows/load-tests-pipeline-options/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_2MB_values.txt diff --git a/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_10kB_values.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_10kB_values.txt rename to .github/workflows/load-tests-pipeline-options/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_10kB_values.txt diff --git a/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_2MB_values.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_2MB_values.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_2MB_values.txt rename to .github/workflows/load-tests-pipeline-options/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_2MB_values.txt diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_100_Counters.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Batch_100_Counters.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_100_Counters.txt rename to .github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Batch_100_Counters.txt diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_10_Counters.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Batch_10_Counters.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_10_Counters.txt rename to .github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Batch_10_Counters.txt diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_10_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Batch_10_Iterations.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_10_Iterations.txt rename to .github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Batch_10_Iterations.txt diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_200_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Batch_200_Iterations.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_200_Iterations.txt rename to .github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Batch_200_Iterations.txt diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_100_Counters.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_100_Counters.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_100_Counters.txt rename to .github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_100_Counters.txt diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_10_Counters.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_10_Counters.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_10_Counters.txt rename to .github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_10_Counters.txt diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_10_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_10_Iterations.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_10_Iterations.txt rename to .github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_10_Iterations.txt diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_200_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_200_Iterations.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_200_Iterations.txt rename to .github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_200_Iterations.txt diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Batch_10_Counters.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Counters.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_ParDo_Flink_Batch_10_Counters.txt rename to .github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Counters.txt diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Batch_10_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Iterations.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_ParDo_Flink_Batch_10_Iterations.txt rename to .github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Iterations.txt diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Batch_200_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_200_Iterations.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_ParDo_Flink_Batch_200_Iterations.txt rename to .github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_200_Iterations.txt diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_100_Counters.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_100_Counters.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_100_Counters.txt rename to .github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_100_Counters.txt diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_10_Counters.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Counters.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_10_Counters.txt rename to .github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Counters.txt diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_10_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Iterations.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_10_Iterations.txt rename to .github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Iterations.txt diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_200_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_200_Iterations.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_200_Iterations.txt rename to .github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_200_Iterations.txt diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_5_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_5_Iterations.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_5_Iterations.txt rename to .github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_5_Iterations.txt diff --git a/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1000window_first_iterable.txt b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1000window_first_iterable.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1000window_first_iterable.txt rename to .github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1000window_first_iterable.txt diff --git a/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1000window_iterable.txt b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1000window_iterable.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1000window_iterable.txt rename to .github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1000window_iterable.txt diff --git a/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1window_first_iterable.txt b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1window_first_iterable.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1window_first_iterable.txt rename to .github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1window_first_iterable.txt diff --git a/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1window_iterable.txt b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1window_iterable.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1window_iterable.txt rename to .github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1window_iterable.txt diff --git a/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1000window_1key_percent_dict.txt b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1000window_1key_percent_dict.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1000window_1key_percent_dict.txt rename to .github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1000window_1key_percent_dict.txt diff --git a/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1000window_99key_percent_dict.txt b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1000window_99key_percent_dict.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1000window_99key_percent_dict.txt rename to .github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1000window_99key_percent_dict.txt diff --git a/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_1key_percent_dict.txt b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_1key_percent_dict.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_1key_percent_dict.txt rename to .github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_1key_percent_dict.txt diff --git a/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_99key_percent_dict.txt b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_99key_percent_dict.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_99key_percent_dict.txt rename to .github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_99key_percent_dict.txt diff --git a/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_first_list.txt b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_first_list.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_first_list.txt rename to .github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_first_list.txt diff --git a/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_list.txt b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_list.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_list.txt rename to .github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_list.txt diff --git a/.github/workflows/load-tests-job-configs/python_Smoke_GroupByKey_Dataflow.txt b/.github/workflows/load-tests-pipeline-options/python_Smoke_GroupByKey_Dataflow.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_Smoke_GroupByKey_Dataflow.txt rename to .github/workflows/load-tests-pipeline-options/python_Smoke_GroupByKey_Dataflow.txt diff --git a/.github/workflows/load-tests-job-configs/python_Smoke_GroupByKey_Direct.txt b/.github/workflows/load-tests-pipeline-options/python_Smoke_GroupByKey_Direct.txt similarity index 100% rename from .github/workflows/load-tests-job-configs/python_Smoke_GroupByKey_Direct.txt rename to .github/workflows/load-tests-pipeline-options/python_Smoke_GroupByKey_Direct.txt diff --git a/.github/workflows/performance-tests-job-configs/config_BigQueryIO_Batch_Java_Avro.txt b/.github/workflows/performance-tests-job-configs/config_BigQueryIO_Batch_Java_Avro.txt deleted file mode 100644 index 5e7e53821231..000000000000 --- a/.github/workflows/performance-tests-job-configs/config_BigQueryIO_Batch_Java_Avro.txt +++ /dev/null @@ -1,38 +0,0 @@ -############################################################################### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -############################################################################### -'["--tempLocation=gs://temp-storage-for-perf-tests/loadtests", -"--project=apache-beam-testing", -"--tempRoot=gs://temp-storage-for-perf-tests/loadtests", -"--writeMethod=FILE_LOADS", -"--writeFormat=AVRO", -"--testBigQueryDataset=beam_performance", -"--testBigQueryTable=bqio_write_10GB_java_avro_", -"--metricsBigQueryDataset=beam_performance", -"--metricsBigQueryTable=bqio_10GB_results_java_batch_avro", -"--influxMeasurement=bqio_10GB_results_java_batch_avro", -"--sourceOptions={ -\"numRecords\":\"10485760\", -\"keySizeBytes\":\"1\", -\"valueSizeBytes\":\"1024\" -}", -"--runner=DataflowRunner", -"--maxNumWorkers=5", -"--numWorkers=5", -"--autoscalingAlgorithm=NONE", -"--influxDatabase=beam_test_metrics", -"--influxHost=http://10.128.0.96:8086"]' \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/config_BigQueryIO_Batch_Java_Json.txt b/.github/workflows/performance-tests-job-configs/config_BigQueryIO_Batch_Java_Json.txt deleted file mode 100644 index 7bd9c30ae738..000000000000 --- a/.github/workflows/performance-tests-job-configs/config_BigQueryIO_Batch_Java_Json.txt +++ /dev/null @@ -1,38 +0,0 @@ -############################################################################### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -############################################################################### -'["--tempLocation=gs://temp-storage-for-perf-tests/loadtests", -"--project=apache-beam-testing", -"--tempRoot=gs://temp-storage-for-perf-tests/loadtests", -"--writeMethod=FILE_LOADS", -"--writeFormat=JSON", -"--testBigQueryDataset=beam_performance", -"--testBigQueryTable=bqio_write_10GB_java_json_", -"--metricsBigQueryDataset=beam_performance", -"--metricsBigQueryTable=bqio_10GB_results_java_batch_json", -"--influxMeasurement=bqio_10GB_results_java_batch_json", -"--sourceOptions={ -\"numRecords\":\"10485760\", -\"keySizeBytes\":\"1\", -\"valueSizeBytes\":\"1024\" -}", -"--runner=DataflowRunner", -"--maxNumWorkers=5", -"--numWorkers=5", -"--autoscalingAlgorithm=NONE", -"--influxDatabase=beam_test_metrics", -"--influxHost=http://10.128.0.96:8086"]' \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/config_BigQueryIO_Streaming_Java.txt b/.github/workflows/performance-tests-job-configs/config_BigQueryIO_Streaming_Java.txt deleted file mode 100644 index 8bddea5fcb8b..000000000000 --- a/.github/workflows/performance-tests-job-configs/config_BigQueryIO_Streaming_Java.txt +++ /dev/null @@ -1,39 +0,0 @@ -############################################################################### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -############################################################################### -'["--tempLocation=gs://temp-storage-for-perf-tests/loadtests", -"--project=apache-beam-testing", -"--tempRoot=gs://temp-storage-for-perf-tests/loadtests", -"--writeMethod=STREAMING_INSERTS", -"--writeFormat=JSON", -"--pipelineTimeout=1200", -"--testBigQueryDataset=beam_performance", -"--testBigQueryTable=bqio_write_10GB_java_stream_", -"--metricsBigQueryDataset=beam_performance", -"--metricsBigQueryTable=bqio_10GB_results_java_stream", -"--influxMeasurement=bqio_10GB_results_java_stream", -"--sourceOptions={ -\"numRecords\":\"10485760\", -\"keySizeBytes\":\"1\", -\"valueSizeBytes\":\"1024\" -}", -"--runner=DataflowRunner", -"--maxNumWorkers=5", -"--numWorkers=5", -"--autoscalingAlgorithm=NONE", -"--influxDatabase=beam_test_metrics", -"--influxHost=http://10.128.0.96:8086"]' \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/JDBC.txt b/.github/workflows/performance-tests-pipeline-options/JDBC.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/JDBC.txt rename to .github/workflows/performance-tests-pipeline-options/JDBC.txt diff --git a/.github/workflows/performance-tests-job-configs/SQLBigQueryIO_Batch_Java.txt b/.github/workflows/performance-tests-pipeline-options/SQLBigQueryIO_Batch_Java.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/SQLBigQueryIO_Batch_Java.txt rename to .github/workflows/performance-tests-pipeline-options/SQLBigQueryIO_Batch_Java.txt diff --git a/.github/workflows/performance-tests-job-configs/config_TFRecordIOIT.txt b/.github/workflows/performance-tests-pipeline-options/TFRecordIOIT.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/config_TFRecordIOIT.txt rename to .github/workflows/performance-tests-pipeline-options/TFRecordIOIT.txt diff --git a/.github/workflows/performance-tests-job-configs/TFRecordIOIT_HDFS.txt b/.github/workflows/performance-tests-pipeline-options/TFRecordIOIT_HDFS.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/TFRecordIOIT_HDFS.txt rename to .github/workflows/performance-tests-pipeline-options/TFRecordIOIT_HDFS.txt diff --git a/.github/workflows/performance-tests-job-configs/config_AvroIOIT.txt b/.github/workflows/performance-tests-pipeline-options/avroIOIT.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/config_AvroIOIT.txt rename to .github/workflows/performance-tests-pipeline-options/avroIOIT.txt diff --git a/.github/workflows/performance-tests-job-configs/config_AvroIOIT_HDFS.txt b/.github/workflows/performance-tests-pipeline-options/avroIOIT_HDFS.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/config_AvroIOIT_HDFS.txt rename to .github/workflows/performance-tests-pipeline-options/avroIOIT_HDFS.txt diff --git a/.github/workflows/performance-tests-pipeline-options/bigQueryIO_Batch_Java_Avro.txt b/.github/workflows/performance-tests-pipeline-options/bigQueryIO_Batch_Java_Avro.txt new file mode 100644 index 000000000000..922114e63d0e --- /dev/null +++ b/.github/workflows/performance-tests-pipeline-options/bigQueryIO_Batch_Java_Avro.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--tempRoot=gs://temp-storage-for-perf-tests/loadtests +--writeMethod=FILE_LOADS +--writeFormat=AVRO +--testBigQueryDataset=beam_performance +--metricsBigQueryDataset=beam_performance +--metricsBigQueryTable=bqio_10GB_results_java_batch_avro +--influxMeasurement=bqio_10GB_results_java_batch_avro +--sourceOptions={\"numRecords\":\"10485760\",\"keySizeBytes\":\"1\",\"valueSizeBytes\":\"1024\"} +--runner=DataflowRunner +--maxNumWorkers=5 +--numWorkers=5 +--autoscalingAlgorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-pipeline-options/bigQueryIO_Batch_Java_Json.txt b/.github/workflows/performance-tests-pipeline-options/bigQueryIO_Batch_Java_Json.txt new file mode 100644 index 000000000000..627368ec0e24 --- /dev/null +++ b/.github/workflows/performance-tests-pipeline-options/bigQueryIO_Batch_Java_Json.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--tempRoot=gs://temp-storage-for-perf-tests/loadtests +--writeMethod=FILE_LOADS +--writeFormat=JSON +--testBigQueryDataset=beam_performance +--metricsBigQueryDataset=beam_performance +--metricsBigQueryTable=bqio_10GB_results_java_batch_json +--influxMeasurement=bqio_10GB_results_java_batch_json +--sourceOptions={\"numRecords\":\"10485760\",\"keySizeBytes\":\"1\",\"valueSizeBytes\":\"1024\"} +--runner=DataflowRunner +--maxNumWorkers=5 +--numWorkers=5 +--autoscalingAlgorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-pipeline-options/bigQueryIO_Streaming_Java.txt b/.github/workflows/performance-tests-pipeline-options/bigQueryIO_Streaming_Java.txt new file mode 100644 index 000000000000..99bd30e25994 --- /dev/null +++ b/.github/workflows/performance-tests-pipeline-options/bigQueryIO_Streaming_Java.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--tempRoot=gs://temp-storage-for-perf-tests/loadtests +--writeMethod=STREAMING_INSERTS +--writeFormat=JSON +--pipelineTimeout=1200 +--testBigQueryDataset=beam_performance +--metricsBigQueryDataset=beam_performance +--metricsBigQueryTable=bqio_10GB_results_java_stream +--influxMeasurement=bqio_10GB_results_java_stream +--sourceOptions={\"numRecords\":\"10485760\",\"keySizeBytes\":\"1\",\"valueSizeBytes\":\"1024\"} +--runner=DataflowRunner +--maxNumWorkers=5 +--numWorkers=5 +--autoscalingAlgorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/biqQueryIO_Read_Python.txt b/.github/workflows/performance-tests-pipeline-options/biqQueryIO_Read_Python.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/biqQueryIO_Read_Python.txt rename to .github/workflows/performance-tests-pipeline-options/biqQueryIO_Read_Python.txt diff --git a/.github/workflows/performance-tests-job-configs/biqQueryIO_Write_Python_Batch.txt b/.github/workflows/performance-tests-pipeline-options/biqQueryIO_Write_Python_Batch.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/biqQueryIO_Write_Python_Batch.txt rename to .github/workflows/performance-tests-pipeline-options/biqQueryIO_Write_Python_Batch.txt diff --git a/.github/workflows/performance-tests-job-configs/cdap.txt b/.github/workflows/performance-tests-pipeline-options/cdap.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/cdap.txt rename to .github/workflows/performance-tests-pipeline-options/cdap.txt diff --git a/.github/workflows/performance-tests-job-configs/config_Compressed_TextIOIT.txt b/.github/workflows/performance-tests-pipeline-options/compressed_TextIOIT.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/config_Compressed_TextIOIT.txt rename to .github/workflows/performance-tests-pipeline-options/compressed_TextIOIT.txt diff --git a/.github/workflows/performance-tests-job-configs/config_Compressed_TextIOIT_HDFS.txt b/.github/workflows/performance-tests-pipeline-options/compressed_TextIOIT_HDFS.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/config_Compressed_TextIOIT_HDFS.txt rename to .github/workflows/performance-tests-pipeline-options/compressed_TextIOIT_HDFS.txt diff --git a/.github/workflows/performance-tests-job-configs/hadoopFormat.txt b/.github/workflows/performance-tests-pipeline-options/hadoopFormat.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/hadoopFormat.txt rename to .github/workflows/performance-tests-pipeline-options/hadoopFormat.txt diff --git a/.github/workflows/performance-tests-job-configs/kafka_IO_Batch.txt b/.github/workflows/performance-tests-pipeline-options/kafka_IO_Batch.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/kafka_IO_Batch.txt rename to .github/workflows/performance-tests-pipeline-options/kafka_IO_Batch.txt diff --git a/.github/workflows/performance-tests-job-configs/kafka_IO_Streaming.txt b/.github/workflows/performance-tests-pipeline-options/kafka_IO_Streaming.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/kafka_IO_Streaming.txt rename to .github/workflows/performance-tests-pipeline-options/kafka_IO_Streaming.txt diff --git a/.github/workflows/performance-tests-job-configs/config_ManyFiles_TextIOIT.txt b/.github/workflows/performance-tests-pipeline-options/manyFiles_TextIOIT.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/config_ManyFiles_TextIOIT.txt rename to .github/workflows/performance-tests-pipeline-options/manyFiles_TextIOIT.txt diff --git a/.github/workflows/performance-tests-job-configs/config_ManyFiles_TextIOIT_HDFS.txt b/.github/workflows/performance-tests-pipeline-options/manyFiles_TextIOIT_HDFS.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/config_ManyFiles_TextIOIT_HDFS.txt rename to .github/workflows/performance-tests-pipeline-options/manyFiles_TextIOIT_HDFS.txt diff --git a/.github/workflows/performance-tests-job-configs/mongoDBIO_IT.txt b/.github/workflows/performance-tests-pipeline-options/mongoDBIO_IT.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/mongoDBIO_IT.txt rename to .github/workflows/performance-tests-pipeline-options/mongoDBIO_IT.txt diff --git a/.github/workflows/performance-tests-job-configs/config_ParquetIOIT.txt b/.github/workflows/performance-tests-pipeline-options/parquetIOIT.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/config_ParquetIOIT.txt rename to .github/workflows/performance-tests-pipeline-options/parquetIOIT.txt diff --git a/.github/workflows/performance-tests-job-configs/config_ParquetIOIT_HDFS.txt b/.github/workflows/performance-tests-pipeline-options/parquetIOIT_HDFS.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/config_ParquetIOIT_HDFS.txt rename to .github/workflows/performance-tests-pipeline-options/parquetIOIT_HDFS.txt diff --git a/.github/workflows/performance-tests-job-configs/pubsubIOIT_Python_Streaming.txt b/.github/workflows/performance-tests-pipeline-options/pubsubIOIT_Python_Streaming.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/pubsubIOIT_Python_Streaming.txt rename to .github/workflows/performance-tests-pipeline-options/pubsubIOIT_Python_Streaming.txt diff --git a/.github/workflows/performance-tests-job-configs/spannerIO_Read_2GB_Python.txt b/.github/workflows/performance-tests-pipeline-options/spannerIO_Read_2GB_Python.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/spannerIO_Read_2GB_Python.txt rename to .github/workflows/performance-tests-pipeline-options/spannerIO_Read_2GB_Python.txt diff --git a/.github/workflows/performance-tests-job-configs/spannerIO_Write_2GB_Python.txt b/.github/workflows/performance-tests-pipeline-options/spannerIO_Write_2GB_Python.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/spannerIO_Write_2GB_Python.txt rename to .github/workflows/performance-tests-pipeline-options/spannerIO_Write_2GB_Python.txt diff --git a/.github/workflows/performance-tests-job-configs/sparkReceiver_IO.txt b/.github/workflows/performance-tests-pipeline-options/sparkReceiver_IO.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/sparkReceiver_IO.txt rename to .github/workflows/performance-tests-pipeline-options/sparkReceiver_IO.txt diff --git a/.github/workflows/performance-tests-job-configs/textIOIT.txt b/.github/workflows/performance-tests-pipeline-options/textIOIT.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/textIOIT.txt rename to .github/workflows/performance-tests-pipeline-options/textIOIT.txt diff --git a/.github/workflows/performance-tests-job-configs/textIOIT_HDFS.txt b/.github/workflows/performance-tests-pipeline-options/textIOIT_HDFS.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/textIOIT_HDFS.txt rename to .github/workflows/performance-tests-pipeline-options/textIOIT_HDFS.txt diff --git a/.github/workflows/performance-tests-job-configs/textIOIT_Python.txt b/.github/workflows/performance-tests-pipeline-options/textIOIT_Python.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/textIOIT_Python.txt rename to .github/workflows/performance-tests-pipeline-options/textIOIT_Python.txt diff --git a/.github/workflows/performance-tests-job-configs/wordCountIT_Python.txt b/.github/workflows/performance-tests-pipeline-options/wordCountIT_Python.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/wordCountIT_Python.txt rename to .github/workflows/performance-tests-pipeline-options/wordCountIT_Python.txt diff --git a/.github/workflows/performance-tests-job-configs/xlang_KafkaIO_Python.txt b/.github/workflows/performance-tests-pipeline-options/xlang_KafkaIO_Python.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/xlang_KafkaIO_Python.txt rename to .github/workflows/performance-tests-pipeline-options/xlang_KafkaIO_Python.txt diff --git a/.github/workflows/performance-tests-job-configs/config_XmlIOIT.txt b/.github/workflows/performance-tests-pipeline-options/xmlIOIT.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/config_XmlIOIT.txt rename to .github/workflows/performance-tests-pipeline-options/xmlIOIT.txt diff --git a/.github/workflows/performance-tests-job-configs/config_XmlIOIT_HDFS.txt b/.github/workflows/performance-tests-pipeline-options/xmlIOIT_HDFS.txt similarity index 100% rename from .github/workflows/performance-tests-job-configs/config_XmlIOIT_HDFS.txt rename to .github/workflows/performance-tests-pipeline-options/xmlIOIT_HDFS.txt