refactored the options

apache · Nov 25, 2024 · dbae730 · dbae730
1 parent 5234ac9
commit dbae730
Show file tree

Hide file tree

Showing 4 changed files with 82 additions and 26 deletions.
diff --git a/.github/workflows/beam_PreCommit_Flink_Container.yml b/.github/workflows/beam_PreCommit_Flink_Container.yml
@@ -98,6 +98,13 @@ jobs:
           comment_phrase: ${{ matrix.job_phrase }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
           github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+      - name: Prepare test arguments
+        uses: ./.github/actions/test-arguments-action
+        with:
+          argument-file-paths: |
+            ${{ github.workspace }}/.github/workflows/flink-tests-pipeline-options/go_Combine_Flink_Batch_small.txt
+            ${{ github.workspace }}/.github/workflows/flink-tests-pipeline-options/python_Combine_Flink_Batch_small.txt
+            ${{ github.workspace }}/.github/workflows/flink-tests-pipeline-options/java_Combine_Flink_Batch_small.txt
       - name: Start Flink with 2 workers
         env:
           FLINK_NUM_WORKERS: 2
@@ -112,15 +119,7 @@ jobs:
           arguments: |
             -PloadTest.mainClass=combine \
             -Prunner=PortableRunner \
-            '-PloadTest.args="--runner=FlinkRunner \
-              --job_endpoint=localhost:8099 \
-              --environment_type=DOCKER \
-              --environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest \
-              --parallelism=1 \
-              --input_options={\"num_records\":200,\"key_size\":1,\"value_size\":9} \
-              --fanout=1 \
-              --top_count=10 \
-              --iterations=1"'
+            '-PloadTest.args=${{ env.beam_PreCommit_Flink_Container_test_arguments_1 }} --job_name=flink-tests-go-${{env.NOW_UTC}}'
 
       # Run a Python Combine load test to verify the Flink container
       - name: Run Flink Container Test with Python Combine
@@ -131,15 +130,7 @@ jobs:
           arguments: |
             -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
             -Prunner=FlinkRunner \
-            '-PloadTest.args="--runner=PortableRunner \
-              --job_endpoint=localhost:8099 \
-              --environment_type=DOCKER \
-              --environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest \
-              --parallelism=1 \
-              --input_options={\"num_records\":200,\"key_size\":1,\"value_size\":9,\"algorithm\":\"lcg\"} \
-              --fanout=1 \
-              --top_count=10 \
-              --iterations=1"'
+            '-PloadTest.args=${{ env.beam_PreCommit_Flink_Container_test_arguments_2 }} --job_name=flink-tests-python-${{env.NOW_UTC}}'
 
       # Run a Java Combine load test to verify the Flink container
       - name: Run Flink Container Test with Java Combine
@@ -150,14 +141,7 @@ jobs:
           arguments: |
             -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CombineLoadTest \
             -Prunner=:runners:flink:1.17 \
-            '-PloadTest.args="--runner=FlinkRunner \
-              --environment_type=DOCKER \
-              --environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_java11_sdk:latest \
-              --parallelism=1 \
-              --sourceOptions={\"numRecords\":200,\"keySizeBytes\":1,\"valueSizeBytes\":9} \
-              --fanout=1 \
-              --iterations=1 \
-              --topCount=10"'
+            '-PloadTest.args=${{ env.beam_PreCommit_Flink_Container_test_arguments_3 }} --job_name=flink-tests-java11-${{env.NOW_UTC}}'
 
       - name: Teardown Flink
         if: always()

diff --git a/.github/workflows/flink-tests-pipeline-options/go_Combine_Flink_Batch_small.txt b/.github/workflows/flink-tests-pipeline-options/go_Combine_Flink_Batch_small.txt
@@ -0,0 +1,24 @@
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+--input_options=''{\"num_records\":200,\"key_size\":1,\"value_size\":9}''
+--fanout=1
+--top_count=10
+--parallelism=2
+--endpoint=localhost:8099
+--environment_type=DOCKER
+--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
+--runner=FlinkRunner
diff --git a/.github/workflows/flink-tests-pipeline-options/java_Combine_Flink_Batch_small.txt b/.github/workflows/flink-tests-pipeline-options/java_Combine_Flink_Batch_small.txt
@@ -0,0 +1,25 @@
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+
+--sourceOptions={"numRecords":200,"keySizeBytes":1,"valueSizeBytes":9}
+--fanout=1
+--iterations=1
+--topCount=10
+--parallelism=2
+--endpoint=localhost:8099
+--environment_type=DOCKER
+--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_java11_sdk:latest
+--runner=FlinkRunner
diff --git a/.github/workflows/flink-tests-pipeline-options/python_Combine_Flink_Batch_small.txt b/.github/workflows/flink-tests-pipeline-options/python_Combine_Flink_Batch_small.txt
@@ -0,0 +1,23 @@
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+--input_options=''{\\"num_records\\":200,\\"key_size\\":1,\\"value_size\\":9,\\"algorithm\\":\\"lcg\\"}''
+--parallelism=2
+--job_endpoint=localhost:8099
+--environment_type=DOCKER
+--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest
+--top_count=10
+--runner=PortableRunner