From dbae730d86d49e83fb29682d0df37376c3735d05 Mon Sep 17 00:00:00 2001 From: XQ Hu Date: Mon, 25 Nov 2024 18:04:16 -0500 Subject: [PATCH] refactored the options --- .../beam_PreCommit_Flink_Container.yml | 36 ++++++------------- .../go_Combine_Flink_Batch_small.txt | 24 +++++++++++++ .../java_Combine_Flink_Batch_small.txt | 25 +++++++++++++ .../python_Combine_Flink_Batch_small.txt | 23 ++++++++++++ 4 files changed, 82 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/flink-tests-pipeline-options/go_Combine_Flink_Batch_small.txt create mode 100644 .github/workflows/flink-tests-pipeline-options/java_Combine_Flink_Batch_small.txt create mode 100644 .github/workflows/flink-tests-pipeline-options/python_Combine_Flink_Batch_small.txt diff --git a/.github/workflows/beam_PreCommit_Flink_Container.yml b/.github/workflows/beam_PreCommit_Flink_Container.yml index 4f825f52405e..f4418d3c944c 100644 --- a/.github/workflows/beam_PreCommit_Flink_Container.yml +++ b/.github/workflows/beam_PreCommit_Flink_Container.yml @@ -98,6 +98,13 @@ jobs: comment_phrase: ${{ matrix.job_phrase }} github_token: ${{ secrets.GITHUB_TOKEN }} github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/flink-tests-pipeline-options/go_Combine_Flink_Batch_small.txt + ${{ github.workspace }}/.github/workflows/flink-tests-pipeline-options/python_Combine_Flink_Batch_small.txt + ${{ github.workspace }}/.github/workflows/flink-tests-pipeline-options/java_Combine_Flink_Batch_small.txt - name: Start Flink with 2 workers env: FLINK_NUM_WORKERS: 2 @@ -112,15 +119,7 @@ jobs: arguments: | -PloadTest.mainClass=combine \ -Prunner=PortableRunner \ - '-PloadTest.args="--runner=FlinkRunner \ - --job_endpoint=localhost:8099 \ - --environment_type=DOCKER \ - --environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest \ - --parallelism=1 \ - --input_options={\"num_records\":200,\"key_size\":1,\"value_size\":9} \ - --fanout=1 \ - --top_count=10 \ - --iterations=1"' + '-PloadTest.args=${{ env.beam_PreCommit_Flink_Container_test_arguments_1 }} --job_name=flink-tests-go-${{env.NOW_UTC}}' # Run a Python Combine load test to verify the Flink container - name: Run Flink Container Test with Python Combine @@ -131,15 +130,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=FlinkRunner \ - '-PloadTest.args="--runner=PortableRunner \ - --job_endpoint=localhost:8099 \ - --environment_type=DOCKER \ - --environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest \ - --parallelism=1 \ - --input_options={\"num_records\":200,\"key_size\":1,\"value_size\":9,\"algorithm\":\"lcg\"} \ - --fanout=1 \ - --top_count=10 \ - --iterations=1"' + '-PloadTest.args=${{ env.beam_PreCommit_Flink_Container_test_arguments_2 }} --job_name=flink-tests-python-${{env.NOW_UTC}}' # Run a Java Combine load test to verify the Flink container - name: Run Flink Container Test with Java Combine @@ -150,14 +141,7 @@ jobs: arguments: | -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CombineLoadTest \ -Prunner=:runners:flink:1.17 \ - '-PloadTest.args="--runner=FlinkRunner \ - --environment_type=DOCKER \ - --environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_java11_sdk:latest \ - --parallelism=1 \ - --sourceOptions={\"numRecords\":200,\"keySizeBytes\":1,\"valueSizeBytes\":9} \ - --fanout=1 \ - --iterations=1 \ - --topCount=10"' + '-PloadTest.args=${{ env.beam_PreCommit_Flink_Container_test_arguments_3 }} --job_name=flink-tests-java11-${{env.NOW_UTC}}' - name: Teardown Flink if: always() diff --git a/.github/workflows/flink-tests-pipeline-options/go_Combine_Flink_Batch_small.txt b/.github/workflows/flink-tests-pipeline-options/go_Combine_Flink_Batch_small.txt new file mode 100644 index 000000000000..6b44f53886b2 --- /dev/null +++ b/.github/workflows/flink-tests-pipeline-options/go_Combine_Flink_Batch_small.txt @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--input_options=''{\"num_records\":200,\"key_size\":1,\"value_size\":9}'' +--fanout=1 +--top_count=10 +--parallelism=2 +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=FlinkRunner \ No newline at end of file diff --git a/.github/workflows/flink-tests-pipeline-options/java_Combine_Flink_Batch_small.txt b/.github/workflows/flink-tests-pipeline-options/java_Combine_Flink_Batch_small.txt new file mode 100644 index 000000000000..e4b04938569c --- /dev/null +++ b/.github/workflows/flink-tests-pipeline-options/java_Combine_Flink_Batch_small.txt @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--sourceOptions={"numRecords":200,"keySizeBytes":1,"valueSizeBytes":9} +--fanout=1 +--iterations=1 +--topCount=10 +--parallelism=2 +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_java11_sdk:latest +--runner=FlinkRunner \ No newline at end of file diff --git a/.github/workflows/flink-tests-pipeline-options/python_Combine_Flink_Batch_small.txt b/.github/workflows/flink-tests-pipeline-options/python_Combine_Flink_Batch_small.txt new file mode 100644 index 000000000000..5522a8f9b823 --- /dev/null +++ b/.github/workflows/flink-tests-pipeline-options/python_Combine_Flink_Batch_small.txt @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--input_options=''{\\"num_records\\":200,\\"key_size\\":1,\\"value_size\\":9,\\"algorithm\\":\\"lcg\\"}'' +--parallelism=2 +--job_endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest +--top_count=10 +--runner=PortableRunner \ No newline at end of file