diff --git a/.github/actions/setup-default-test-properties/test-properties.json b/.github/actions/setup-default-test-properties/test-properties.json index 89ef25b21160..a11af748efea 100644 --- a/.github/actions/setup-default-test-properties/test-properties.json +++ b/.github/actions/setup-default-test-properties/test-properties.json @@ -1,15 +1,15 @@ { "PythonTestProperties": { - "ALL_SUPPORTED_VERSIONS": ["3.8", "3.9", "3.10", "3.11"], - "LOWEST_SUPPORTED": ["3.8"], - "HIGHEST_SUPPORTED": ["3.11"], - "ESSENTIAL_VERSIONS": ["3.8", "3.11"], - "CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS": ["3.8", "3.11"], + "ALL_SUPPORTED_VERSIONS": ["3.9", "3.10", "3.11", "3.12"], + "LOWEST_SUPPORTED": ["3.9"], + "HIGHEST_SUPPORTED": ["3.12"], + "ESSENTIAL_VERSIONS": ["3.9", "3.12"], + "CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS": ["3.9", "3.12"], "CROSS_LANGUAGE_VALIDATES_RUNNER_DATAFLOW_USING_SQL_PYTHON_VERSIONS": ["3.11"], - "VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS": ["3.8", "3.9", "3.10", "3.11" ], - "LOAD_TEST_PYTHON_VERSION": "3.8", - "CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION": "3.8", - "DEFAULT_INTERPRETER": "python3.8", + "VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS": ["3.9", "3.10", "3.11", "3.12"], + "LOAD_TEST_PYTHON_VERSION": "3.9", + "CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION": "3.9", + "DEFAULT_INTERPRETER": "python3.9", "TOX_ENV": ["Cloud", "Cython"] }, "JavaTestProperties": { diff --git a/.github/actions/setup-environment-action/action.yml b/.github/actions/setup-environment-action/action.yml index d3d26c1ad3c0..2da2cf0becd5 100644 --- a/.github/actions/setup-environment-action/action.yml +++ b/.github/actions/setup-environment-action/action.yml @@ -42,7 +42,7 @@ runs: if: ${{ inputs.python-version != '' }} uses: actions/setup-python@v4 with: - python-version: ${{ inputs.python-version == 'default' && '3.8' || inputs.python-version }} + python-version: ${{ inputs.python-version == 'default' && '3.9' || inputs.python-version }} - name: Install Java if: ${{ inputs.java-version != '' }} uses: actions/setup-java@v3 diff --git a/.github/workflows/beam_CloudML_Benchmarks_Dataflow.yml b/.github/workflows/beam_CloudML_Benchmarks_Dataflow.yml index f3db177001ec..1448783a24c4 100644 --- a/.github/workflows/beam_CloudML_Benchmarks_Dataflow.yml +++ b/.github/workflows/beam_CloudML_Benchmarks_Dataflow.yml @@ -73,8 +73,8 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: | - 3.8 3.9 + 3.10 - name: Prepare test arguments uses: ./.github/actions/test-arguments-action with: diff --git a/.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml b/.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml index c96e84e4d7bf..2145a23a80fd 100644 --- a/.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml +++ b/.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml @@ -95,7 +95,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_1 }} --job_name=benchmark-tests-pytorch-imagenet-python-101-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_resnet101-${{env.NOW_UTC}}.txt' \ - name: run Pytorch Imagenet Classification with Resnet 152 @@ -106,7 +106,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_2 }} --job_name=benchmark-tests-pytorch-imagenet-python-152-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_resnet152-${{env.NOW_UTC}}.txt' \ - name: run Pytorch Language Modeling using Hugging face bert-base-uncased model @@ -117,7 +117,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_language_modeling_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_3 }} --job_name=benchmark-tests-pytorch-language-modeling-bert-base-uncased-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_bert_base_uncased-${{env.NOW_UTC}}.txt' \ - name: run Pytorch Langauge Modeling using Hugging Face bert-large-uncased model @@ -128,7 +128,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_language_modeling_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_4 }} --job_name=benchmark-tests-pytorch-language-modeling-bert-large-uncased-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_bert_large_uncased-${{env.NOW_UTC}}.txt' \ - name: run Pytorch Imagenet Classification with Resnet 152 with Tesla T4 GPU @@ -139,6 +139,6 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_5 }} --job_name=benchmark-tests-pytorch-imagenet-python-gpu-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_resnet152_gpu-${{env.NOW_UTC}}.txt' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml index 87854bdb8f84..2bc4e50d4792 100644 --- a/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml @@ -96,7 +96,7 @@ jobs: --info \ -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-cogbk-1-${{ steps.datetime.outputs.datetime }}' \ - name: run CoGBK 2GB of 100B records with multiple keys uses: ./.github/actions/gradle-command-self-hosted-action @@ -105,7 +105,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-cogbk-2-${{ steps.datetime.outputs.datetime }}' \ - name: run CoGBK reiterate 4 times 10kB values uses: ./.github/actions/gradle-command-self-hosted-action @@ -114,7 +114,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-cogbk-3-${{ steps.datetime.outputs.datetime }}' \ - name: run CoGBK reiterate 4 times 2MB values uses: ./.github/actions/gradle-command-self-hosted-action @@ -123,5 +123,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_4 }} --job_name=load-tests-python-dataflow-batch-cogbk-4-${{ steps.datetime.outputs.datetime }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml index f9e62e9965a8..b8607fb71603 100644 --- a/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml @@ -95,7 +95,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-cogbk-1-${{ steps.datetime.outputs.datetime }}' \ - name: run CoGBK 2GB of 100B records with multiple keys uses: ./.github/actions/gradle-command-self-hosted-action @@ -104,7 +104,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_2 }} --job_name=load-tests-python-dataflow-streaming-cogbk-2-${{ steps.datetime.outputs.datetime }}' \ - name: run CoGBK reiterate 4 times 10kB values uses: ./.github/actions/gradle-command-self-hosted-action @@ -113,7 +113,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_3 }} --job_name=load-tests-python-dataflow-streaming-cogbk-3-${{ steps.datetime.outputs.datetime }}' \ - name: run CoGBK reiterate 4 times 2MB values uses: ./.github/actions/gradle-command-self-hosted-action @@ -122,5 +122,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_4 }} --job_name=load-tests-python-dataflow-streaming-cogbk-4-${{ steps.datetime.outputs.datetime }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml index 76e2a417c25c..866a5871962d 100644 --- a/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml @@ -92,7 +92,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-combine-1-${{env.NOW_UTC}}' \ - name: run Combine Dataflow Batch Python Load Test 2 (fanout 4) uses: ./.github/actions/gradle-command-self-hosted-action @@ -101,7 +101,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-combine-2-${{env.NOW_UTC}}' \ - name: run Combine Dataflow Batch Python Load Test 3 (fanout 8) uses: ./.github/actions/gradle-command-self-hosted-action @@ -110,5 +110,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-combine-3-${{env.NOW_UTC}}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml index 306f71662eb4..e77db13ecb7f 100644 --- a/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml @@ -92,7 +92,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-combine-1-${{env.NOW_UTC}}' \ - name: run 2GB Fanout 4 test uses: ./.github/actions/gradle-command-self-hosted-action @@ -101,7 +101,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Streaming_test_arguments_2 }} --job_name=load-tests-python-dataflow-streaming-combine-4-${{env.NOW_UTC}}' \ - name: run 2GB Fanout 8 test uses: ./.github/actions/gradle-command-self-hosted-action @@ -110,5 +110,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Streaming_test_arguments_3 }} --job_name=load-tests-python-dataflow-streaming-combine-5-${{env.NOW_UTC}}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml index 230f6398e522..0f666a0b7db6 100644 --- a/.github/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml @@ -107,7 +107,7 @@ jobs: with: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=PortableRunner \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Batch_test_arguments_1 }} --job_name=load-tests-python-flink-batch-combine-1-${{env.NOW_UTC}}' \ @@ -121,7 +121,7 @@ jobs: with: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=PortableRunner \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Batch_test_arguments_2 }} --job_name=load-tests-python-flink-batch-combine-4-${{env.NOW_UTC}}' \ @@ -130,7 +130,7 @@ jobs: with: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=PortableRunner \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Batch_test_arguments_3 }} --job_name=load-tests-python-flink-batch-combine-5-${{env.NOW_UTC}}' \ diff --git a/.github/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml b/.github/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml index 0ea12f49a5f1..6f491e6b9fa9 100644 --- a/.github/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml @@ -104,7 +104,7 @@ jobs: with: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=PortableRunner \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Streaming_test_arguments_1 }} --job_name=load-tests-python-flink-streaming-combine-4-${{env.NOW_UTC}}' \ @@ -113,7 +113,7 @@ jobs: with: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=PortableRunner \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Streaming_test_arguments_2 }} --job_name=load-tests-python-flink-streaming-combine-5-${{env.NOW_UTC}}' \ diff --git a/.github/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml b/.github/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml index da6b0beee92c..d69efb4636bb 100644 --- a/.github/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml +++ b/.github/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml @@ -87,5 +87,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.microbenchmarks_test \ -Prunner=DirectRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_FnApiRunner_Microbenchmark_test_arguments_1 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml index 6e1e6a09f200..d2924a081255 100644 --- a/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml @@ -94,7 +94,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-gbk-1-${{env.NOW_UTC}}' \ - name: run 2GB of 100B records test uses: ./.github/actions/gradle-command-self-hosted-action @@ -103,7 +103,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-gbk-2-${{env.NOW_UTC}}' \ - name: run 2GB of 100kB records test uses: ./.github/actions/gradle-command-self-hosted-action @@ -112,7 +112,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-gbk-3-${{env.NOW_UTC}}' \ - name: run fanout 4 times with 2GB 10-byte records test uses: ./.github/actions/gradle-command-self-hosted-action @@ -121,7 +121,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_4 }} --job_name=load-tests-python-dataflow-batch-gbk-4-${{env.NOW_UTC}}' \ - name: run fanout 8 times with 2GB 10-byte records total test uses: ./.github/actions/gradle-command-self-hosted-action @@ -130,5 +130,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_5 }} --job_name=load-tests-python-dataflow-batch-gbk-5-${{env.NOW_UTC}}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml index d8d3e35f17a8..70321f2414a0 100644 --- a/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml @@ -90,7 +90,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-gbk-3-${{env.NOW_UTC}}' \ # // TODO(https://github.com/apache/beam/issues/20403). Skipping some cases because they are too slow: diff --git a/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml index 7f8e2f197359..f99d2a3f7387 100644 --- a/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml @@ -91,7 +91,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-gbk-6-${{env.NOW_UTC}}' \ - name: run reiterate 4 times 2MB values test uses: ./.github/actions/gradle-command-self-hosted-action @@ -100,5 +100,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-gbk-7-${{env.NOW_UTC}}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml index 5b07d15337fa..d7e31f1edcac 100644 --- a/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml @@ -91,7 +91,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-gbk-6-${{env.NOW_UTC}}' \ - name: run reiterate 4 times 2MB values test uses: ./.github/actions/gradle-command-self-hosted-action @@ -100,5 +100,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming_test_arguments_2 }} --job_name=load-tests-python-dataflow-streaming-gbk-7-${{env.NOW_UTC}}' \ diff --git a/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml index 8dc8f031b7e6..b4f505648702 100644 --- a/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml @@ -95,7 +95,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-pardo-1-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Dataflow Batch Python Load Test 2 (200 iterations) uses: ./.github/actions/gradle-command-self-hosted-action @@ -104,7 +104,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-pardo-2-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Dataflow Batch Python Load Test 3 (10 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -113,7 +113,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-pardo-3-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Dataflow Batch Python Load Test 4 (100 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -122,5 +122,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Batch_test_arguments_4 }} --job_name=load-tests-python-dataflow-batch-pardo-4-${{ steps.datetime.outputs.datetime }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml index f0d0778d98ca..fabd893afaaf 100644 --- a/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml @@ -95,7 +95,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-pardo-1-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Dataflow Streaming Python Load Test 2 (200 iterations) uses: ./.github/actions/gradle-command-self-hosted-action @@ -104,7 +104,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Streaming_test_arguments_2 }} --job_name=load-tests-python-dataflow-streaming-pardo-2-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Dataflow Streaming Python Load Test 3 (10 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -113,7 +113,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Streaming_test_arguments_3 }} --job_name=load-tests-python-dataflow-streaming-pardo-3-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Dataflow Streaming Python Load Test 4 (100 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -122,5 +122,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Streaming_test_arguments_4 }} --job_name=load-tests-python-dataflow-streaming-pardo-4-${{ steps.datetime.outputs.datetime }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml index bae2f9f82ee1..b6c86e01c299 100644 --- a/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml @@ -109,7 +109,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Batch_test_arguments_1 }} --job_name=load-tests-python-flink-batch-pardo-1-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Flink Batch Python Load Test 2 (200 iterations) uses: ./.github/actions/gradle-command-self-hosted-action @@ -118,7 +118,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Batch_test_arguments_2 }} --job_name=load-tests-python-flink-batch-pardo-3-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Flink Batch Python Load Test 3 (10 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -127,5 +127,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Batch_test_arguments_3 }} --job_name=load-tests-python-flink-batch-pardo-4-${{ steps.datetime.outputs.datetime }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml b/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml index 4485b7187f80..a6443c0df10b 100644 --- a/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml @@ -111,7 +111,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Streaming_test_arguments_1 }} --job_name=load-tests-python-flink-streaming-pardo-1-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Flink Streaming Python Load Test 2 (200 iterations) uses: ./.github/actions/gradle-command-self-hosted-action @@ -120,7 +120,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Streaming_test_arguments_2 }} --job_name=load-tests-python-flink-streaming-pardo-2-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Flink Streaming Python Load Test 3 (10 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -129,7 +129,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Streaming_test_arguments_3 }} --job_name=load-tests-python-flink-streaming-pardo-3-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Flink Streaming Python Load Test 4 (100 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -138,7 +138,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Streaming_test_arguments_4 }} --job_name=load-tests-python-flink-streaming-pardo-4-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Flink Streaming Python Load Test 5 (5 iterations) uses: ./.github/actions/gradle-command-self-hosted-action @@ -147,7 +147,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Streaming_test_arguments_5 }} --job_name=load-tests-python-flink-streaming-pardo-6-${{ steps.datetime.outputs.datetime }}' \ - name: Teardown Flink if: always() diff --git a/.github/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml index b93474b2a21d..7917af0ff5d2 100644 --- a/.github/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml @@ -101,7 +101,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-sideinput-1-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 2 (1gb-1kb-10workers-1window-99key-percent-dict) uses: ./.github/actions/gradle-command-self-hosted-action @@ -110,7 +110,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-sideinput-2-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 3 (10gb-1kb-10workers-1window-first-iterable) uses: ./.github/actions/gradle-command-self-hosted-action @@ -119,7 +119,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-sideinput-3-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 4 (10gb-1kb-10workers-1window-iterable) uses: ./.github/actions/gradle-command-self-hosted-action @@ -128,7 +128,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_4 }} --job_name=load-tests-python-dataflow-batch-sideinput-4-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 5 (1gb-1kb-10workers-1window-first-list) uses: ./.github/actions/gradle-command-self-hosted-action @@ -137,7 +137,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_5 }} --job_name=load-tests-python-dataflow-batch-sideinput-5-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 6 (1gb-1kb-10workers-1window-list) uses: ./.github/actions/gradle-command-self-hosted-action @@ -146,7 +146,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_6 }} --job_name=load-tests-python-dataflow-batch-sideinput-6-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 7 (1gb-1kb-10workers-1000window-1key-percent-dict) uses: ./.github/actions/gradle-command-self-hosted-action @@ -155,7 +155,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_7 }} --job_name=load-tests-python-dataflow-batch-sideinput-7-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 8 (1gb-1kb-10workers-1000window-99key-percent-dict) uses: ./.github/actions/gradle-command-self-hosted-action @@ -164,7 +164,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_8 }} --job_name=load-tests-python-dataflow-batch-sideinput-8-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 9 (10gb-1kb-10workers-1000window-first-iterable) uses: ./.github/actions/gradle-command-self-hosted-action @@ -173,7 +173,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_9 }} --job_name=load-tests-python-dataflow-batch-sideinput-9-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 10 (10gb-1kb-10workers-1000window-iterable) uses: ./.github/actions/gradle-command-self-hosted-action @@ -182,5 +182,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_10 }} --job_name=load-tests-python-dataflow-batch-sideinput-10-${{ steps.datetime.outputs.datetime }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_Smoke.yml b/.github/workflows/beam_LoadTests_Python_Smoke.yml index 39e8e4b56102..22dc63f19faa 100644 --- a/.github/workflows/beam_LoadTests_Python_Smoke.yml +++ b/.github/workflows/beam_LoadTests_Python_Smoke.yml @@ -90,7 +90,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DirectRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Smoke_test_arguments_1 }} --job_name=load-tests-python-direct-batch-gbk-smoke-${{ steps.datetime.outputs.datetime }}' \ - name: run GroupByKey Python load test Dataflow uses: ./.github/actions/gradle-command-self-hosted-action @@ -99,5 +99,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Smoke_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-gbk-smoke-${{ steps.datetime.outputs.datetime }}' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml index 6f8e5fc9bc8d..3673ca170555 100644 --- a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml +++ b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml @@ -89,6 +89,6 @@ jobs: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | -PloadTest.mainClass=apache_beam.io.gcp.bigquery_read_perf_test \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ -Prunner=DataflowRunner \ '-PloadTest.args=${{env.beam_PerformanceTests_BiqQueryIO_Read_Python_test_arguments_1}}' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml index 5d7b451071f1..f52eec3fa2c8 100644 --- a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml +++ b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml @@ -89,6 +89,6 @@ jobs: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | -PloadTest.mainClass=apache_beam.io.gcp.bigquery_write_perf_test \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ -Prunner=DataflowRunner \ '-PloadTest.args=${{env.beam_PerformanceTests_BiqQueryIO_Write_Python_Batch_test_arguments_1}}' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml b/.github/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml index 22a71967d3a4..e358f113a668 100644 --- a/.github/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml +++ b/.github/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml @@ -90,5 +90,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.io.gcp.pubsub_io_perf_test \ -Prunner=TestDataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_PerformanceTests_PubsubIOIT_Python_Streaming_test_arguments_1 }}' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml b/.github/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml index 2de75b2496a8..3448bbcf96c5 100644 --- a/.github/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml +++ b/.github/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml @@ -90,5 +90,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.io.gcp.experimental.spannerio_read_perf_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ -PloadTest.args='${{env.beam_PerformanceTests_SpannerIO_Read_2GB_Python_test_arguments_1}}' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml b/.github/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml index 00292aee45ac..ba9d05e46838 100644 --- a/.github/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml +++ b/.github/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml @@ -90,5 +90,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.io.gcp.experimental.spannerio_write_perf_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ -PloadTest.args='${{env.beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch_test_arguments_1}}' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_TextIOIT_Python.yml b/.github/workflows/beam_PerformanceTests_TextIOIT_Python.yml index f9ca3d949ced..39bd0ab467d2 100644 --- a/.github/workflows/beam_PerformanceTests_TextIOIT_Python.yml +++ b/.github/workflows/beam_PerformanceTests_TextIOIT_Python.yml @@ -88,7 +88,7 @@ jobs: with: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ -PloadTest.mainClass=apache_beam.io.filebasedio_perf_test \ -Prunner=DataflowRunner \ '-PloadTest.args=${{env.beam_PerformanceTests_TextIOIT_Python_test_arguments_1}}' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml b/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml index 21dfa13b25dc..e9ef9cd1716a 100644 --- a/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml +++ b/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml @@ -64,7 +64,7 @@ jobs: job_name: ["beam_PerformanceTests_WordCountIT_PythonVersions"] job_phrase_1: [Run Python] job_phrase_2: [WordCountIT Performance Test] - python_version: ['3.8'] + python_version: ['3.9'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml b/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml index 8abc8a3199dd..00c62edc34ad 100644 --- a/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml +++ b/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml @@ -118,5 +118,5 @@ jobs: arguments: | -Prunner=DataflowRunner \ -PloadTest.mainClass=apache_beam.io.external.xlang_kafkaio_perf_test \ - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ '-PloadTest.args=${{ env.beam_PerformanceTests_xlang_KafkaIO_Python_test_arguments_1 }}' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_PortableJar_Flink.yml b/.github/workflows/beam_PostCommit_PortableJar_Flink.yml index 5347dc45642b..37bfe68d9b20 100644 --- a/.github/workflows/beam_PostCommit_PortableJar_Flink.yml +++ b/.github/workflows/beam_PostCommit_PortableJar_Flink.yml @@ -79,9 +79,9 @@ jobs: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:test-suites:portable:py38:testPipelineJarFlinkRunner + gradle-command: :sdks:python:test-suites:portable:py39:testPipelineJarFlinkRunner arguments: | - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ - name: Archive Python Test Results uses: actions/upload-artifact@v4 if: failure() diff --git a/.github/workflows/beam_PostCommit_PortableJar_Spark.yml b/.github/workflows/beam_PostCommit_PortableJar_Spark.yml index 3778f017d1cc..ce7be60133d7 100644 --- a/.github/workflows/beam_PostCommit_PortableJar_Spark.yml +++ b/.github/workflows/beam_PostCommit_PortableJar_Spark.yml @@ -79,9 +79,9 @@ jobs: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:test-suites:portable:py38:testPipelineJarSparkRunner + gradle-command: :sdks:python:test-suites:portable:py39:testPipelineJarSparkRunner arguments: | - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ - name: Archive Python Test Results uses: actions/upload-artifact@v4 if: failure() diff --git a/.github/workflows/beam_PostCommit_Python.yml b/.github/workflows/beam_PostCommit_Python.yml index 6705268143e9..4770515c75fb 100644 --- a/.github/workflows/beam_PostCommit_Python.yml +++ b/.github/workflows/beam_PostCommit_Python.yml @@ -60,7 +60,7 @@ jobs: matrix: job_name: [beam_PostCommit_Python] job_phrase: [Run Python PostCommit] - python_version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python_version: ['3.9', '3.10', '3.11', '3.12'] if: | github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PostCommit_Python_Arm.yml b/.github/workflows/beam_PostCommit_Python_Arm.yml index f9438c3c644d..48fb00b1bb9d 100644 --- a/.github/workflows/beam_PostCommit_Python_Arm.yml +++ b/.github/workflows/beam_PostCommit_Python_Arm.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: [beam_PostCommit_Python_Arm] job_phrase: [Run Python PostCommit Arm] - python_version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python_version: ['3.9', '3.10', '3.11', '3.12'] if: | github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PostCommit_Python_Dependency.yml b/.github/workflows/beam_PostCommit_Python_Dependency.yml index 04c22e4ab07d..6e7c4ddbd3eb 100644 --- a/.github/workflows/beam_PostCommit_Python_Dependency.yml +++ b/.github/workflows/beam_PostCommit_Python_Dependency.yml @@ -81,7 +81,7 @@ jobs: - name: Run postCommitPyDep uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:test-suites:tox:py38:postCommitPyDep + gradle-command: :sdks:python:test-suites:tox:py39:postCommitPyDep arguments: -PuseWheelDistribution - name: Archive Python Test Results uses: actions/upload-artifact@v4 diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml b/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml index 7d3bb65a20e1..a6bb49f4e444 100644 --- a/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml +++ b/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_Examples_Direct"] job_phrase: ["Run Python Examples_Direct"] - python_version: ['3.8','3.9','3.10','3.11','3.12'] + python_version: ['3.9','3.10','3.11','3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml b/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml index f88d7e205cc6..bda807eb147b 100644 --- a/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml +++ b/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_Examples_Flink"] job_phrase: ["Run Python Examples_Flink"] - python_version: ['3.8', '3.12'] + python_version: ['3.9', '3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml b/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml index 4300179421b5..d866d412507b 100644 --- a/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml +++ b/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_Examples_Spark"] job_phrase: ["Run Python Examples_Spark"] - python_version: ['3.8', '3.12'] + python_version: ['3.9', '3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_Nexmark_Direct.yml b/.github/workflows/beam_PostCommit_Python_Nexmark_Direct.yml index 601cb99a44fb..3d47fb86889d 100644 --- a/.github/workflows/beam_PostCommit_Python_Nexmark_Direct.yml +++ b/.github/workflows/beam_PostCommit_Python_Nexmark_Direct.yml @@ -133,7 +133,7 @@ jobs: with: gradle-command: :sdks:python:apache_beam:testing:benchmarks:nexmark:run arguments: | - -PpythonVersion=3.8 \ + -PpythonVersion=3.9 \ "-Pnexmark.args=${{ env.GRADLE_PYTHON_COMMAND_ARGUMENTS }} \ --query=${{ matrix.query }} \ --input=gs://temp-storage-for-perf-tests/nexmark/eventFiles/beam_PostCommit_Python_Nexmark_Direct/query${{ matrix.query }}-\*" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml index ec7a28d2db2c..bcd936324124 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml @@ -65,7 +65,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesContainer_Dataflow"] job_phrase: ["Run Python Dataflow ValidatesContainer"] - python_version: ['3.8','3.9','3.10','3.11','3.12'] + python_version: ['3.9','3.10','3.11','3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml index b90c150291dd..f2eba045722c 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC"] job_phrase: ["Run Python RC Dataflow ValidatesContainer"] - python_version: ['3.8','3.9','3.10','3.11','3.12'] + python_version: ['3.9','3.10','3.11','3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml index 8df5d00287bc..1876950c7a93 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesRunner_Dataflow"] job_phrase: ["Run Python Dataflow ValidatesRunner"] - python_version: ['3.8', '3.12'] + python_version: ['3.9', '3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml index b301402f4de2..f837c7476e12 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesRunner_Flink"] job_phrase: ["Run Python Flink ValidatesRunner"] - python_version: ['3.8', '3.12'] + python_version: ['3.9', '3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml index 6c89b110ec7a..91c249adf338 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesRunner_Samza"] job_phrase: ["Run Python Samza ValidatesRunner"] - python_version: ['3.8', '3.12'] + python_version: ['3.9', '3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml index 66d1ac6756c4..7e87aaff22cc 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesRunner_Spark"] job_phrase: ["Run Python Spark ValidatesRunner"] - python_version: ['3.8', '3.12'] + python_version: ['3.9', '3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml index eb204ce50349..b3f37c6b39f0 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml @@ -74,7 +74,7 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: | - 3.8 + 3.9 3.12 - name: run PostCommit Python Xlang Gcp Dataflow script uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml index 63417231cb96..137d7bc13d2f 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml @@ -74,7 +74,7 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: | - 3.8 + 3.9 3.12 - name: run PostCommit Python Xlang Gcp Direct script uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml index ac90c2cd66b8..8fc0db189078 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml @@ -74,7 +74,7 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: | - 3.8 + 3.9 3.12 - name: run PostCommit Python Xlang IO Dataflow script uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_PostCommit_Sickbay_Python.yml b/.github/workflows/beam_PostCommit_Sickbay_Python.yml index 91211df97f7f..6d253e03723d 100644 --- a/.github/workflows/beam_PostCommit_Sickbay_Python.yml +++ b/.github/workflows/beam_PostCommit_Sickbay_Python.yml @@ -59,7 +59,7 @@ jobs: job_name: [beam_PostCommit_Sickbay_Python] job_phrase_1: [Run Python] job_phrase_2: [PostCommit Sickbay] - python_version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python_version: ['3.9', '3.10', '3.11', '3.12'] if: | github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PostCommit_TransformService_Direct.yml b/.github/workflows/beam_PostCommit_TransformService_Direct.yml index 966938e43e92..cb339eb9fb40 100644 --- a/.github/workflows/beam_PostCommit_TransformService_Direct.yml +++ b/.github/workflows/beam_PostCommit_TransformService_Direct.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_TransformService_Direct"] job_phrase: ["Run TransformService_Direct PostCommit"] - python_version: ['3.8','3.12'] + python_version: ['3.9','3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -76,7 +76,7 @@ jobs: with: java-version: 11 python-version: | - 3.8 + 3.9 ${{ matrix.python_version }} - name: run TransformService Direct script uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_PostCommit_XVR_Direct.yml b/.github/workflows/beam_PostCommit_XVR_Direct.yml index ec66cab88e9b..023ae4f8cd31 100644 --- a/.github/workflows/beam_PostCommit_XVR_Direct.yml +++ b/.github/workflows/beam_PostCommit_XVR_Direct.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_Direct"] job_phrase: ["Run XVR_Direct PostCommit"] - python_version: ['3.8','3.12'] + python_version: ['3.9','3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -75,12 +75,12 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: | - 3.8 + 3.9 ${{ matrix.python_version }} - name: run PostCommit XVR Direct script env: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} - if: ${{ matrix.python_version != '3.8' }} + if: ${{ matrix.python_version != '3.9' }} uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:python:test-suites:direct:xlang:validatesCrossLanguageRunner @@ -90,7 +90,7 @@ jobs: - name: run PostCommit XVR Direct script env: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} - if: ${{ matrix.python_version == '3.8' }} + if: ${{ matrix.python_version == '3.9' }} uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:python:test-suites:direct:xlang:validatesCrossLanguageRunner diff --git a/.github/workflows/beam_PostCommit_XVR_Flink.yml b/.github/workflows/beam_PostCommit_XVR_Flink.yml index d88b502988ef..5cde38d24244 100644 --- a/.github/workflows/beam_PostCommit_XVR_Flink.yml +++ b/.github/workflows/beam_PostCommit_XVR_Flink.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_Flink"] job_phrase: ["Run XVR_Flink PostCommit"] - python_version: ['3.8','3.12'] + python_version: ['3.9','3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -76,12 +76,12 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: | - 3.8 + 3.9 ${{ matrix.python_version }} - name: run PostCommit XVR Flink script env: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} - if: ${{ matrix.python_version != '3.8' }} + if: ${{ matrix.python_version != '3.9' }} uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:flink:${{ env.FlinkVersion }}:job-server:validatesCrossLanguageRunner @@ -91,7 +91,7 @@ jobs: - name: run PostCommit XVR Flink script env: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} - if: ${{ matrix.python_version == '3.8' }} + if: ${{ matrix.python_version == '3.9' }} uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:flink:${{ env.FlinkVersion }}:job-server:validatesCrossLanguageRunner diff --git a/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml index 779d5881ca7a..66770c9a1683 100644 --- a/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_JavaUsingPython_Dataflow"] job_phrase: ["Run XVR_JavaUsingPython_Dataflow PostCommit"] - python_version: ['3.8','3.12'] + python_version: ['3.9','3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -75,7 +75,7 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: | - 3.8 + 3.9 ${{ matrix.python_version }} - name: run PostCommit XVR JavaUsingPython Dataflow script uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml index 14404e8a9a41..f1269a0ddd09 100644 --- a/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_PythonUsingJava_Dataflow"] job_phrase: ["Run XVR_PythonUsingJava_Dataflow PostCommit"] - python_version: ['3.8','3.12'] + python_version: ['3.9','3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -75,7 +75,7 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: | - 3.8 + 3.9 ${{ matrix.python_version }} - name: run PostCommit XVR PythonUsingJava Dataflow script uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_PostCommit_XVR_Samza.yml b/.github/workflows/beam_PostCommit_XVR_Samza.yml index 2d854a3678e7..2d26c9131839 100644 --- a/.github/workflows/beam_PostCommit_XVR_Samza.yml +++ b/.github/workflows/beam_PostCommit_XVR_Samza.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_Samza"] job_phrase: ["Run XVR_Samza PostCommit"] - python_version: ['3.8','3.12'] + python_version: ['3.9','3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -77,12 +77,12 @@ jobs: with: java-version: 8 python-version: | - 3.8 + 3.9 ${{ matrix.python_version }} - name: run PostCommit XVR Samza script env: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} - if: ${{ matrix.python_version != '3.8' }} + if: ${{ matrix.python_version != '3.9' }} uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:samza:job-server:validatesCrossLanguageRunner @@ -92,7 +92,7 @@ jobs: - name: run PostCommit XVR Samza script env: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} - if: ${{ matrix.python_version == '3.8' }} + if: ${{ matrix.python_version == '3.9' }} uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:samza:job-server:validatesCrossLanguageRunner diff --git a/.github/workflows/beam_PostCommit_XVR_Spark3.yml b/.github/workflows/beam_PostCommit_XVR_Spark3.yml index 83554ecfa84c..c1880e01292b 100644 --- a/.github/workflows/beam_PostCommit_XVR_Spark3.yml +++ b/.github/workflows/beam_PostCommit_XVR_Spark3.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_Spark3"] job_phrase: ["Run XVR_Spark3 PostCommit"] - python_version: ['3.8','3.12'] + python_version: ['3.9','3.12'] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -75,12 +75,12 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: | - 3.8 + 3.9 ${{ matrix.python_version }} - name: run PostCommit XVR Spark3 script env: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} - if: ${{ matrix.python_version != '3.8' }} + if: ${{ matrix.python_version != '3.9' }} uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:spark:3:job-server:validatesCrossLanguageRunner @@ -90,7 +90,7 @@ jobs: - name: run PostCommit XVR Spark3 script env: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} - if: ${{ matrix.python_version == '3.8' }} + if: ${{ matrix.python_version == '3.9' }} uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:spark:3:job-server:validatesCrossLanguageRunner diff --git a/.github/workflows/beam_PreCommit_Portable_Python.yml b/.github/workflows/beam_PreCommit_Portable_Python.yml index e1e1e6033087..037df9a17c45 100644 --- a/.github/workflows/beam_PreCommit_Portable_Python.yml +++ b/.github/workflows/beam_PreCommit_Portable_Python.yml @@ -86,7 +86,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Portable_Python'] job_phrase: ['Run Portable_Python PreCommit'] - python_version: ['3.8', '3.12'] + python_version: ['3.9', '3.12'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || @@ -106,7 +106,7 @@ jobs: java-version: default python-version: | ${{ matrix.python_version }} - 3.8 + 3.9 - name: Set PY_VER_CLEAN id: set_py_ver_clean run: | diff --git a/.github/workflows/beam_PreCommit_Python.yml b/.github/workflows/beam_PreCommit_Python.yml index 2fbab55819f3..fb1c6c80873a 100644 --- a/.github/workflows/beam_PreCommit_Python.yml +++ b/.github/workflows/beam_PreCommit_Python.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python'] job_phrase: ['Run Python PreCommit'] - python_version: ['3.8','3.9','3.10','3.11','3.12'] + python_version: ['3.9','3.10','3.11','3.12'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_PythonDocker.yml b/.github/workflows/beam_PreCommit_PythonDocker.yml index 129429238b1f..63fc6d55e19a 100644 --- a/.github/workflows/beam_PreCommit_PythonDocker.yml +++ b/.github/workflows/beam_PreCommit_PythonDocker.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ["beam_PreCommit_PythonDocker"] job_phrase: ["Run PythonDocker PreCommit"] - python_version: ['3.8','3.9','3.10','3.11','3.12'] + python_version: ['3.9','3.10','3.11','3.12'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python_Coverage.yml b/.github/workflows/beam_PreCommit_Python_Coverage.yml index a22e3c338eed..0e295250817d 100644 --- a/.github/workflows/beam_PreCommit_Python_Coverage.yml +++ b/.github/workflows/beam_PreCommit_Python_Coverage.yml @@ -87,7 +87,7 @@ jobs: - name: Run preCommitPyCoverage uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:test-suites:tox:py38:preCommitPyCoverage + gradle-command: :sdks:python:test-suites:tox:py39:preCommitPyCoverage - uses: codecov/codecov-action@v3 with: flags: python diff --git a/.github/workflows/beam_PreCommit_Python_Dataframes.yml b/.github/workflows/beam_PreCommit_Python_Dataframes.yml index f498dadae92d..f045842e061d 100644 --- a/.github/workflows/beam_PreCommit_Python_Dataframes.yml +++ b/.github/workflows/beam_PreCommit_Python_Dataframes.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_Dataframes'] job_phrase: ['Run Python_Dataframes PreCommit'] - python_version: ['3.8','3.9','3.10','3.11','3.12'] + python_version: ['3.9','3.10','3.11','3.12'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python_Examples.yml b/.github/workflows/beam_PreCommit_Python_Examples.yml index 0fb404e7f55b..09d46217d6d6 100644 --- a/.github/workflows/beam_PreCommit_Python_Examples.yml +++ b/.github/workflows/beam_PreCommit_Python_Examples.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_Examples'] job_phrase: ['Run Python_Examples PreCommit'] - python_version: ['3.8','3.9','3.10','3.11','3.12'] + python_version: ['3.9','3.10','3.11','3.12'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python_Integration.yml b/.github/workflows/beam_PreCommit_Python_Integration.yml index a2d80806d2bf..20aade431f6d 100644 --- a/.github/workflows/beam_PreCommit_Python_Integration.yml +++ b/.github/workflows/beam_PreCommit_Python_Integration.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_Integration'] job_phrase: ['Run Python_Integration PreCommit'] - python_version: ['3.8', '3.12'] + python_version: ['3.9', '3.12'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python_ML.yml b/.github/workflows/beam_PreCommit_Python_ML.yml index c5e596e3b421..714eceef5f6b 100644 --- a/.github/workflows/beam_PreCommit_Python_ML.yml +++ b/.github/workflows/beam_PreCommit_Python_ML.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_ML'] job_phrase: ['Run Python_ML PreCommit'] - python_version: ['3.8','3.9','3.10','3.11','3.12'] + python_version: ['3.9','3.10','3.11','3.12'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python_Runners.yml b/.github/workflows/beam_PreCommit_Python_Runners.yml index 66037cfaffdb..5db6e94be781 100644 --- a/.github/workflows/beam_PreCommit_Python_Runners.yml +++ b/.github/workflows/beam_PreCommit_Python_Runners.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_Runners'] job_phrase: ['Run Python_Runners PreCommit'] - python_version: ['3.8','3.9','3.10','3.11','3.12'] + python_version: ['3.9','3.10','3.11','3.12'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python_Transforms.yml b/.github/workflows/beam_PreCommit_Python_Transforms.yml index caec491a7515..820ca3e26df6 100644 --- a/.github/workflows/beam_PreCommit_Python_Transforms.yml +++ b/.github/workflows/beam_PreCommit_Python_Transforms.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_Transforms'] job_phrase: ['Run Python_Transforms PreCommit'] - python_version: ['3.8','3.9','3.10','3.11','3.12'] + python_version: ['3.9','3.10','3.11','3.12'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml b/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml index c7ba234bef4b..f79712ac2d76 100644 --- a/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml +++ b/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml @@ -82,7 +82,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Xlang_Generated_Transforms'] job_phrase: ['Run Xlang_Generated_Transforms PreCommit'] - python_version: ['3.8'] + python_version: ['3.9'] if: | github.event_name == 'push' || github.event_name == 'workflow_dispatch' || diff --git a/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml b/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml index 72de1f15229f..c971ef36a081 100644 --- a/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml +++ b/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml @@ -66,7 +66,6 @@ jobs: - "java:container:java11" - "java:container:java17" - "java:container:java21" - - "python:container:py38" - "python:container:py39" - "python:container:py310" - "python:container:py311" diff --git a/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml b/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml index e95b2a43845e..c2b360a7ce71 100644 --- a/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml +++ b/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml @@ -54,7 +54,7 @@ jobs: matrix: job_name: [beam_Python_ValidatesContainer_Dataflow_ARM] job_phrase: [Run Python ValidatesContainer Dataflow ARM] - python_version: ['3.8','3.9','3.10','3.11','3.12'] + python_version: ['3.9','3.10','3.11','3.12'] if: | github.event_name == 'push' || (github.event_name == 'schedule' && github.repository == 'apache/beam') || diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml index f944ce90c9f1..ec65ae99072a 100644 --- a/.github/workflows/build_release_candidate.yml +++ b/.github/workflows/build_release_candidate.yml @@ -260,10 +260,10 @@ jobs: with: distribution: 'temurin' java-version: '11' - - name: Install Python 3.8 + - name: Install Python 3.9 uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - run: echo $JAVA_HOME - run: echo "JAVA11_HOME=${JAVA_HOME}" >> "$GITHUB_OUTPUT" id: export-java11 @@ -310,10 +310,10 @@ jobs: path: beam-site token: ${{ github.event.inputs.REPO_TOKEN }} ref: release-docs - - name: Install Python 3.8 + - name: Install Python 3.9 uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Install node uses: actions/setup-node@v4 with: diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index f97f4de50aee..a2283ceb6e97 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -49,7 +49,7 @@ jobs: env: EVENT_NAME: ${{ github.event_name }} # Keep in sync with py_version matrix value below - if changed, change that as well. - PY_VERSIONS_FULL: "cp38-* cp39-* cp310-* cp311-* cp312-*" + PY_VERSIONS_FULL: "cp39-* cp310-* cp311-* cp312-*" outputs: gcp-variables-set: ${{ steps.check_gcp_variables.outputs.gcp-variables-set }} py-versions-full: ${{ steps.set-py-versions.outputs.py-versions-full }} @@ -92,7 +92,7 @@ jobs: - name: Install python uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - name: Get tag id: get_tag run: | @@ -230,7 +230,7 @@ jobs: {"os": "ubuntu-20.04", "runner": [self-hosted, ubuntu-20.04, main], "python": "${{ needs.check_env_variables.outputs.py-versions-test }}", arch: "aarch64" } ] # Keep in sync with PY_VERSIONS_FULL env var abvove - if changed, change that as well. - py_version: ["cp38-*", "cp39-*", "cp310-*", "cp311-*", "cp312-*"] + py_version: ["cp39-*", "cp310-*", "cp311-*", "cp312-*"] steps: - name: Download python source distribution from artifacts # Pinned to v3 because of https://github.com/actions/download-artifact/issues/249 @@ -248,7 +248,7 @@ jobs: - name: Install Python uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - uses: docker/setup-qemu-action@v1 if: ${{matrix.os_python.arch == 'aarch64'}} name: Set up QEMU diff --git a/.github/workflows/dask_runner_tests.yml b/.github/workflows/dask_runner_tests.yml index 5f39852c228c..f87c70d8b720 100644 --- a/.github/workflows/dask_runner_tests.yml +++ b/.github/workflows/dask_runner_tests.yml @@ -43,7 +43,7 @@ jobs: - name: Install python uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - name: Build source working-directory: ./sdks/python run: pip install -U build && python -m build --sdist @@ -64,7 +64,6 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] params: [ - {"py_ver": "3.8", "tox_env": "py38"}, {"py_ver": "3.9", "tox_env": "py39"}, {"py_ver": "3.10", "tox_env": "py310" }, ] diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_10_byte_records.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_10_byte_records.txt index 12ffc1790e46..5d1a0be9950e 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_10_byte_records.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_10_byte_records.txt @@ -22,7 +22,7 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest --top_count=20 --streaming --use_stateful_load_generator diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_4.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_4.txt index c7d5552a03bd..650236a9c500 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_4.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_4.txt @@ -22,7 +22,7 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest --fanout=4 --top_count=20 --streaming diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_8.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_8.txt index bffdeab2cb11..4208571fef62 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_8.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_8.txt @@ -22,7 +22,7 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest --fanout=8 --top_count=20 --streaming diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_100B_records.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_100B_records.txt index 4cb5bfb0d988..f4f5e7de8369 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_100B_records.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_100B_records.txt @@ -24,5 +24,5 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt index bf9085141eab..df27dc7c4470 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt @@ -24,5 +24,5 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt index a59f873eb775..6b87f61eed8a 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt @@ -24,5 +24,5 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt index 0e5d00b96151..621777663be0 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt @@ -24,5 +24,5 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Counters.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Counters.txt index 4d8bda8ac2f8..fe451559e625 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Counters.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Counters.txt @@ -25,5 +25,5 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Iterations.txt index e84cee2f50cf..dd5addb65d14 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Iterations.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Iterations.txt @@ -25,5 +25,5 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_200_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_200_Iterations.txt index 4d8bda8ac2f8..fe451559e625 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_200_Iterations.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_200_Iterations.txt @@ -25,5 +25,5 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_100_Counters.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_100_Counters.txt index b17e2cecc2c8..308deb3ecf4d 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_100_Counters.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_100_Counters.txt @@ -26,6 +26,6 @@ --streaming --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest --use_stateful_load_generator --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Counters.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Counters.txt index 957bc6c086d8..78ecc1fd98dd 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Counters.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Counters.txt @@ -26,6 +26,6 @@ --streaming --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest --use_stateful_load_generator --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Iterations.txt index baa34ec455b5..04a1213d4039 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Iterations.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Iterations.txt @@ -27,6 +27,6 @@ --stateful --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest --use_stateful_load_generator --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_200_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_200_Iterations.txt index 44483a6e51cc..a2f7d7600da8 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_200_Iterations.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_200_Iterations.txt @@ -26,6 +26,6 @@ --streaming --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest --use_stateful_load_generator --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_5_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_5_Iterations.txt index 571b33fb7a49..f49be6c70582 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_5_Iterations.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_5_Iterations.txt @@ -30,6 +30,6 @@ --shutdown_sources_after_idle_ms=300000 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest --use_stateful_load_generator --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/playground_backend_precommit.yml b/.github/workflows/playground_backend_precommit.yml index 79517e705c27..9ba6cf20534f 100644 --- a/.github/workflows/playground_backend_precommit.yml +++ b/.github/workflows/playground_backend_precommit.yml @@ -33,7 +33,7 @@ jobs: runs-on: ubuntu-latest env: DATASTORE_EMULATOR_VERSION: '423.0.0' - PYTHON_VERSION: '3.8' + PYTHON_VERSION: '3.9' JAVA_VERSION: '11' steps: - name: Check out the repo diff --git a/.github/workflows/python_dependency_tests.yml b/.github/workflows/python_dependency_tests.yml index fed1056b90b2..2eaa9e4ce5aa 100644 --- a/.github/workflows/python_dependency_tests.yml +++ b/.github/workflows/python_dependency_tests.yml @@ -26,7 +26,6 @@ jobs: matrix: os: [ubuntu-latest] params: [ - {"py_ver": "3.8", "py_env": "py38"}, {"py_ver": "3.9", "py_env": "py39"}, {"py_ver": "3.10", "py_env": "py310" }, { "py_ver": "3.11", "py_env": "py311" }, diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index e7461be2507e..3000d1871be3 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -98,7 +98,6 @@ jobs: matrix: os: [macos-latest, windows-latest] params: [ - {"py_ver": "3.8", "tox_env": "py38"}, {"py_ver": "3.9", "tox_env": "py39"}, {"py_ver": "3.10", "tox_env": "py310" }, { "py_ver": "3.11", "tox_env": "py311" }, @@ -139,7 +138,7 @@ jobs: fail-fast: false matrix: os: [[self-hosted, ubuntu-20.04, main], macos-latest, windows-latest] - python: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python: ["3.9", "3.10", "3.11", "3.12"] steps: - name: Checkout code uses: actions/checkout@v4 @@ -166,7 +165,7 @@ jobs: fail-fast: false matrix: os: [[self-hosted, ubuntu-20.04, main], macos-latest, windows-latest] - python: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python: ["3.9", "3.10", "3.11", "3.12"] steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/run_perf_alert_tool.yml b/.github/workflows/run_perf_alert_tool.yml index 4bb5df41dcfb..a6aae616efec 100644 --- a/.github/workflows/run_perf_alert_tool.yml +++ b/.github/workflows/run_perf_alert_tool.yml @@ -39,7 +39,7 @@ jobs: - name: Install python uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - name: Install Apache Beam working-directory: ./sdks/python run: pip install -e .[gcp,test] diff --git a/.github/workflows/run_rc_validation.yml b/.github/workflows/run_rc_validation.yml index 15979a9e1acd..801a72d37130 100644 --- a/.github/workflows/run_rc_validation.yml +++ b/.github/workflows/run_rc_validation.yml @@ -106,7 +106,7 @@ jobs: if: ${{github.event.inputs.RUN_SQL_TAXI_WITH_DATAFLOW == 'true'}} strategy: matrix: - py_version: [3.8] + py_version: [3.9] steps: - name: Checkout code uses: actions/checkout@v4 @@ -171,7 +171,7 @@ jobs: if: ${{github.event.inputs.RUN_PYTHON_CROSS_VALIDATION == 'true'}} strategy: matrix: - py_version: [3.8] + py_version: [3.9] steps: - name: Checkout code uses: actions/checkout@v4 @@ -286,7 +286,7 @@ jobs: - name: Install Python uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Setting python env uses: ./.github/actions/common-rc-validation @@ -351,7 +351,7 @@ jobs: if: ${{github.event.inputs.RUN_DIRECT_RUNNER_TESTS == 'true' }} strategy: matrix: - py_version: [3.8] + py_version: [3.9] needs: generate_shared_pubsub steps: - name: Checkout code @@ -399,7 +399,7 @@ jobs: if: ${{github.event.inputs.RUN_DATAFLOW_RUNNER_TESTS=='true'}} strategy: matrix: - py_version: [3.8] + py_version: [3.9] needs: [generate_shared_pubsub] steps: - name: Checkout code @@ -452,7 +452,7 @@ jobs: if: ${{github.event.inputs.RUN_DIRECT_RUNNER_TESTS == 'true' }} strategy: matrix: - py_version: [3.8] + py_version: [3.9] needs: [generate_shared_pubsub] steps: - name: Checkout code @@ -501,7 +501,7 @@ jobs: if: ${{github.event.inputs.RUN_DATAFLOW_RUNNER_TESTS=='true'}} strategy: matrix: - py_version: [3.8] + py_version: [3.9] needs: [generate_shared_pubsub] steps: - name: Checkout code diff --git a/.github/workflows/typescript_tests.yml b/.github/workflows/typescript_tests.yml index 1b45ea67b5c6..a4e4c2926f84 100644 --- a/.github/workflows/typescript_tests.yml +++ b/.github/workflows/typescript_tests.yml @@ -85,7 +85,7 @@ jobs: - name: Install Python uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - name: Setup Beam Python working-directory: ./sdks/python run: | @@ -140,7 +140,7 @@ jobs: - name: Install python uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - name: Setup Beam Python working-directory: ./sdks/python run: | diff --git a/.github/workflows/update_python_dependencies.yml b/.github/workflows/update_python_dependencies.yml index a91aff39f29a..0ab52e97b9f0 100644 --- a/.github/workflows/update_python_dependencies.yml +++ b/.github/workflows/update_python_dependencies.yml @@ -56,7 +56,6 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: | - 3.8 3.9 3.10 3.11 diff --git a/.test-infra/jenkins/PythonTestProperties.groovy b/.test-infra/jenkins/PythonTestProperties.groovy index 98257a6e1c28..7e8e4ad3d8fd 100644 --- a/.test-infra/jenkins/PythonTestProperties.groovy +++ b/.test-infra/jenkins/PythonTestProperties.groovy @@ -20,10 +20,10 @@ class PythonTestProperties { // Indicates all supported Python versions. // This must be sorted in ascending order. final static List ALL_SUPPORTED_VERSIONS = [ - '3.8', '3.9', '3.10', - '3.11' + '3.11', + '3.12' ] final static List SUPPORTED_CONTAINER_TASKS = ALL_SUPPORTED_VERSIONS.collect { "py${it.replace('.', '')}" @@ -37,10 +37,10 @@ class PythonTestProperties { final static List CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS = ESSENTIAL_VERSIONS final static List CROSS_LANGUAGE_VALIDATES_RUNNER_DATAFLOW_USING_SQL_PYTHON_VERSIONS = [HIGHEST_SUPPORTED] final static List VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS = ALL_SUPPORTED_VERSIONS - final static String LOAD_TEST_PYTHON_VERSION = '3.8' - final static String RUN_INFERENCE_TEST_PYTHON_VERSION = '3.8' - final static String CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION = '3.8' + final static String LOAD_TEST_PYTHON_VERSION = '3.9' + final static String RUN_INFERENCE_TEST_PYTHON_VERSION = '3.9' + final static String CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION = '3.9' // Use for various shell scripts triggered by Jenkins. // Gradle scripts should use project.ext.pythonVersion defined by PythonNature/BeamModulePlugin. - final static String DEFAULT_INTERPRETER = 'python3.8' + final static String DEFAULT_INTERPRETER = 'python3.9' } diff --git a/.test-infra/metrics/influxdb/Dockerfile b/.test-infra/metrics/influxdb/Dockerfile index 57a541fb9955..0ec7bd6f2677 100644 --- a/.test-infra/metrics/influxdb/Dockerfile +++ b/.test-infra/metrics/influxdb/Dockerfile @@ -16,7 +16,7 @@ # limitations under the License. ################################################################################ -FROM python:3.8-slim +FROM python:3.9-slim RUN pip install --no-cache-dir gsutil diff --git a/.test-infra/metrics/influxdb/gsutil/Dockerfile b/.test-infra/metrics/influxdb/gsutil/Dockerfile index 09ccf150b938..ea6621e2cf9d 100644 --- a/.test-infra/metrics/influxdb/gsutil/Dockerfile +++ b/.test-infra/metrics/influxdb/gsutil/Dockerfile @@ -16,7 +16,7 @@ # limitations under the License. ################################################################################ -FROM python:3.8-slim +FROM python:3.9-slim # google-compute-engine package allows to obtain credentials for service # account specified in .boto file. diff --git a/.test-infra/metrics/sync/github/Dockerfile b/.test-infra/metrics/sync/github/Dockerfile index e686a9b2f682..3116d0f211fa 100644 --- a/.test-infra/metrics/sync/github/Dockerfile +++ b/.test-infra/metrics/sync/github/Dockerfile @@ -16,7 +16,7 @@ # limitations under the License. ################################################################################ -FROM python:3.8-slim +FROM python:3.9-slim WORKDIR /usr/src/app diff --git a/.test-infra/metrics/sync/jenkins/Dockerfile b/.test-infra/metrics/sync/jenkins/Dockerfile index 32247b438b59..62829ada38ee 100644 --- a/.test-infra/metrics/sync/jenkins/Dockerfile +++ b/.test-infra/metrics/sync/jenkins/Dockerfile @@ -16,7 +16,7 @@ # limitations under the License. ################################################################################ -FROM python:3.8-slim +FROM python:3.9-slim WORKDIR /usr/src/app diff --git a/.test-infra/mock-apis/poetry.lock b/.test-infra/mock-apis/poetry.lock index 98985df7ea4a..322980cc2e75 100644 --- a/.test-infra/mock-apis/poetry.lock +++ b/.test-infra/mock-apis/poetry.lock @@ -171,7 +171,7 @@ name = "protobuf" version = "4.25.0" description = "" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ {file = "protobuf-4.25.0-cp310-abi3-win32.whl", hash = "sha256:5c1203ac9f50e4853b0a0bfffd32c67118ef552a33942982eeab543f5c634395"}, {file = "protobuf-4.25.0-cp310-abi3-win_amd64.whl", hash = "sha256:c40ff8f00aa737938c5378d461637d15c442a12275a81019cc2fef06d81c9419"}, @@ -191,7 +191,7 @@ name = "setuptools" version = "70.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ {file = "setuptools-70.0.0-py3-none-any.whl", hash = "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4"}, {file = "setuptools-70.0.0.tar.gz", hash = "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0"}, @@ -206,7 +206,7 @@ name = "soupsieve" version = "2.5" description = "A modern CSS selector implementation for Beautiful Soup." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, @@ -214,5 +214,5 @@ files = [ [metadata] lock-version = "2.0" -python-versions = "^3.8" +python-versions = "^3.9" content-hash = "35ed5a98dd3f951bbfc44b949ad9148634159976cb54ac6f257d119c12d9d924" diff --git a/build.gradle.kts b/build.gradle.kts index e6295384b753..a9605cce5645 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -470,7 +470,6 @@ tasks.register("playgroundPreCommit") { tasks.register("pythonPreCommit") { dependsOn(":sdks:python:test-suites:tox:pycommon:preCommitPyCommon") - dependsOn(":sdks:python:test-suites:tox:py38:preCommitPy38") dependsOn(":sdks:python:test-suites:tox:py39:preCommitPy39") dependsOn(":sdks:python:test-suites:tox:py310:preCommitPy310") dependsOn(":sdks:python:test-suites:tox:py311:preCommitPy311") @@ -487,7 +486,6 @@ tasks.register("pythonDocsPreCommit") { } tasks.register("pythonDockerBuildPreCommit") { - dependsOn(":sdks:python:container:py38:docker") dependsOn(":sdks:python:container:py39:docker") dependsOn(":sdks:python:container:py310:docker") dependsOn(":sdks:python:container:py311:docker") @@ -546,12 +544,11 @@ tasks.register("python312PostCommit") { } tasks.register("portablePythonPreCommit") { - dependsOn(":sdks:python:test-suites:portable:py38:preCommitPy38") + dependsOn(":sdks:python:test-suites:portable:py39:preCommitPy39") dependsOn(":sdks:python:test-suites:portable:py312:preCommitPy312") } tasks.register("pythonSparkPostCommit") { - dependsOn(":sdks:python:test-suites:portable:py38:sparkValidatesRunner") dependsOn(":sdks:python:test-suites:portable:py39:sparkValidatesRunner") dependsOn(":sdks:python:test-suites:portable:py312:sparkValidatesRunner") } @@ -576,15 +573,15 @@ tasks.register("javaExamplesDataflowPrecommit") { tasks.register("whitespacePreCommit") { // TODO(https://github.com/apache/beam/issues/20209): Find a better way to specify the tasks without hardcoding py version. - dependsOn(":sdks:python:test-suites:tox:py38:archiveFilesToLint") - dependsOn(":sdks:python:test-suites:tox:py38:unpackFilesToLint") - dependsOn(":sdks:python:test-suites:tox:py38:whitespacelint") + dependsOn(":sdks:python:test-suites:tox:py39:archiveFilesToLint") + dependsOn(":sdks:python:test-suites:tox:py39:unpackFilesToLint") + dependsOn(":sdks:python:test-suites:tox:py39:whitespacelint") } tasks.register("typescriptPreCommit") { // TODO(https://github.com/apache/beam/issues/20209): Find a better way to specify the tasks without hardcoding py version. - dependsOn(":sdks:python:test-suites:tox:py38:eslint") - dependsOn(":sdks:python:test-suites:tox:py38:jest") + dependsOn(":sdks:python:test-suites:tox:py39:eslint") + dependsOn(":sdks:python:test-suites:tox:py39:jest") } tasks.register("pushAllRunnersDockerImages") { diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 7b27cb8e140a..0110f71974e2 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -2867,7 +2867,7 @@ class BeamModulePlugin implements Plugin { // Transform service delivers transforms that refer to SDK harness containers with following sufixes. def transformServiceJavaContainerSuffix = 'java11' - def transformServicePythonContainerSuffix = '38' + def transformServicePythonContainerSuffix = '39' def setupTask = project.tasks.register(config.name+"Setup", Exec) { // Containers for main SDKs when running tests. @@ -2958,7 +2958,7 @@ class BeamModulePlugin implements Plugin { // If none of them applied, version set here will be used as default value. // TODO(BEAM-12000): Move default value to Py3.9. project.ext.pythonVersion = project.hasProperty('pythonVersion') ? - project.pythonVersion : '3.8' + project.pythonVersion : '3.9' def setupVirtualenv = project.tasks.register('setupVirtualenv') { doLast { @@ -3145,10 +3145,10 @@ class BeamModulePlugin implements Plugin { mustRunAfter = [ ":runners:flink:${project.ext.latestFlinkVersion}:job-server:shadowJar", ':runners:spark:3:job-server:shadowJar', - ':sdks:python:container:py38:docker', ':sdks:python:container:py39:docker', ':sdks:python:container:py310:docker', ':sdks:python:container:py311:docker', + ':sdks:python:container:py312:docker', ] doLast { // TODO: Figure out GCS credentials and use real GCS input and output. diff --git a/gradle.properties b/gradle.properties index 084d7d8999b3..20a478cd6aab 100644 --- a/gradle.properties +++ b/gradle.properties @@ -41,4 +41,4 @@ docker_image_default_repo_prefix=beam_ # supported flink versions flink_versions=1.15,1.16,1.17,1.18 # supported python versions -python_versions=3.8,3.9,3.10,3.11,3.12 +python_versions=3.9,3.10,3.11,3.12 diff --git a/local-env-setup.sh b/local-env-setup.sh index f13dc88432a6..ba30813b2bcc 100755 --- a/local-env-setup.sh +++ b/local-env-setup.sh @@ -55,7 +55,7 @@ if [ "$kernelname" = "Linux" ]; then exit fi - for ver in 3.8 3.9 3.10 3.11 3.12 3; do + for ver in 3.9 3.10 3.11 3.12 3; do apt install --yes python$ver-venv done @@ -89,7 +89,7 @@ elif [ "$kernelname" = "Darwin" ]; then echo "Installing openjdk@8" brew install openjdk@8 fi - for ver in 3.8 3.9 3.10 3.11 3.12; do + for ver in 3.9 3.10 3.11 3.12; do if brew ls --versions python@$ver > /dev/null; then echo "python@$ver already installed. Skipping" brew info python@$ver diff --git a/release/src/main/Dockerfile b/release/src/main/Dockerfile index 8f981fddfcd8..8d0f2229adfa 100644 --- a/release/src/main/Dockerfile +++ b/release/src/main/Dockerfile @@ -42,12 +42,11 @@ RUN curl https://pyenv.run | bash && \ echo 'command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"' >> /root/.bashrc && \ echo ''eval "$(pyenv init -)"'' >> /root/.bashrc && \ source /root/.bashrc && \ - pyenv install 3.8.9 && \ pyenv install 3.9.4 && \ pyenv install 3.10.7 && \ pyenv install 3.11.3 && \ pyenv install 3.12.3 && \ - pyenv global 3.8.9 3.9.4 3.10.7 3.11.3 3.12.3 + pyenv global 3.9.4 3.10.7 3.11.3 3.12.3 # Install a Go version >= 1.16 so we can bootstrap higher # Go versions diff --git a/release/src/main/python-release/python_release_automation.sh b/release/src/main/python-release/python_release_automation.sh index 2f6986885a96..248bdd9b65ac 100755 --- a/release/src/main/python-release/python_release_automation.sh +++ b/release/src/main/python-release/python_release_automation.sh @@ -19,7 +19,7 @@ source release/src/main/python-release/run_release_candidate_python_quickstart.sh source release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh -for version in 3.8 3.9 3.10 3.11 3.12 +for version in 3.9 3.10 3.11 3.12 do run_release_candidate_python_quickstart "tar" "python${version}" run_release_candidate_python_mobile_gaming "tar" "python${version}" diff --git a/release/src/main/scripts/run_rc_validation.sh b/release/src/main/scripts/run_rc_validation.sh index 91bfa9e2f8bb..9c93ed4ef4d4 100755 --- a/release/src/main/scripts/run_rc_validation.sh +++ b/release/src/main/scripts/run_rc_validation.sh @@ -99,7 +99,7 @@ HUB_VERSION=2.12.0 HUB_ARTIFACTS_NAME=hub-linux-amd64-${HUB_VERSION} BACKUP_BASHRC=.bashrc_backup_$(date +"%Y%m%d%H%M%S") BACKUP_M2=settings_backup_$(date +"%Y%m%d%H%M%S").xml -declare -a PYTHON_VERSIONS_TO_VALIDATE=("python3.8") +declare -a PYTHON_VERSIONS_TO_VALIDATE=("python3.9") echo "" echo "====================Checking Environment & Variables=================" echo "PLEASE update RC_VALIDATE_CONFIGS in file script.config first." diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java index 37c20c61ad8e..01ceac9da585 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java @@ -1255,8 +1255,8 @@ public void testNoStagingLocationAndNoTempLocationFails() { @Test public void testApplySdkEnvironmentOverrides() throws IOException { DataflowPipelineOptions options = buildPipelineOptions(); - String dockerHubPythonContainerUrl = "apache/beam_python3.8_sdk:latest"; - String gcrPythonContainerUrl = "gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest"; + String dockerHubPythonContainerUrl = "apache/beam_python3.9_sdk:latest"; + String gcrPythonContainerUrl = "gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest"; options.setSdkHarnessContainerImageOverrides(".*python.*," + gcrPythonContainerUrl); DataflowRunner runner = DataflowRunner.fromOptions(options); RunnerApi.Pipeline pipeline = @@ -1297,8 +1297,8 @@ public void testApplySdkEnvironmentOverrides() throws IOException { @Test public void testApplySdkEnvironmentOverridesByDefault() throws IOException { DataflowPipelineOptions options = buildPipelineOptions(); - String dockerHubPythonContainerUrl = "apache/beam_python3.8_sdk:latest"; - String gcrPythonContainerUrl = "gcr.io/cloud-dataflow/v1beta3/beam_python3.8_sdk:latest"; + String dockerHubPythonContainerUrl = "apache/beam_python3.9_sdk:latest"; + String gcrPythonContainerUrl = "gcr.io/cloud-dataflow/v1beta3/beam_python3.9_sdk:latest"; DataflowRunner runner = DataflowRunner.fromOptions(options); RunnerApi.Pipeline pipeline = RunnerApi.Pipeline.newBuilder() diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py index 20cae582f320..97996bd6cbb2 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py @@ -82,7 +82,7 @@ _LOGGER = logging.getLogger(__name__) -_PYTHON_VERSIONS_SUPPORTED_BY_DATAFLOW = ['3.8', '3.9', '3.10', '3.11', '3.12'] +_PYTHON_VERSIONS_SUPPORTED_BY_DATAFLOW = ['3.9', '3.10', '3.11', '3.12'] class Environment(object): diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py index 8331d9cf3919..022136aae9a2 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py @@ -938,7 +938,7 @@ def test_experiment_use_multiple_sdk_containers(self): @mock.patch( 'apache_beam.runners.dataflow.internal.apiclient.sys.version_info', - (3, 8)) + (3, 9)) def test_get_python_sdk_name(self): pipeline_options = PipelineOptions([ '--project', @@ -957,7 +957,7 @@ def test_get_python_sdk_name(self): 1, FAKE_PIPELINE_URL) self.assertEqual( - 'Apache Beam Python 3.8 SDK', environment._get_python_sdk_name()) + 'Apache Beam Python 3.9 SDK', environment._get_python_sdk_name()) @mock.patch( 'apache_beam.runners.dataflow.internal.apiclient.sys.version_info', diff --git a/sdks/python/apache_beam/runners/interactive/interactive_beam.py b/sdks/python/apache_beam/runners/interactive/interactive_beam.py index 0e170eb0f508..9554abf3a47a 100644 --- a/sdks/python/apache_beam/runners/interactive/interactive_beam.py +++ b/sdks/python/apache_beam/runners/interactive/interactive_beam.py @@ -36,7 +36,9 @@ import logging from datetime import timedelta +from typing import Any from typing import Dict +from typing import Iterable from typing import List from typing import Optional from typing import Union @@ -46,6 +48,7 @@ import apache_beam as beam from apache_beam.dataframe.frame_base import DeferredBase from apache_beam.options.pipeline_options import FlinkRunnerOptions +from apache_beam.pvalue import PCollection from apache_beam.runners.interactive import interactive_environment as ie from apache_beam.runners.interactive.dataproc.dataproc_cluster_manager import DataprocClusterManager from apache_beam.runners.interactive.dataproc.types import ClusterIdentifier @@ -681,13 +684,11 @@ def run_pipeline(self): @progress_indicated def show( - *pcolls, - include_window_info=False, - visualize_data=False, - n='inf', - duration='inf'): - # type: (*Union[Dict[Any, PCollection], Iterable[PCollection], PCollection], bool, bool, Union[int, str], Union[int, str]) -> None # noqa: F821 - + *pcolls: Union[Dict[Any, PCollection], Iterable[PCollection], PCollection], + include_window_info: bool = False, + visualize_data: bool = False, + n: Union[int, str] = 'inf', + duration: Union[int, str] = 'inf'): """Shows given PCollections in an interactive exploratory way if used within a notebook, or prints a heading sampled data if used within an ipython shell. Noop if used in a non-interactive environment. diff --git a/sdks/python/apache_beam/transforms/external_transform_provider_it_test.py b/sdks/python/apache_beam/transforms/external_transform_provider_it_test.py index df30611598fe..6b26206908fb 100644 --- a/sdks/python/apache_beam/transforms/external_transform_provider_it_test.py +++ b/sdks/python/apache_beam/transforms/external_transform_provider_it_test.py @@ -193,7 +193,7 @@ def test_pretty_types(self): expected_type_names = [('List[str]', True), ('numpy.int16', False), ('str', False), ('Dict[str, numpy.float64]', False), ('Dict[str, List[numpy.int64]]', True), - ('Dict[int, Union[str, NoneType]]', False)] + ('Dict[int, Optional[str]]', False)] for i in range(len(types)): self.assertEqual( diff --git a/sdks/python/apache_beam/yaml/yaml_io.py b/sdks/python/apache_beam/yaml/yaml_io.py index 689b5d9eb086..e4c3f43f8472 100644 --- a/sdks/python/apache_beam/yaml/yaml_io.py +++ b/sdks/python/apache_beam/yaml/yaml_io.py @@ -240,7 +240,7 @@ def _validate_schema(): return ( beam_schema, lambda record: covert_to_row( - fastavro.schemaless_reader(io.BytesIO(record), schema))) # type: ignore[call-arg] + fastavro.schemaless_reader(io.BytesIO(record), schema))) else: raise ValueError(f'Unknown format: {format}') diff --git a/sdks/python/expansion-service-container/Dockerfile b/sdks/python/expansion-service-container/Dockerfile index d3cd4a4afad3..5a5ef0f410bc 100644 --- a/sdks/python/expansion-service-container/Dockerfile +++ b/sdks/python/expansion-service-container/Dockerfile @@ -18,7 +18,7 @@ # We just need to support one Python version supported by Beam. # Picking the current default Beam Python version which is Python 3.8. -FROM python:3.8-bookworm as expansion-service +FROM python:3.9-bookworm as expansion-service LABEL Author "Apache Beam " ARG TARGETOS ARG TARGETARCH diff --git a/sdks/python/mypy.ini b/sdks/python/mypy.ini index 298f249ffbff..562cb8d56dcc 100644 --- a/sdks/python/mypy.ini +++ b/sdks/python/mypy.ini @@ -16,7 +16,7 @@ # [mypy] -python_version = 3.8 +python_version = 3.9 ignore_missing_imports = true follow_imports = normal warn_no_return = true @@ -28,6 +28,7 @@ files = apache_beam color_output = true # uncomment this to see how close we are to being complete # check_untyped_defs = true +disable_error_code = var-annotated [mypy-apache_beam.coders.proto2_coder_test_messages_pb2] ignore_errors = true diff --git a/sdks/python/test-suites/containers/tensorrt_runinference/tensor_rt.dockerfile b/sdks/python/test-suites/containers/tensorrt_runinference/tensor_rt.dockerfile index 6d99a6393fa9..c51099264e8c 100644 --- a/sdks/python/test-suites/containers/tensorrt_runinference/tensor_rt.dockerfile +++ b/sdks/python/test-suites/containers/tensorrt_runinference/tensor_rt.dockerfile @@ -22,7 +22,7 @@ ENV PATH="/usr/src/tensorrt/bin:${PATH}" WORKDIR /workspace -COPY --from=apache/beam_python3.8_sdk:latest /opt/apache/beam /opt/apache/beam +COPY --from=apache/beam_python3.9_sdk:latest /opt/apache/beam /opt/apache/beam RUN pip install --upgrade pip \ && pip install torch>=1.7.1 \ @@ -32,4 +32,4 @@ RUN pip install --upgrade pip \ && pip install cuda-python ENTRYPOINT [ "/opt/apache/beam/boot" ] -RUN apt-get update && apt-get install -y python3.8-venv +RUN apt-get update && apt-get install -y python3.9-venv diff --git a/sdks/python/test-suites/gradle.properties b/sdks/python/test-suites/gradle.properties index 3d16ee63f12a..f8c04e0f5609 100644 --- a/sdks/python/test-suites/gradle.properties +++ b/sdks/python/test-suites/gradle.properties @@ -23,13 +23,13 @@ # dataflow test-suites # (TODO): https://github.com/apache/beam/issues/21971 # Add python 3.10 to dataflow test-suites -dataflow_precommit_it_task_py_versions=3.8,3.12 -dataflow_mongodbio_it_task_py_versions=3.8 -dataflow_chicago_taxi_example_task_py_versions=3.8 +dataflow_precommit_it_task_py_versions=3.9,3.12 +dataflow_mongodbio_it_task_py_versions=3.9 +dataflow_chicago_taxi_example_task_py_versions=3.9 # TODO: Enable following tests after making sure we have enough capacity. -dataflow_validates_runner_batch_tests=3.8,3.12 -dataflow_validates_runner_streaming_tests=3.8,3.12 +dataflow_validates_runner_batch_tests=3.9,3.12 +dataflow_validates_runner_streaming_tests=3.9,3.12 dataflow_examples_postcommit_py_versions=3.12 # TFX_BSL is not yet supported on Python 3.10. dataflow_cloudml_benchmark_tests_py_versions=3.9 @@ -38,14 +38,14 @@ direct_mongodbio_it_task_py_versions=3.12 # flink runner test-suites flink_validates_runner_precommit_py_versions=3.12 -flink_validates_runner_postcommit_py_versions=3.8,3.12 -flink_examples_postcommit_py_versions=3.8,3.12 +flink_validates_runner_postcommit_py_versions=3.9,3.12 +flink_examples_postcommit_py_versions=3.9,3.12 # samza runner test-suites -samza_validates_runner_postcommit_py_versions=3.8,3.12 +samza_validates_runner_postcommit_py_versions=3.9,3.12 # spark runner test-suites -spark_examples_postcommit_py_versions=3.8,3.12 +spark_examples_postcommit_py_versions=3.9,3.12 # cross language postcommit python test suites -cross_language_validates_py_versions=3.8,3.12 +cross_language_validates_py_versions=3.9,3.12 diff --git a/sdks/python/test-suites/tox/py39/build.gradle b/sdks/python/test-suites/tox/py39/build.gradle index 5bb73b60a5d2..ea02e9d5b1e8 100644 --- a/sdks/python/test-suites/tox/py39/build.gradle +++ b/sdks/python/test-suites/tox/py39/build.gradle @@ -26,4 +26,199 @@ applyPythonNature() // Required to setup a Python 3 virtualenv and task names. pythonVersion = '3.9' +def posargs = project.findProperty("posargs") ?: "" + apply from: "../common.gradle" + +toxTask "testPy39CloudCoverage", "py39-cloudcoverage", "${posargs}" +test.dependsOn "testPy39CloudCoverage" +project.tasks.register("preCommitPyCoverage") { + dependsOn = ["testPy39CloudCoverage"] +} + +// Dep Postcommit runs test suites that evaluate compatibility of particular +// dependencies. It is exercised on a single Python version. +// +// Should still leave at least one version in PreCommit unless the marked tests +// are also exercised by existing PreCommit +// e.g. pyarrow and pandas also run on PreCommit Dataframe and Coverage +project.tasks.register("postCommitPyDep") {} + +// Create a test task for supported major versions of pyarrow +// We should have a test for the lowest supported version and +// For versions that we would like to prioritize for testing, +// for example versions released in a timeframe of last 1-2 years. + +toxTask "testPy39pyarrow-3", "py39-pyarrow-3", "${posargs}" +test.dependsOn "testPy39pyarrow-3" +postCommitPyDep.dependsOn "testPy39pyarrow-3" + +toxTask "testPy39pyarrow-9", "py39-pyarrow-9", "${posargs}" +test.dependsOn "testPy39pyarrow-9" +postCommitPyDep.dependsOn "testPy39pyarrow-9" + +toxTask "testPy39pyarrow-10", "py39-pyarrow-10", "${posargs}" +test.dependsOn "testPy39pyarrow-10" +postCommitPyDep.dependsOn "testPy39pyarrow-10" + +toxTask "testPy39pyarrow-11", "py39-pyarrow-11", "${posargs}" +test.dependsOn "testPy39pyarrow-11" +postCommitPyDep.dependsOn "testPy39pyarrow-11" + +toxTask "testPy39pyarrow-12", "py39-pyarrow-12", "${posargs}" +test.dependsOn "testPy39pyarrow-12" +postCommitPyDep.dependsOn "testPy39pyarrow-12" + +toxTask "testPy39pyarrow-13", "py39-pyarrow-13", "${posargs}" +test.dependsOn "testPy39pyarrow-13" +postCommitPyDep.dependsOn "testPy39pyarrow-13" + +toxTask "testPy39pyarrow-14", "py39-pyarrow-14", "${posargs}" +test.dependsOn "testPy39pyarrow-14" +postCommitPyDep.dependsOn "testPy39pyarrow-14" + +toxTask "testPy39pyarrow-15", "py39-pyarrow-15", "${posargs}" +test.dependsOn "testPy39pyarrow-15" +postCommitPyDep.dependsOn "testPy39pyarrow-15" + +toxTask "testPy39pyarrow-16", "py39-pyarrow-16", "${posargs}" +test.dependsOn "testPy39pyarrow-16" +postCommitPyDep.dependsOn "testPy39pyarrow-16" + +// Create a test task for each supported minor version of pandas +toxTask "testPy39pandas-14", "py39-pandas-14", "${posargs}" +test.dependsOn "testPy39pandas-14" +postCommitPyDep.dependsOn "testPy39pandas-14" + +toxTask "testPy39pandas-15", "py39-pandas-15", "${posargs}" +test.dependsOn "testPy39pandas-15" +postCommitPyDep.dependsOn "testPy39pandas-15" + +toxTask "testPy39pandas-20", "py39-pandas-20", "${posargs}" +test.dependsOn "testPy39pandas-20" +postCommitPyDep.dependsOn "testPy39pandas-20" + +// TODO(https://github.com/apache/beam/issues/31192): Add below suites +// after dependency compat tests suite switches to Python 3.9 or we add +// Python 2.2 support. + +// toxTask "testPy39pandas-21", "py39-pandas-21", "${posargs}" +// test.dependsOn "testPy39pandas-21" +// postCommitPyDep.dependsOn "testPy39pandas-21" + +// toxTask "testPy39pandas-22", "py39-pandas-22", "${posargs}" +// test.dependsOn "testPy39pandas-22" +// postCommitPyDep.dependsOn "testPy39pandas-22" + +// TODO(https://github.com/apache/beam/issues/30908): Revise what are we testing + +// Create a test task for each minor version of pytorch +toxTask "testPy39pytorch-19", "py39-pytorch-19", "${posargs}" +test.dependsOn "testPy39pytorch-19" +postCommitPyDep.dependsOn "testPy39pytorch-19" + +toxTask "testPy39pytorch-110", "py39-pytorch-110", "${posargs}" +test.dependsOn "testPy39pytorch-110" +postCommitPyDep.dependsOn "testPy39pytorch-110" + +toxTask "testPy39pytorch-111", "py39-pytorch-111", "${posargs}" +test.dependsOn "testPy39pytorch-111" +postCommitPyDep.dependsOn "testPy39pytorch-111" + +toxTask "testPy39pytorch-112", "py39-pytorch-112", "${posargs}" +test.dependsOn "testPy39pytorch-112" +postCommitPyDep.dependsOn "testPy39pytorch-112" + +toxTask "testPy39pytorch-113", "py39-pytorch-113", "${posargs}" +test.dependsOn "testPy39pytorch-113" +postCommitPyDep.dependsOn "testPy39pytorch-113" + +// run on precommit +toxTask "testPy39pytorch-200", "py39-pytorch-200", "${posargs}" +test.dependsOn "testPy39pytorch-200" +postCommitPyDep.dependsOn "testPy39pytorch-200" + +toxTask "testPy39tft-113", "py39-tft-113", "${posargs}" +test.dependsOn "testPy39tft-113" +postCommitPyDep.dependsOn "testPy39tft-113" + +// TODO(https://github.com/apache/beam/issues/25796) - uncomment onnx tox task once onnx supports protobuf 4.x.x +// Create a test task for each minor version of onnx +// toxTask "testPy39onnx-113", "py39-onnx-113", "${posargs}" +// test.dependsOn "testPy39onnx-113" +// postCommitPyDep.dependsOn "testPy39onnx-113" + +// Create a test task for each minor version of tensorflow +toxTask "testPy39tensorflow-212", "py39-tensorflow-212", "${posargs}" +test.dependsOn "testPy39tensorflow-212" +postCommitPyDep.dependsOn "testPy39tensorflow-212" + +// Create a test task for each minor version of transformers +toxTask "testPy39transformers-428", "py39-transformers-428", "${posargs}" +test.dependsOn "testPy39transformers-428" +postCommitPyDep.dependsOn "testPy39transformers-428" + +toxTask "testPy39transformers-429", "py39-transformers-429", "${posargs}" +test.dependsOn "testPy39transformers-429" +postCommitPyDep.dependsOn "testPy39transformers-429" + +toxTask "testPy39transformers-430", "py39-transformers-430", "${posargs}" +test.dependsOn "testPy39transformers-430" +postCommitPyDep.dependsOn "testPy39transformers-430" + +toxTask "testPy39embeddingsMLTransform", "py39-embeddings", "${posargs}" +test.dependsOn "testPy39embeddingsMLTransform" +postCommitPyDep.dependsOn "testPy39embeddingsMLTransform" + +// Part of MLTransform embeddings test suite but requires tensorflow hub, which we need to test on +// mutliple versions so keeping this suite separate. +toxTask "testPy39TensorflowHubEmbeddings-014", "py39-TFHubEmbeddings-014", "${posargs}" +test.dependsOn "testPy39TensorflowHubEmbeddings-014" +postCommitPyDep.dependsOn "testPy39TensorflowHubEmbeddings-014" + +toxTask "testPy39TensorflowHubEmbeddings-015", "py39-TFHubEmbeddings-015", "${posargs}" +test.dependsOn "testPy39TensorflowHubEmbeddings-015" +postCommitPyDep.dependsOn "testPy39TensorflowHubEmbeddings-015" + +toxTask "whitespacelint", "whitespacelint", "${posargs}" + +task archiveFilesToLint(type: Zip) { + archiveFileName = "files-to-whitespacelint.zip" + destinationDirectory = file("$buildDir/dist") + + from ("$rootProject.projectDir") { + include "**/*.md" + include "**/build.gradle" + include '**/build.gradle.kts' + exclude '**/build/**' // intermediate build directory + exclude 'website/www/site/themes/docsy/**' // fork to google/docsy + exclude "**/node_modules/*" + exclude "**/.gogradle/*" + } +} + +task unpackFilesToLint(type: Copy) { + from zipTree("$buildDir/dist/files-to-whitespacelint.zip") + into "$buildDir/files-to-whitespacelint" +} + +whitespacelint.dependsOn archiveFilesToLint, unpackFilesToLint +unpackFilesToLint.dependsOn archiveFilesToLint +archiveFilesToLint.dependsOn cleanPython + +toxTask "jest", "jest", "${posargs}" + +toxTask "eslint", "eslint", "${posargs}" + +task copyTsSource(type: Copy) { + from ("$rootProject.projectDir") { + include "sdks/python/apache_beam/runners/interactive/extensions/**/*" + exclude "sdks/python/apache_beam/runners/interactive/extensions/**/lib/*" + exclude "sdks/python/apache_beam/runners/interactive/extensions/**/node_modules/*" + } + into "$buildDir/ts" +} + +jest.dependsOn copyTsSource +eslint.dependsOn copyTsSource +copyTsSource.dependsOn cleanPython diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index da4cde009ce9..4f299003706a 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -17,7 +17,7 @@ [tox] # new environments will be excluded by default unless explicitly added to envlist. -envlist = py38,py39,py310,py311,py312,py38-{cloud,cloudcoverage,dask},py39-{cloud},py310-{cloud,dask},py311-{cloud,dask},py312-{cloud,dask},docs,lint,mypy,whitespacelint +envlist = py39,py310,py311,py312,py39-{cloud,cloudcoverage,dask},py310-{cloud,dask},py311-{cloud,dask},py312-{cloud,dask},docs,lint,mypy,whitespacelint toxworkdir = {toxinidir}/target/{env:ENV_NAME:.tox} [pycodestyle] @@ -67,7 +67,7 @@ commands_post = commands = false {envname} is misconfigured -[testenv:py{38,39,310,311,312}] +[testenv:py{39,310,311,312}] commands_pre = python --version pip --version @@ -77,43 +77,40 @@ commands = python apache_beam/examples/complete/autocomplete_test.py bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" -[testenv:py{38,39,310,311,312}-macos] +[testenv:py{39,310,311,312}-macos] commands_pre = python --version pip --version # pip check bash {toxinidir}/scripts/run_tox_cleanup.sh -commands = - python apache_beam/examples/complete/autocomplete_test.py - bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" -[testenv:py{38,39,310,311,312}-win] +[testenv:py{39,310,311,312}-win] commands = python apache_beam/examples/complete/autocomplete_test.py bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" install_command = {envbindir}/python.exe {envbindir}/pip.exe install --retries 10 {opts} {packages} list_dependencies_command = {envbindir}/python.exe {envbindir}/pip.exe freeze -[testenv:py{38,39,310,311,312}-cloud] +[testenv:py{39,310,311,312}-cloud] ; extras = test,gcp,interactive,dataframe,aws,azure extras = test,gcp,interactive,dataframe,aws,azure commands = python apache_beam/examples/complete/autocomplete_test.py bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" -[testenv:py{38,39,310,311,312}-ml] +[testenv:py{39,310,311,312}-ml] # Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests. setenv = extras = test,gcp,dataframe,ml_test commands = bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" -[testenv:py{38,39,310,311,312}-dask] +[testenv:py{39,310,311,312}-dask] extras = test,dask commands = bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" -[testenv:py38-cloudcoverage] +[testenv:py39-cloudcoverage] deps = pytest-cov==3.0.0 # Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests. @@ -286,7 +283,7 @@ commands = bash {toxinidir}/scripts/pytest_validates_runner.sh {envname} {toxinidir}/apache_beam/runners/portability/spark_runner_test.py {posargs} -[testenv:py{38,39}-pyarrow-{3,9,10,11,12,13,14,15,16}] +[testenv:py{39,310}-pyarrow-{3,9,10,11,12,13,14,15,16}] deps = # As a courtesy to users, test against the oldest allowed version of Pyarrow. # We'd have to increase the pyarrow lower bound when Python 3.9 is deprecated. @@ -311,7 +308,7 @@ commands = /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pyarrow {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{38,39}-pandas-{14,15,20}] +[testenv:py{39,310}-pandas-{14,15,20}] deps = 14: pandas>=1.4.3,<1.5.0 # Exclude 1.5.0 and 1.5.1 because of https://github.com/pandas-dev/pandas/issues/45725 @@ -324,7 +321,7 @@ commands = # Run all DataFrame API unit tests bash {toxinidir}/scripts/run_pytest.sh {envname} 'apache_beam/dataframe' -[testenv:py{38,39}-tft-{113,114}] +[testenv:py{39,310}-tft-{113,114}] deps = # Help pip resolve conflict with typing-extensions due to an old version of tensorflow https://github.com/apache/beam/issues/30852 113: pydantic<2.0 @@ -332,7 +329,7 @@ deps = commands = bash {toxinidir}/scripts/run_pytest.sh {envname} 'apache_beam/ml/transforms apache_beam/examples/snippets/transforms/elementwise/mltransform_test.py' -[testenv:py{38,39}-pytorch-{19,110,111,112,113}] +[testenv:py{39,310}-pytorch-{19,110,111,112,113}] deps = 19: torch>=1.9.0,<1.10.0 110: torch>=1.10.0,<1.11.0 @@ -349,7 +346,7 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{38,39}-pytorch-200] +[testenv:py{39,310}-pytorch-200] deps = 200: torch>=2.0.0,<2.1.0 @@ -365,7 +362,7 @@ commands = /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' # TODO(https://github.com/apache/beam/issues/25796) - uncomment onnx tox task in tox/py38/build.gradle once onnx supports protobuf 4.x.x -[testenv:py{38,39}-onnx-113] +[testenv:py{39,310}-onnx-113] # TODO(https://github.com/apache/beam/issues/25443) # apparently tox has problem when substitution key has single value. Change back to -onnx-{113,...} # when multiple onnx versions are tested. @@ -384,7 +381,7 @@ commands = # Run all ONNX unit tests pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_onnx {posargs} -[testenv:py{38,39}-tensorflow-212] +[testenv:py39-tensorflow-212] deps = 212: tensorflow>=2.12rc1,<2.13 # Help pip resolve conflict with typing-extensions for old version of TF https://github.com/apache/beam/issues/30852 @@ -397,7 +394,7 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_tf {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{38,39}-xgboost-{160,170}] +[testenv:py39-xgboost-{160,170}] deps = 160: xgboost>=1.6.0,<1.7.0 @@ -413,7 +410,7 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_xgboost {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{38,39}-transformers-{428,429,430}] +[testenv:py{39,310}-transformers-{428,429,430}] deps = 428: transformers>=4.28.0,<4.29.0 429: transformers>=4.29.0,<4.30.0 @@ -430,7 +427,7 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_transformers {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{38,311}-vertex-ai] +[testenv:py{39,312}-vertex-ai] deps = tensorflow==2.12.0 extras = test,gcp @@ -443,7 +440,7 @@ commands = /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_vertex_ai {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{38,39}-embeddings] +[testenv:py{39,310}-embeddings] deps = sentence-transformers==2.2.2 passenv = HF_INFERENCE_TOKEN @@ -456,7 +453,7 @@ commands = /bin/sh -c 'pytest apache_beam/ml/transforms/embeddings -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{38,39}-TFHubEmbeddings-{014,015}] +[testenv:py{39,310}-TFHubEmbeddings-{014,015}] deps = 014: tensorflow-hub>=0.14.0,<0.15.0 # Help pip resolve conflict with typing-extensions due to an old version of tensorboard https://github.com/apache/beam/issues/30852 diff --git a/sdks/standard_external_transforms.yaml b/sdks/standard_external_transforms.yaml index 05aa3c9b9de5..3c8c09ceb361 100644 --- a/sdks/standard_external_transforms.yaml +++ b/sdks/standard_external_transforms.yaml @@ -19,7 +19,7 @@ # configuration in /sdks/standard_expansion_services.yaml. # Refer to gen_xlang_wrappers.py for more info. # -# Last updated on: 2024-04-18 +# Last updated on: 2024-08-27 - default_service: sdks:java:io:expansion-service:shadowJar description: 'Outputs a PCollection of Beam Rows, each containing a single INT64 @@ -43,8 +43,7 @@ description: Specifies the rate to generate a given number of elements per a given number of seconds. Applicable only to unbounded sequences. nullable: true - type: Row(elements=, seconds=typing.Union[numpy.int64, - NoneType]) + type: Row(elements=, seconds=typing.Optional[numpy.int64]) start: description: The minimum number to generate (inclusive). nullable: false