Add pyproject.toml (apache#28385)

* Add pyproject.toml Update numpy bounds * Use subprocess to run grpcio since it is not imported in pyproject.toml Update setup.py Remove ImportError from gen_protos.py Update subprocess run and raise RuntimeError if proto generation fails Print output of setup.py Fix linting issues * Remove build-requirements.txt and use build to build the sdist Modify buildPython task * Use wheels to run tox precommit tests Upgrade pip in Dockerfile Move _ELEMENTS to shared file. tests are not importable by each other Add missing element Remove shared_test_variables Remove installing wheel in a test suite Retry run_tox.sh with no installPkg flag Remove natural language test. codepath is covered in the postCommits. Add back tox exit code * Remove cython tests. default tests will run with Cython extensions FIx toxTask name Add no-extra test suite to precommit and remove GH duplicate ubuntu test Skip failing non-cython test Fix tox test name * Force type cast inputs to list * Update stager to use build. If it fails, use legacy setup to build sdist Fix mypy issue * Remove cython env and build-requirements for tox.ini
damondouglas · Oct 16, 2023 · d64acf5 · d64acf5
1 parent c846eda
commit d64acf5
Show file tree

Hide file tree

Showing 47 changed files with 268 additions and 286 deletions.
diff --git a/.github/workflows/beam_PreCommit_Python.yml b/.github/workflows/beam_PreCommit_Python.yml
@@ -97,6 +97,7 @@ jobs:
           arguments: |
             -Pposargs="--ignore=apache_beam/dataframe/ --ignore=apache_beam/examples/ --ignore=apache_beam/runners/ --ignore=apache_beam/transforms/" \
             -PpythonVersion=${{ matrix.python_version }} \
+            -PuseWheelDistribution
       - name: Archive code coverage results
         uses: actions/upload-artifact@v3
         with:

diff --git a/.github/workflows/beam_PreCommit_Python_Coverage.yml b/.github/workflows/beam_PreCommit_Python_Coverage.yml
@@ -86,6 +86,8 @@ jobs:
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
           gradle-command: :sdks:python:test-suites:tox:py38:preCommitPyCoverage
+          arguments: |
+            -PuseWheelDistribution
       - name: Archive code coverage results
         uses: actions/upload-artifact@v3
         with:

diff --git a/.github/workflows/beam_PreCommit_Python_Dataframes.yml b/.github/workflows/beam_PreCommit_Python_Dataframes.yml
@@ -97,6 +97,7 @@ jobs:
           arguments: |
             -Pposargs=apache_beam/dataframe/ \
             -PpythonVersion=${{ matrix.python_version }} \
+            -PuseWheelDistribution
       - name: Archive code coverage results
         uses: actions/upload-artifact@v3
         with:

diff --git a/.github/workflows/beam_PreCommit_Python_Examples.yml b/.github/workflows/beam_PreCommit_Python_Examples.yml
@@ -97,6 +97,7 @@ jobs:
           arguments: |
             -Pposargs=apache_beam/examples/ \
             -PpythonVersion=${{ matrix.python_version }} \
+            -PuseWheelDistribution
       - name: Archive code coverage results
         uses: actions/upload-artifact@v3
         with:

diff --git a/.github/workflows/beam_PreCommit_Python_Runners.yml b/.github/workflows/beam_PreCommit_Python_Runners.yml
@@ -97,6 +97,7 @@ jobs:
           arguments: |
             -Pposargs=apache_beam/runners/ \
             -PpythonVersion=${{ matrix.python_version }} \
+            -PuseWheelDistribution
       - name: Archive code coverage results
         uses: actions/upload-artifact@v3
         with:

diff --git a/.github/workflows/beam_PreCommit_Python_Transforms.yml b/.github/workflows/beam_PreCommit_Python_Transforms.yml
@@ -97,6 +97,7 @@ jobs:
           arguments: |
             -Pposargs=apache_beam/transforms/ \
             -PpythonVersion=${{ matrix.python_version }} \
+            -PuseWheelDistribution
       - name: Archive code coverage results
         uses: actions/upload-artifact@v3
         with:

diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml
@@ -314,7 +314,7 @@ jobs:
             pip install -U pip
             pip install tox
             # TODO(https://github.com/apache/beam/issues/20209): Don't hardcode py version in this file.
-            pip install -r build-requirements.txt && tox -e py38-docs
+            tox -e py38-docs
             rm -rf target/docs/_build/.doctrees
       - name: Build Typescript Docs
         working-directory: beam/sdks/typescript

diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
@@ -92,11 +92,6 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: 3.8
-      - name: Get build dependencies
-        working-directory: ./sdks/python
-        run: python -m pip install -r build-requirements.txt
-      - name: Install wheels
-        run: python -m pip install wheel
       - name: Get tag
         id: get_tag
         run: |
@@ -117,15 +112,15 @@ jobs:
           echo "RELEASE_VERSION=$RELEASE_VERSION" >> $GITHUB_OUTPUT
       - name: Build source
         working-directory: ./sdks/python
-        run: python setup.py sdist --formats=zip
+        run: pip install -U build && python -m build --sdist
       - name: Add checksums
         working-directory: ./sdks/python/dist
         run: |
-          file=$(ls | grep .zip | head -n 1)
+          file=$(ls | grep .tar.gz | head -n 1)
           sha512sum $file > ${file}.sha512
       - name: Unzip source
         working-directory: ./sdks/python
-        run: unzip dist/$(ls dist | grep .zip | head -n 1)
+        run: tar -xzvf dist/$(ls dist | grep .tar.gz | head -n 1)
       - name: Rename source directory
         working-directory: ./sdks/python
         run: mv $(ls | grep apache-beam) apache-beam-source
@@ -155,17 +150,17 @@ jobs:
       - name: Build RC source
         if: steps.is_rc.outputs.is_rc == 1
         working-directory: ./sdks/python
-        run: python setup.py sdist --formats=zip
+        run: pip install -U build && pythom -m build --sdist
       - name: Add RC checksums
         if: steps.is_rc.outputs.is_rc == 1
         working-directory: ./sdks/python/dist
         run: |
-          file=$(ls | grep .zip | head -n 1)
+          file=$(ls | grep .tar.gz | head -n 1)
           sha512sum $file > ${file}.sha512
       - name: Unzip RC source
         if: steps.is_rc.outputs.is_rc == 1
         working-directory: ./sdks/python
-        run: unzip dist/$(ls dist | grep .zip | head -n 1)
+        run: tar -xzvf dist/$(ls dist | grep .tar.gz | head -n 1)
       - name: Rename RC source directory
         if: steps.is_rc.outputs.is_rc == 1
         working-directory: ./sdks/python

diff --git a/.github/workflows/dask_runner_tests.yml b/.github/workflows/dask_runner_tests.yml
@@ -44,12 +44,9 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: 3.8
-      - name: Get build dependencies
-        working-directory: ./sdks/python
-        run: pip install pip setuptools --upgrade && pip install -r build-requirements.txt
       - name: Build source
         working-directory: ./sdks/python
-        run: python setup.py sdist
+        run: pip install -U build && python -m build --sdist
       - name: Rename source file
         working-directory: ./sdks/python/dist
         run: mv $(ls | grep "apache-beam.*tar\.gz") apache-beam-source.tar.gz
@@ -78,9 +75,6 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.params.py_ver }}
-      - name: Get build dependencies
-        working-directory: ./sdks/python
-        run: pip install -r build-requirements.txt
       - name: Install tox
         run: pip install tox
       - name: Install SDK with dask

diff --git a/.github/workflows/python_dependency_tests.yml b/.github/workflows/python_dependency_tests.yml
@@ -7,7 +7,7 @@ on:
     branches: ['master', 'release-*']
     tags: 'v*'
     # paths where Beam Python's dependencies are configured.
-    paths: ['sdks/python/setup.py', 'sdks/python/build-requirements.txt', 'sdks/python/container/base_image_requirements_manual.txt']
+    paths: ['sdks/python/setup.py', 'sdks/python/pyproject.toml', 'sdks/python/container/base_image_requirements_manual.txt']
 
 # This allows a subsequently queued workflow run to interrupt previous runs
 concurrency:
@@ -38,9 +38,6 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.params.py_ver }}
-      - name: Install Build dependencies
-        working-directory: ./sdks/python
-        run: pip install -r build-requirements.txt
       - name: Install base_image_requirements.txt
         working-directory: ./sdks/python
         run: pip install --no-deps -r container/${{ matrix.params.py_env }}/base_image_requirements.txt

diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml
@@ -78,12 +78,9 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: 3.8
-      - name: Get build dependencies
-        working-directory: ./sdks/python
-        run: pip install pip setuptools --upgrade && pip install -r build-requirements.txt
       - name: Build source
         working-directory: ./sdks/python
-        run: python setup.py sdist
+        run: pip install -U build && python -m build --sdist
       - name: Rename source file
         working-directory: ./sdks/python/dist
         run: mv $(ls | grep "apache-beam.*tar\.gz") apache-beam-source.tar.gz
@@ -99,7 +96,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [macos-latest, windows-latest]
         params: [
           {"py_ver": "3.8", "tox_env": "py38"},
           {"py_ver": "3.9", "tox_env": "py39"},
@@ -113,9 +110,6 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.params.py_ver }}
-      - name: Get build dependencies
-        working-directory: ./sdks/python
-        run: pip install -r build-requirements.txt --use-pep517
       - name: Install tox
         run: pip install tox
       - name: Run tests basic unix
@@ -148,9 +142,6 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
-      - name: Get build dependencies
-        working-directory: ./sdks/python
-        run: pip install -r build-requirements.txt
       - name: Install requirements
         working-directory: ./sdks/python
         run: pip install setuptools --upgrade && pip install -e .
@@ -192,9 +183,6 @@ jobs:
           service_account_key: ${{ secrets.GCP_SA_KEY }}
           project_id: ${{ secrets.GCP_PROJECT_ID }}
           export_default_credentials: true
-      - name: Get build dependencies
-        working-directory: ./sdks/python
-        run: pip install -r build-requirements.txt
       - name: Install requirements
         working-directory: ./sdks/python
         run: pip install setuptools --upgrade && pip install -e ".[gcp]"

diff --git a/.github/workflows/run_perf_alert_tool.yml b/.github/workflows/run_perf_alert_tool.yml
@@ -46,9 +46,6 @@ jobs:
         with:
           service_account_key: ${{ secrets.GCP_SA_KEY }}
           export_default_credentials: true
-      - name: Get Apache Beam Build dependencies
-        working-directory: ./sdks/python
-        run: pip install pip setuptools --upgrade && pip install -r build-requirements.txt
       - name: Install Apache Beam
         working-directory: ./sdks/python
         run: pip install -e .[gcp,test]

diff --git a/.github/workflows/typescript_tests.yml b/.github/workflows/typescript_tests.yml
@@ -89,10 +89,8 @@ jobs:
       - name: Setup Beam Python
         working-directory: ./sdks/python
         run: |
-          pip install pip setuptools --upgrade
-          pip install -r build-requirements.txt
           pip install 'pandas>=1.0,<1.5'
-          python setup.py develop
+          pip install -e .
       - run: npm ci
         working-directory: ./sdks/typescript
       - run: npm run build
@@ -146,10 +144,7 @@ jobs:
       - name: Setup Beam Python
         working-directory: ./sdks/python
         run: |
-          pip install pip setuptools --upgrade
-          pip install -r build-requirements.txt
           pip install 'pandas>=1.0,<1.5'
-          python setup.py develop
           pip install -e ".[gcp]"
       - name: Authenticate on GCP
         uses: google-github-actions/setup-gcloud@v0

diff --git a/.test-infra/jenkins/job_PreCommit_Python.groovy b/.test-infra/jenkins/job_PreCommit_Python.groovy
@@ -29,6 +29,7 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
       '^release/.*$',
     ],
     gradleSwitches: [
+      '-PuseWheelDistribution',
       '-Pposargs=\"--ignore=apache_beam/dataframe/ --ignore=apache_beam/examples/ --ignore=apache_beam/runners/ --ignore=apache_beam/transforms/\"' // All these tests are covered by different jobs.
     ],
     numBuildsToRetain: 40

diff --git a/.test-infra/jenkins/job_PreCommit_Python_Coverage.groovy b/.test-infra/jenkins/job_PreCommit_Python_Coverage.groovy
@@ -22,6 +22,9 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
     scope: this,
     nameBase: 'Python_Coverage',
     gradleTask: ':sdks:python:test-suites:tox:py38:preCommitPyCoverage',
+    gradleSwitches: [
+      '-PuseWheelDistribution'
+    ],
     timeoutMins: 180,
     triggerPathPatterns: [
       '^model/.*$',

diff --git a/.test-infra/jenkins/job_PreCommit_Python_Dataframes.groovy b/.test-infra/jenkins/job_PreCommit_Python_Dataframes.groovy
@@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
     nameBase: 'Python_Dataframes',
     gradleTask: ':pythonPreCommit',
     gradleSwitches: [
-      '-Pposargs=apache_beam/dataframe/'
+      '-Pposargs=apache_beam/dataframe/',
+      '-PuseWheelDistribution'
     ],
     timeoutMins: 180,
     triggerPathPatterns: [

diff --git a/.test-infra/jenkins/job_PreCommit_Python_Examples.groovy b/.test-infra/jenkins/job_PreCommit_Python_Examples.groovy
@@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
     nameBase: 'Python_Examples',
     gradleTask: ':pythonPreCommit',
     gradleSwitches: [
-      '-Pposargs=apache_beam/examples/'
+      '-Pposargs=apache_beam/examples/',
+      '-PuseWheelDistribution'
     ],
     timeoutMins: 180,
     triggerPathPatterns: [

diff --git a/.test-infra/jenkins/job_PreCommit_Python_Runners.groovy b/.test-infra/jenkins/job_PreCommit_Python_Runners.groovy
@@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
     nameBase: 'Python_Runners',
     gradleTask: ':pythonPreCommit',
     gradleSwitches: [
-      '-Pposargs=apache_beam/runners/'
+      '-Pposargs=apache_beam/runners/',
+      '-PuseWheelDistribution'
     ],
     timeoutMins: 180,
     triggerPathPatterns: [

diff --git a/.test-infra/jenkins/job_PreCommit_Python_Transforms.groovy b/.test-infra/jenkins/job_PreCommit_Python_Transforms.groovy
@@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
     nameBase: 'Python_Transforms',
     gradleTask: ':pythonPreCommit',
     gradleSwitches: [
-      '-Pposargs=apache_beam/transforms/'
+      '-Pposargs=apache_beam/transforms/',
+      '-PuseWheelDistribution'
     ],
     timeoutMins: 180,
     triggerPathPatterns: [

diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -2998,7 +2998,7 @@ class BeamModulePlugin implements Plugin<Project> {
             executable 'sh'
             args '-c', ". ${project.ext.envdir}/bin/activate && " +
                 "pip install --pre --retries 10 --upgrade pip && " +
-                "pip install --pre --retries 10 --upgrade tox -r ${project.rootDir}/sdks/python/build-requirements.txt"
+                "pip install --pre --retries 10 --upgrade tox"
           }
         }
         // Gradle will delete outputs whenever it thinks they are stale. Putting a
@@ -3081,30 +3081,40 @@ class BeamModulePlugin implements Plugin<Project> {
         }
         return argList.join(' ')
       }
-
       project.ext.toxTask = { name, tox_env, posargs='' ->
         project.tasks.register(name) {
           dependsOn setupVirtualenv
           dependsOn ':sdks:python:sdist'
-
-          doLast {
-            // Python source directory is also tox execution workspace, We want
-            // to isolate them per tox suite to avoid conflict when running
-            // multiple tox suites in parallel.
-            project.copy { from project.pythonSdkDeps; into copiedSrcRoot }
-
-            def copiedPyRoot = "${copiedSrcRoot}/sdks/python"
-            def distTarBall = "${pythonRootDir}/build/apache-beam.tar.gz"
-            project.exec {
-              executable 'sh'
-              args '-c', ". ${project.ext.envdir}/bin/activate && cd ${copiedPyRoot} && scripts/run_tox.sh $tox_env $distTarBall '$posargs'"
+          if (project.hasProperty('useWheelDistribution')) {
+            def pythonVersionNumber  = project.ext.pythonVersion.replace('.', '')
+            dependsOn ":sdks:python:bdistPy${pythonVersionNumber}linux"
+            doLast {
+              project.copy { from project.pythonSdkDeps; into copiedSrcRoot }
+              def copiedPyRoot = "${copiedSrcRoot}/sdks/python"
+              def collection = project.fileTree(project.project(':sdks:python').buildDir){
+                include "**/apache_beam-*cp${pythonVersionNumber}*manylinux*.whl"
+              }
+              String packageFilename = collection.singleFile.toString()
+              project.exec {
+                executable 'sh'
+                args '-c', ". ${project.ext.envdir}/bin/activate && cd ${copiedPyRoot} && scripts/run_tox.sh $tox_env ${packageFilename} '$posargs' "
+              }
+            }
+          } else {
+            // tox task will run in editable mode, which is configured in the tox.ini file.
+            doLast {
+              project.copy { from project.pythonSdkDeps; into copiedSrcRoot }
+              def copiedPyRoot = "${copiedSrcRoot}/sdks/python"
+              project.exec {
+                executable 'sh'
+                args '-c', ". ${project.ext.envdir}/bin/activate && cd ${copiedPyRoot} && scripts/run_tox.sh $tox_env '$posargs'"
+              }
             }
           }
           inputs.files project.pythonSdkDeps
           outputs.files project.fileTree(dir: "${pythonRootDir}/target/.tox/${tox_env}/log/")
         }
       }
-
       // Run single or a set of integration tests with provided test options and pipeline options.
       project.ext.enablePythonPerformanceTest = {
 

diff --git a/release/src/main/scripts/build_release_candidate.sh b/release/src/main/scripts/build_release_candidate.sh
@@ -346,7 +346,7 @@ if [[ $confirmation = "y" ]]; then
   cd ${BEAM_ROOT_DIR}
   RELEASE_COMMIT=$(git rev-list -n 1 "tags/${RC_TAG}")
   # TODO(https://github.com/apache/beam/issues/20209): Don't hardcode py version in this file.
-  cd sdks/python && pip install -r build-requirements.txt && tox -e py38-docs
+  cd sdks/python && tox -e py38-docs
   GENERATED_PYDOC=~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_PYTHON_DOC}/${BEAM_ROOT_DIR}/sdks/python/target/docs/_build
   rm -rf ${GENERATED_PYDOC}/.doctrees
 

diff --git a/sdks/python/apache_beam/coders/slow_coders_test.py b/sdks/python/apache_beam/coders/slow_coders_test.py
@@ -25,6 +25,9 @@
 from apache_beam.coders.coders_test_common import *
 
 
+@unittest.skip(
+    'Remove non-cython tests.'
+    'https://github.com/apache/beam/issues/28307')
 class SlowCoders(unittest.TestCase):
   def test_using_slow_impl(self):
     try:

diff --git a/sdks/python/apache_beam/examples/inference/README.md b/sdks/python/apache_beam/examples/inference/README.md
@@ -29,7 +29,6 @@ Some examples are also used in [our benchmarks](http://s.apache.org/beam-communi
 You must have the latest (possibly unreleased) `apache-beam` or greater installed from the Beam repo in order to run these pipelines,
 because some examples rely on the latest features that are actively in development. To install Beam, run the following from the `sdks/python` directory:
 ```
-pip install -r build-requirements.txt
 pip install -e .[gcp]
 ```
 

diff --git a/sdks/python/apache_beam/examples/kafkataxi/README.md b/sdks/python/apache_beam/examples/kafkataxi/README.md
@@ -157,9 +157,9 @@ Install Beam and dependencies and build a Beam distribution.
 
 ```sh
 cd beam/sdks/python
-pip install -r build-requirements.txt
 pip install -e '.[gcp]'
-python setup.py sdist
+pip install -q build
+python -m build --sdist
 ```
 
 Run the Beam pipeline. You can either use the default Kafka topic name or specify