Add pyproject.toml (#28385)

* Add pyproject.toml

Update numpy bounds

* Use subprocess to run grpcio since it is not imported in pyproject.toml

Update setup.py

Remove ImportError from gen_protos.py

Update subprocess run and raise RuntimeError if proto generation fails

Print output of setup.py
Fix linting issues

* Remove build-requirements.txt and use build to build the sdist
Modify buildPython task

* Use wheels to run tox precommit tests

Upgrade pip in Dockerfile

Move _ELEMENTS to shared file. tests are not importable by each other


Add missing element

Remove shared_test_variables

Remove installing wheel in a test suite


Retry run_tox.sh with no installPkg flag

Remove natural language test. codepath is covered in the postCommits.

Add back tox exit code

* Remove cython tests. default tests will run with Cython extensions

FIx toxTask name

Add no-extra test suite to precommit and remove GH duplicate ubuntu test

Skip failing non-cython test

Fix tox test name

* Force type cast inputs to list

* Update stager to use build. If it fails, use legacy setup to build sdist

Fix mypy issue

* Remove cython env and build-requirements for tox.ini

Author: AnandInguva

.github/workflows/beam_PreCommit_Python.yml

@@ -97,6 +97,7 @@ jobs:
           arguments: |
             -Pposargs="--ignore=apache_beam/dataframe/ --ignore=apache_beam/examples/ --ignore=apache_beam/runners/ --ignore=apache_beam/transforms/" \
             -PpythonVersion=${{ matrix.python_version }} \
+            -PuseWheelDistribution
       - name: Archive code coverage results
         uses: actions/upload-artifact@v3
         with:

.github/workflows/beam_PreCommit_Python_Coverage.yml

@@ -86,6 +86,8 @@ jobs:
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
           gradle-command: :sdks:python:test-suites:tox:py38:preCommitPyCoverage
+          arguments: |
+            -PuseWheelDistribution
       - name: Archive code coverage results
         uses: actions/upload-artifact@v3
         with:

.github/workflows/beam_PreCommit_Python_Dataframes.yml

@@ -97,6 +97,7 @@ jobs:
           arguments: |
             -Pposargs=apache_beam/dataframe/ \
             -PpythonVersion=${{ matrix.python_version }} \
+            -PuseWheelDistribution
       - name: Archive code coverage results
         uses: actions/upload-artifact@v3
         with:

.github/workflows/beam_PreCommit_Python_Examples.yml

@@ -97,6 +97,7 @@ jobs:
           arguments: |
             -Pposargs=apache_beam/examples/ \
             -PpythonVersion=${{ matrix.python_version }} \
+            -PuseWheelDistribution
       - name: Archive code coverage results
         uses: actions/upload-artifact@v3
         with:

.github/workflows/beam_PreCommit_Python_Runners.yml

@@ -97,6 +97,7 @@ jobs:
           arguments: |
             -Pposargs=apache_beam/runners/ \
             -PpythonVersion=${{ matrix.python_version }} \
+            -PuseWheelDistribution
       - name: Archive code coverage results
         uses: actions/upload-artifact@v3
         with:

.github/workflows/beam_PreCommit_Python_Transforms.yml

@@ -97,6 +97,7 @@ jobs:
           arguments: |
             -Pposargs=apache_beam/transforms/ \
             -PpythonVersion=${{ matrix.python_version }} \
+            -PuseWheelDistribution
       - name: Archive code coverage results
         uses: actions/upload-artifact@v3
         with:

.github/workflows/build_release_candidate.yml

@@ -314,7 +314,7 @@ jobs:
             pip install -U pip
             pip install tox
             # TODO(https://github.com/apache/beam/issues/20209): Don't hardcode py version in this file.
-            pip install -r build-requirements.txt && tox -e py38-docs
+            tox -e py38-docs
             rm -rf target/docs/_build/.doctrees
       - name: Build Typescript Docs
         working-directory: beam/sdks/typescript

.github/workflows/build_wheels.yml

@@ -92,11 +92,6 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: 3.8
-      - name: Get build dependencies
-        working-directory: ./sdks/python
-        run: python -m pip install -r build-requirements.txt
-      - name: Install wheels
-        run: python -m pip install wheel
       - name: Get tag
         id: get_tag
         run: |
@@ -117,15 +112,15 @@ jobs:
           echo "RELEASE_VERSION=$RELEASE_VERSION" >> $GITHUB_OUTPUT
       - name: Build source
         working-directory: ./sdks/python
-        run: python setup.py sdist --formats=zip
+        run: pip install -U build && python -m build --sdist
       - name: Add checksums
         working-directory: ./sdks/python/dist
         run: |
-          file=$(ls | grep .zip | head -n 1)
+          file=$(ls | grep .tar.gz | head -n 1)
           sha512sum $file > ${file}.sha512
       - name: Unzip source
         working-directory: ./sdks/python
-        run: unzip dist/$(ls dist | grep .zip | head -n 1)
+        run: tar -xzvf dist/$(ls dist | grep .tar.gz | head -n 1)
       - name: Rename source directory
         working-directory: ./sdks/python
         run: mv $(ls | grep apache-beam) apache-beam-source
@@ -155,17 +150,17 @@ jobs:
       - name: Build RC source
         if: steps.is_rc.outputs.is_rc == 1
         working-directory: ./sdks/python
-        run: python setup.py sdist --formats=zip
+        run: pip install -U build && pythom -m build --sdist
       - name: Add RC checksums
         if: steps.is_rc.outputs.is_rc == 1
         working-directory: ./sdks/python/dist
         run: |
-          file=$(ls | grep .zip | head -n 1)
+          file=$(ls | grep .tar.gz | head -n 1)
           sha512sum $file > ${file}.sha512
       - name: Unzip RC source
         if: steps.is_rc.outputs.is_rc == 1
         working-directory: ./sdks/python
-        run: unzip dist/$(ls dist | grep .zip | head -n 1)
+        run: tar -xzvf dist/$(ls dist | grep .tar.gz | head -n 1)
       - name: Rename RC source directory
         if: steps.is_rc.outputs.is_rc == 1
         working-directory: ./sdks/python

.github/workflows/dask_runner_tests.yml

@@ -44,12 +44,9 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: 3.8
-      - name: Get build dependencies
-        working-directory: ./sdks/python
-        run: pip install pip setuptools --upgrade && pip install -r build-requirements.txt
       - name: Build source
         working-directory: ./sdks/python
-        run: python setup.py sdist
+        run: pip install -U build && python -m build --sdist
       - name: Rename source file
         working-directory: ./sdks/python/dist
         run: mv $(ls | grep "apache-beam.*tar\.gz") apache-beam-source.tar.gz
@@ -78,9 +75,6 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.params.py_ver }}
-      - name: Get build dependencies
-        working-directory: ./sdks/python
-        run: pip install -r build-requirements.txt
       - name: Install tox
         run: pip install tox
       - name: Install SDK with dask

.github/workflows/python_dependency_tests.yml

@@ -7,7 +7,7 @@ on:
     branches: ['master', 'release-*']
     tags: 'v*'
     # paths where Beam Python's dependencies are configured.
-    paths: ['sdks/python/setup.py', 'sdks/python/build-requirements.txt', 'sdks/python/container/base_image_requirements_manual.txt']
+    paths: ['sdks/python/setup.py', 'sdks/python/pyproject.toml', 'sdks/python/container/base_image_requirements_manual.txt']
 
 # This allows a subsequently queued workflow run to interrupt previous runs
 concurrency:
@@ -38,9 +38,6 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.params.py_ver }}
-      - name: Install Build dependencies
-        working-directory: ./sdks/python
-        run: pip install -r build-requirements.txt
       - name: Install base_image_requirements.txt
         working-directory: ./sdks/python
         run: pip install --no-deps -r container/${{ matrix.params.py_env }}/base_image_requirements.txt