Bump Pyarrow version to include 14.0.0 (#29536)

* generate requirements file * update setup.py pyarrow * Update pyarrow except for windows * Skip tests for version 13 or greater. * Add reason for unittest skipIf * Fix pyarrow lower bound * Update sdks/python/setup.py Co-authored-by: tvalentyn <[email protected]> --------- Co-authored-by: tvalentyn <[email protected]>
apache · Dec 3, 2023 · f7ae6b4 · f7ae6b4
1 parent 6593a0f
commit f7ae6b4
Show file tree

Hide file tree

Showing 6 changed files with 173 additions and 158 deletions.
diff --git a/sdks/python/apache_beam/io/parquetio_test.py b/sdks/python/apache_beam/io/parquetio_test.py
@@ -334,6 +334,9 @@ def test_write_batched_display_data(self):
     ]
     hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))
 
+  @unittest.skipIf(
+      ARROW_MAJOR_VERSION >= 13,
+      'pyarrow 13.x and above does not throw ArrowInvalid error')
   def test_sink_transform_int96(self):
     with tempfile.NamedTemporaryFile() as dst:
       path = dst.name

diff --git a/sdks/python/container/py310/base_image_requirements.txt b/sdks/python/container/py310/base_image_requirements.txt
@@ -25,123 +25,123 @@ attrs==23.1.0
 beautifulsoup4==4.12.2
 bs4==0.0.1
 build==1.0.3
-cachetools==5.3.1
-certifi==2023.7.22
+cachetools==5.3.2
+certifi==2023.11.17
 cffi==1.16.0
-charset-normalizer==3.3.0
+charset-normalizer==3.3.2
 click==8.1.7
 cloudpickle==2.2.1
 crcmod==1.7
-cryptography==41.0.4
+cryptography==41.0.6
 Cython==0.29.36
 deprecation==2.1.0
 dill==0.3.1.1
 dnspython==2.4.2
 docker==6.1.3
 docopt==0.6.2
-exceptiongroup==1.1.3
+exceptiongroup==1.2.0
 execnet==2.0.2
-fastavro==1.8.4
+fastavro==1.9.0
 fasteners==0.19
 freezegun==1.2.2
 future==0.18.3
-google-api-core==2.12.0
-google-api-python-client==2.104.0
+google-api-core==2.14.0
+google-api-python-client==2.108.0
 google-apitools==0.5.31
-google-auth==2.23.3
+google-auth==2.23.4
 google-auth-httplib2==0.1.1
-google-cloud-aiplatform==1.35.0
-google-cloud-bigquery==3.12.0
+google-cloud-aiplatform==1.36.4
+google-cloud-bigquery==3.13.0
 google-cloud-bigquery-storage==2.22.0
 google-cloud-bigtable==2.21.0
 google-cloud-core==2.3.3
 google-cloud-datastore==2.18.0
-google-cloud-dlp==3.12.3
+google-cloud-dlp==3.13.0
 google-cloud-language==2.11.1
 google-cloud-profiler==4.1.0
 google-cloud-pubsub==2.18.4
 google-cloud-pubsublite==1.8.3
 google-cloud-recommendations-ai==0.10.5
 google-cloud-resource-manager==1.10.4
 google-cloud-spanner==3.40.1
-google-cloud-storage==2.12.0
+google-cloud-storage==2.13.0
 google-cloud-videointelligence==2.11.4
 google-cloud-vision==3.4.5
 google-crc32c==1.5.0
 google-resumable-media==2.6.0
 googleapis-common-protos==1.61.0
-greenlet==3.0.0
-grpc-google-iam-v1==0.12.6
-grpcio==1.59.0
-grpcio-status==1.59.0
-guppy3==3.1.4
+greenlet==3.0.1
+grpc-google-iam-v1==0.12.7
+grpcio==1.59.3
+grpcio-status==1.59.3
+guppy3==3.1.4.post1
 hdfs==2.7.3
 httplib2==0.22.0
-hypothesis==6.88.1
-idna==3.4
+hypothesis==6.91.0
+idna==3.6
 iniconfig==2.0.0
 joblib==1.3.2
 Js2Py==0.74
-jsonschema==4.19.1
-jsonschema-specifications==2023.7.1
+jsonschema==4.20.0
+jsonschema-specifications==2023.11.1
 mmh3==4.0.1
 mock==5.1.0
 nltk==3.8.1
 nose==1.3.7
 numpy==1.24.4
 oauth2client==4.1.3
 objsize==0.6.1
-orjson==3.9.9
+orjson==3.9.10
 overrides==6.5.0
 packaging==23.2
 pandas==1.5.3
 parameterized==0.9.0
 pluggy==1.3.0
 proto-plus==1.22.3
-protobuf==4.25.0
+protobuf==4.25.1
 psycopg2-binary==2.9.9
-pyarrow==11.0.0
-pyarrow-hotfix==0.4
-pyasn1==0.5.0
+pyarrow==14.0.1
+pyarrow-hotfix==0.6
+pyasn1==0.5.1
 pyasn1-modules==0.3.0
 pycparser==2.21
 pydot==1.4.2
-PyHamcrest==2.0.4
+PyHamcrest==2.1.0
 pyjsparser==2.7.1
-pymongo==4.5.0
+pymongo==4.6.0
 PyMySQL==1.1.0
 pyparsing==3.1.1
 pyproject_hooks==1.0.0
-pytest==7.4.2
+pytest==7.4.3
 pytest-timeout==2.2.0
-pytest-xdist==3.3.1
+pytest-xdist==3.5.0
 python-dateutil==2.8.2
 python-snappy==0.6.1
 pytz==2023.3.post1
 PyYAML==6.0.1
-referencing==0.30.2
+referencing==0.31.0
 regex==2023.10.3
 requests==2.31.0
 requests-mock==1.11.0
-rpds-py==0.10.6
+rpds-py==0.13.1
 rsa==4.9
-scikit-learn==1.3.1
-scipy==1.11.3
+scikit-learn==1.3.2
+scipy==1.11.4
 shapely==2.0.2
 six==1.16.0
 sortedcontainers==2.4.0
 soupsieve==2.5
-SQLAlchemy==1.4.49
+SQLAlchemy==1.4.50
 sqlparse==0.4.4
 tenacity==8.2.3
 testcontainers==3.7.1
 threadpoolctl==3.2.0
 tomli==2.0.1
 tqdm==4.66.1
 typing_extensions==4.8.0
-tzlocal==5.1
+tzlocal==5.2
 uritemplate==4.1.1
-urllib3==2.0.7
+urllib3==2.1.0
 websocket-client==1.6.4
-wrapt==1.15.0
-zstandard==0.21.0
+wrapt==1.16.0
+zstandard==0.22.0
diff --git a/sdks/python/container/py311/base_image_requirements.txt b/sdks/python/container/py311/base_image_requirements.txt
@@ -25,117 +25,117 @@ attrs==23.1.0
 beautifulsoup4==4.12.2
 bs4==0.0.1
 build==1.0.3
-cachetools==5.3.1
-certifi==2023.7.22
+cachetools==5.3.2
+certifi==2023.11.17
 cffi==1.16.0
-charset-normalizer==3.3.0
+charset-normalizer==3.3.2
 click==8.1.7
 cloudpickle==2.2.1
 crcmod==1.7
-cryptography==41.0.4
+cryptography==41.0.6
 Cython==0.29.36
 deprecation==2.1.0
 dill==0.3.1.1
 dnspython==2.4.2
 docker==6.1.3
 docopt==0.6.2
 execnet==2.0.2
-fastavro==1.8.4
+fastavro==1.9.0
 fasteners==0.19
 freezegun==1.2.2
 future==0.18.3
-google-api-core==2.12.0
+google-api-core==2.14.0
 google-apitools==0.5.31
-google-auth==2.23.3
+google-auth==2.23.4
 google-auth-httplib2==0.1.1
-google-cloud-aiplatform==1.35.0
-google-cloud-bigquery==3.12.0
+google-cloud-aiplatform==1.36.4
+google-cloud-bigquery==3.13.0
 google-cloud-bigquery-storage==2.22.0
 google-cloud-bigtable==2.21.0
 google-cloud-core==2.3.3
 google-cloud-datastore==2.18.0
-google-cloud-dlp==3.12.3
+google-cloud-dlp==3.13.0
 google-cloud-language==2.11.1
 google-cloud-pubsub==2.18.4
 google-cloud-pubsublite==1.8.3
 google-cloud-recommendations-ai==0.10.5
 google-cloud-resource-manager==1.10.4
 google-cloud-spanner==3.40.1
-google-cloud-storage==2.12.0
+google-cloud-storage==2.13.0
 google-cloud-videointelligence==2.11.4
 google-cloud-vision==3.4.5
 google-crc32c==1.5.0
 google-resumable-media==2.6.0
 googleapis-common-protos==1.61.0
-greenlet==3.0.0
-grpc-google-iam-v1==0.12.6
-grpcio==1.59.0
-grpcio-status==1.59.0
-guppy3==3.1.4
+greenlet==3.0.1
+grpc-google-iam-v1==0.12.7
+grpcio==1.59.3
+grpcio-status==1.59.3
+guppy3==3.1.4.post1
 hdfs==2.7.3
 httplib2==0.22.0
-hypothesis==6.88.1
-idna==3.4
+hypothesis==6.91.0
+idna==3.6
 iniconfig==2.0.0
 joblib==1.3.2
 Js2Py==0.74
-jsonschema==4.19.1
-jsonschema-specifications==2023.7.1
+jsonschema==4.20.0
+jsonschema-specifications==2023.11.1
 mmh3==4.0.1
 mock==5.1.0
 nltk==3.8.1
 nose==1.3.7
 numpy==1.24.4
 oauth2client==4.1.3
 objsize==0.6.1
-orjson==3.9.9
+orjson==3.9.10
 overrides==6.5.0
 packaging==23.2
 pandas==1.5.3
 parameterized==0.9.0
 pluggy==1.3.0
 proto-plus==1.22.3
-protobuf==4.25.0
+protobuf==4.25.1
 psycopg2-binary==2.9.9
-pyarrow==11.0.0
-pyarrow-hotfix==0.4
-pyasn1==0.5.0
+pyarrow==14.0.1
+pyarrow-hotfix==0.6
+pyasn1==0.5.1
 pyasn1-modules==0.3.0
 pycparser==2.21
 pydot==1.4.2
-PyHamcrest==2.0.4
+PyHamcrest==2.1.0
 pyjsparser==2.7.1
-pymongo==4.5.0
+pymongo==4.6.0
 PyMySQL==1.1.0
 pyparsing==3.1.1
 pyproject_hooks==1.0.0
-pytest==7.4.2
+pytest==7.4.3
 pytest-timeout==2.2.0
-pytest-xdist==3.3.1
+pytest-xdist==3.5.0
 python-dateutil==2.8.2
 pytz==2023.3.post1
 PyYAML==6.0.1
-referencing==0.30.2
+referencing==0.31.0
 regex==2023.10.3
 requests==2.31.0
 requests-mock==1.11.0
-rpds-py==0.10.6
+rpds-py==0.13.1
 rsa==4.9
-scikit-learn==1.3.1
-scipy==1.11.3
+scikit-learn==1.3.2
+scipy==1.11.4
 shapely==2.0.2
 six==1.16.0
 sortedcontainers==2.4.0
 soupsieve==2.5
-SQLAlchemy==1.4.49
+SQLAlchemy==1.4.50
 sqlparse==0.4.4
 tenacity==8.2.3
 testcontainers==3.7.1
 threadpoolctl==3.2.0
 tqdm==4.66.1
 typing_extensions==4.8.0
-tzlocal==5.1
-urllib3==2.0.7
+tzlocal==5.2
+urllib3==2.1.0
 websocket-client==1.6.4
-wrapt==1.15.0
-zstandard==0.21.0
+wrapt==1.16.0
+zstandard==0.22.0