Bump Pyarrow version to include 14.0.0 (apache#29536)

* generate requirements file * update setup.py pyarrow * Update pyarrow except for windows * Skip tests for version 13 or greater. * Add reason for unittest skipIf * Fix pyarrow lower bound * Update sdks/python/setup.py Co-authored-by: tvalentyn <[email protected]> --------- Co-authored-by: tvalentyn <[email protected]>
lyft · Jul 24, 2024 · ed20488 · ed20488
1 parent 8fc90db
commit ed20488
Show file tree

Hide file tree

Showing 6 changed files with 309 additions and 255 deletions.
diff --git a/sdks/python/apache_beam/io/parquetio_test.py b/sdks/python/apache_beam/io/parquetio_test.py
@@ -333,6 +333,9 @@ def test_write_batched_display_data(self):
     ]
     hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))
 
+  @unittest.skipIf(
+      ARROW_MAJOR_VERSION >= 13,
+      'pyarrow 13.x and above does not throw ArrowInvalid error')
   def test_sink_transform_int96(self):
     with tempfile.NamedTemporaryFile() as dst:
       path = dst.name

diff --git a/sdks/python/container/py310/base_image_requirements.txt b/sdks/python/container/py310/base_image_requirements.txt
@@ -26,120 +26,128 @@ astunparse==1.6.3
 attrs==23.1.0
 beautifulsoup4==4.12.2
 bs4==0.0.1
-cachetools==5.3.1
-certifi==2023.7.22
-cffi==1.15.1
-charset-normalizer==3.2.0
-click==8.1.6
+build==1.0.3
+cachetools==5.3.2
+certifi==2023.11.17
+cffi==1.16.0
+charset-normalizer==3.3.2
+click==8.1.7
 cloudpickle==2.2.1
 crcmod==1.7
-cryptography==41.0.3
+cryptography==41.0.6
 Cython==0.29.36
 deprecation==2.1.0
 dill==0.3.1.1
 dnspython==2.4.2
 docker==6.1.3
 docopt==0.6.2
-exceptiongroup==1.1.2
+exceptiongroup==1.2.0
 execnet==2.0.2
-fastavro==1.8.2
-fasteners==0.18
+fastavro==1.9.0
+fasteners==0.19
 flatbuffers==23.5.26
 freezegun==1.2.2
 future==0.18.3
 gast==0.4.0
-google-api-core==2.11.1
-google-api-python-client==2.96.0
+google-api-core==2.14.0
+google-api-python-client==2.108.0
 google-apitools==0.5.31
-google-auth==2.22.0
-google-auth-httplib2==0.1.0
+google-auth==2.23.4
+google-auth-httplib2==0.1.1
 google-auth-oauthlib==1.0.0
-google-cloud-aiplatform==1.29.0
-google-cloud-bigquery==3.11.4
+google-cloud-aiplatform==1.36.4
+google-cloud-bigquery==3.13.0
 google-cloud-bigquery-storage==2.22.0
 google-cloud-bigtable==2.21.0
 google-cloud-core==2.3.3
-google-cloud-datastore==2.17.0
-google-cloud-dlp==3.12.2
-google-cloud-language==2.10.1
-google-cloud-profiler==4.0.0
-google-cloud-pubsub==2.18.2
+google-cloud-datastore==2.18.0
+google-cloud-dlp==3.13.0
+google-cloud-language==2.11.1
+google-cloud-profiler==4.1.0
+google-cloud-pubsub==2.18.4
 google-cloud-pubsublite==1.8.3
-google-cloud-recommendations-ai==0.10.4
-google-cloud-resource-manager==1.10.3
-google-cloud-spanner==3.40.0
-google-cloud-storage==2.10.0
-google-cloud-videointelligence==2.11.3
-google-cloud-vision==3.4.4
+google-cloud-recommendations-ai==0.10.5
+google-cloud-resource-manager==1.10.4
+google-cloud-spanner==3.40.1
+google-cloud-storage==2.13.0
+google-cloud-videointelligence==2.11.4
+google-cloud-vision==3.4.5
 google-crc32c==1.5.0
 google-pasta==0.2.0
-google-resumable-media==2.5.0
-googleapis-common-protos==1.60.0
-greenlet==2.0.2
-grpc-google-iam-v1==0.12.6
-grpcio==1.56.2
-grpcio-status==1.56.2
-guppy3==3.1.3
+google-resumable-media==2.6.0
+googleapis-common-protos==1.61.0
+greenlet==3.0.1
+grpc-google-iam-v1==0.12.7
+grpcio==1.59.3
+grpcio-status==1.59.3
+guppy3==3.1.4.post1
 h5py==3.9.0
-hdfs==2.7.2
+hdfs==2.7.3
 httplib2==0.22.0
-hypothesis==6.82.3
-idna==3.4
+hypothesis==6.91.0
+idna==3.6
 iniconfig==2.0.0
 joblib==1.3.2
 keras==2.13.1
 libclang==16.0.6
 Markdown==3.4.4
 MarkupSafe==2.1.3
+Js2Py==0.74
+jsonschema==4.20.0
+jsonschema-specifications==2023.11.1
 mmh3==4.0.1
 mock==5.1.0
 nltk==3.8.1
 nose==1.3.7
-numpy==1.24.3
+numpy==1.24.4
 oauth2client==4.1.3
 oauthlib==3.2.2
 objsize==0.6.1
 opt-einsum==3.3.0
-orjson==3.9.4
+orjson==3.9.10
 overrides==6.5.0
-packaging==23.1
+packaging==23.2
 pandas==1.5.3
 parameterized==0.9.0
-pluggy==1.2.0
+pluggy==1.3.0
 proto-plus==1.22.3
-protobuf==4.23.4
-psycopg2-binary==2.9.7
-pyarrow==11.0.0
-pyarrow-hotfix==0.4
-pyasn1==0.5.0
+protobuf==4.25.1
+psycopg2-binary==2.9.9
+pyarrow==14.0.1
+pyarrow-hotfix==0.6
+pyasn1==0.5.1
 pyasn1-modules==0.3.0
 pycparser==2.21
 pydot==1.4.2
-PyHamcrest==2.0.4
-pymongo==4.4.1
+PyHamcrest==2.1.0
+pyjsparser==2.7.1
+pymongo==4.6.0
 PyMySQL==1.1.0
 pyparsing==3.1.1
-pytest==7.4.0
-pytest-timeout==2.1.0
-pytest-xdist==3.3.1
+pyproject_hooks==1.0.0
+pytest==7.4.3
+pytest-timeout==2.2.0
+pytest-xdist==3.5.0
 python-dateutil==2.8.2
 python-snappy==0.6.1
-pytz==2023.3
+pytz==2023.3.post1
 PyYAML==6.0.1
-regex==2023.8.8
+referencing==0.31.0
+regex==2023.10.3
 requests==2.31.0
 requests-mock==1.11.0
 requests-oauthlib==1.3.1
+rpds-py==0.13.1
 rsa==4.9
-scikit-learn==1.3.0
-scipy==1.11.1
-Shapely==1.8.5.post1
+scikit-learn==1.3.2
+scipy==1.11.4
+shapely==2.0.2
 six==1.16.0
 sortedcontainers==2.4.0
-soupsieve==2.4.1
-SQLAlchemy==1.4.49
+soupsieve==2.5
+SQLAlchemy==1.4.50
 sqlparse==0.4.4
-tenacity==8.2.2
+tenacity==8.2.3
 tensorboard==2.13.0
 tensorboard-data-server==0.7.1
 tensorflow==2.13.0
@@ -150,11 +158,12 @@ termcolor==2.3.0
 testcontainers==3.7.1
 threadpoolctl==3.2.0
 tomli==2.0.1
-tqdm==4.66.0
-typing_extensions==4.5.0
+tqdm==4.66.1
+typing_extensions==4.8.0
+tzlocal==5.2
 uritemplate==4.1.1
-urllib3==1.26.16
-websocket-client==1.6.1
+urllib3==2.1.0
+websocket-client==1.6.4
 Werkzeug==2.3.6
-wrapt==1.15.0
-zstandard==0.21.0
+wrapt==1.16.0
+zstandard==0.22.0