diff --git a/sdks/python/apache_beam/io/parquetio_test.py b/sdks/python/apache_beam/io/parquetio_test.py index 1cd5f1208cc2..06d9a5271404 100644 --- a/sdks/python/apache_beam/io/parquetio_test.py +++ b/sdks/python/apache_beam/io/parquetio_test.py @@ -334,6 +334,9 @@ def test_write_batched_display_data(self): ] hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items)) + @unittest.skipIf( + ARROW_MAJOR_VERSION >= 13, + 'pyarrow 13.x and above does not throw ArrowInvalid error') def test_sink_transform_int96(self): with tempfile.NamedTemporaryFile() as dst: path = dst.name diff --git a/sdks/python/container/py310/base_image_requirements.txt b/sdks/python/container/py310/base_image_requirements.txt index a9f94104374e..a80e6611d9cf 100644 --- a/sdks/python/container/py310/base_image_requirements.txt +++ b/sdks/python/container/py310/base_image_requirements.txt @@ -25,38 +25,38 @@ attrs==23.1.0 beautifulsoup4==4.12.2 bs4==0.0.1 build==1.0.3 -cachetools==5.3.1 -certifi==2023.7.22 +cachetools==5.3.2 +certifi==2023.11.17 cffi==1.16.0 -charset-normalizer==3.3.0 +charset-normalizer==3.3.2 click==8.1.7 cloudpickle==2.2.1 crcmod==1.7 -cryptography==41.0.4 +cryptography==41.0.6 Cython==0.29.36 deprecation==2.1.0 dill==0.3.1.1 dnspython==2.4.2 docker==6.1.3 docopt==0.6.2 -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 execnet==2.0.2 -fastavro==1.8.4 +fastavro==1.9.0 fasteners==0.19 freezegun==1.2.2 future==0.18.3 -google-api-core==2.12.0 -google-api-python-client==2.104.0 +google-api-core==2.14.0 +google-api-python-client==2.108.0 google-apitools==0.5.31 -google-auth==2.23.3 +google-auth==2.23.4 google-auth-httplib2==0.1.1 -google-cloud-aiplatform==1.35.0 -google-cloud-bigquery==3.12.0 +google-cloud-aiplatform==1.36.4 +google-cloud-bigquery==3.13.0 google-cloud-bigquery-storage==2.22.0 google-cloud-bigtable==2.21.0 google-cloud-core==2.3.3 google-cloud-datastore==2.18.0 -google-cloud-dlp==3.12.3 +google-cloud-dlp==3.13.0 google-cloud-language==2.11.1 google-cloud-profiler==4.1.0 google-cloud-pubsub==2.18.4 @@ -64,26 +64,26 @@ google-cloud-pubsublite==1.8.3 google-cloud-recommendations-ai==0.10.5 google-cloud-resource-manager==1.10.4 google-cloud-spanner==3.40.1 -google-cloud-storage==2.12.0 +google-cloud-storage==2.13.0 google-cloud-videointelligence==2.11.4 google-cloud-vision==3.4.5 google-crc32c==1.5.0 google-resumable-media==2.6.0 googleapis-common-protos==1.61.0 -greenlet==3.0.0 -grpc-google-iam-v1==0.12.6 -grpcio==1.59.0 -grpcio-status==1.59.0 -guppy3==3.1.4 +greenlet==3.0.1 +grpc-google-iam-v1==0.12.7 +grpcio==1.59.3 +grpcio-status==1.59.3 +guppy3==3.1.4.post1 hdfs==2.7.3 httplib2==0.22.0 -hypothesis==6.88.1 -idna==3.4 +hypothesis==6.91.0 +idna==3.6 iniconfig==2.0.0 joblib==1.3.2 Js2Py==0.74 -jsonschema==4.19.1 -jsonschema-specifications==2023.7.1 +jsonschema==4.20.0 +jsonschema-specifications==2023.11.1 mmh3==4.0.1 mock==5.1.0 nltk==3.8.1 @@ -91,47 +91,47 @@ nose==1.3.7 numpy==1.24.4 oauth2client==4.1.3 objsize==0.6.1 -orjson==3.9.9 +orjson==3.9.10 overrides==6.5.0 packaging==23.2 pandas==1.5.3 parameterized==0.9.0 pluggy==1.3.0 proto-plus==1.22.3 -protobuf==4.25.0 +protobuf==4.25.1 psycopg2-binary==2.9.9 -pyarrow==11.0.0 -pyarrow-hotfix==0.4 -pyasn1==0.5.0 +pyarrow==14.0.1 +pyarrow-hotfix==0.6 +pyasn1==0.5.1 pyasn1-modules==0.3.0 pycparser==2.21 pydot==1.4.2 -PyHamcrest==2.0.4 +PyHamcrest==2.1.0 pyjsparser==2.7.1 -pymongo==4.5.0 +pymongo==4.6.0 PyMySQL==1.1.0 pyparsing==3.1.1 pyproject_hooks==1.0.0 -pytest==7.4.2 +pytest==7.4.3 pytest-timeout==2.2.0 -pytest-xdist==3.3.1 +pytest-xdist==3.5.0 python-dateutil==2.8.2 python-snappy==0.6.1 pytz==2023.3.post1 PyYAML==6.0.1 -referencing==0.30.2 +referencing==0.31.0 regex==2023.10.3 requests==2.31.0 requests-mock==1.11.0 -rpds-py==0.10.6 +rpds-py==0.13.1 rsa==4.9 -scikit-learn==1.3.1 -scipy==1.11.3 +scikit-learn==1.3.2 +scipy==1.11.4 shapely==2.0.2 six==1.16.0 sortedcontainers==2.4.0 soupsieve==2.5 -SQLAlchemy==1.4.49 +SQLAlchemy==1.4.50 sqlparse==0.4.4 tenacity==8.2.3 testcontainers==3.7.1 @@ -139,9 +139,9 @@ threadpoolctl==3.2.0 tomli==2.0.1 tqdm==4.66.1 typing_extensions==4.8.0 -tzlocal==5.1 +tzlocal==5.2 uritemplate==4.1.1 -urllib3==2.0.7 +urllib3==2.1.0 websocket-client==1.6.4 -wrapt==1.15.0 -zstandard==0.21.0 +wrapt==1.16.0 +zstandard==0.22.0 diff --git a/sdks/python/container/py311/base_image_requirements.txt b/sdks/python/container/py311/base_image_requirements.txt index 865b856683a4..04d9fc28c273 100644 --- a/sdks/python/container/py311/base_image_requirements.txt +++ b/sdks/python/container/py311/base_image_requirements.txt @@ -25,14 +25,14 @@ attrs==23.1.0 beautifulsoup4==4.12.2 bs4==0.0.1 build==1.0.3 -cachetools==5.3.1 -certifi==2023.7.22 +cachetools==5.3.2 +certifi==2023.11.17 cffi==1.16.0 -charset-normalizer==3.3.0 +charset-normalizer==3.3.2 click==8.1.7 cloudpickle==2.2.1 crcmod==1.7 -cryptography==41.0.4 +cryptography==41.0.6 Cython==0.29.36 deprecation==2.1.0 dill==0.3.1.1 @@ -40,47 +40,47 @@ dnspython==2.4.2 docker==6.1.3 docopt==0.6.2 execnet==2.0.2 -fastavro==1.8.4 +fastavro==1.9.0 fasteners==0.19 freezegun==1.2.2 future==0.18.3 -google-api-core==2.12.0 +google-api-core==2.14.0 google-apitools==0.5.31 -google-auth==2.23.3 +google-auth==2.23.4 google-auth-httplib2==0.1.1 -google-cloud-aiplatform==1.35.0 -google-cloud-bigquery==3.12.0 +google-cloud-aiplatform==1.36.4 +google-cloud-bigquery==3.13.0 google-cloud-bigquery-storage==2.22.0 google-cloud-bigtable==2.21.0 google-cloud-core==2.3.3 google-cloud-datastore==2.18.0 -google-cloud-dlp==3.12.3 +google-cloud-dlp==3.13.0 google-cloud-language==2.11.1 google-cloud-pubsub==2.18.4 google-cloud-pubsublite==1.8.3 google-cloud-recommendations-ai==0.10.5 google-cloud-resource-manager==1.10.4 google-cloud-spanner==3.40.1 -google-cloud-storage==2.12.0 +google-cloud-storage==2.13.0 google-cloud-videointelligence==2.11.4 google-cloud-vision==3.4.5 google-crc32c==1.5.0 google-resumable-media==2.6.0 googleapis-common-protos==1.61.0 -greenlet==3.0.0 -grpc-google-iam-v1==0.12.6 -grpcio==1.59.0 -grpcio-status==1.59.0 -guppy3==3.1.4 +greenlet==3.0.1 +grpc-google-iam-v1==0.12.7 +grpcio==1.59.3 +grpcio-status==1.59.3 +guppy3==3.1.4.post1 hdfs==2.7.3 httplib2==0.22.0 -hypothesis==6.88.1 -idna==3.4 +hypothesis==6.91.0 +idna==3.6 iniconfig==2.0.0 joblib==1.3.2 Js2Py==0.74 -jsonschema==4.19.1 -jsonschema-specifications==2023.7.1 +jsonschema==4.20.0 +jsonschema-specifications==2023.11.1 mmh3==4.0.1 mock==5.1.0 nltk==3.8.1 @@ -88,54 +88,54 @@ nose==1.3.7 numpy==1.24.4 oauth2client==4.1.3 objsize==0.6.1 -orjson==3.9.9 +orjson==3.9.10 overrides==6.5.0 packaging==23.2 pandas==1.5.3 parameterized==0.9.0 pluggy==1.3.0 proto-plus==1.22.3 -protobuf==4.25.0 +protobuf==4.25.1 psycopg2-binary==2.9.9 -pyarrow==11.0.0 -pyarrow-hotfix==0.4 -pyasn1==0.5.0 +pyarrow==14.0.1 +pyarrow-hotfix==0.6 +pyasn1==0.5.1 pyasn1-modules==0.3.0 pycparser==2.21 pydot==1.4.2 -PyHamcrest==2.0.4 +PyHamcrest==2.1.0 pyjsparser==2.7.1 -pymongo==4.5.0 +pymongo==4.6.0 PyMySQL==1.1.0 pyparsing==3.1.1 pyproject_hooks==1.0.0 -pytest==7.4.2 +pytest==7.4.3 pytest-timeout==2.2.0 -pytest-xdist==3.3.1 +pytest-xdist==3.5.0 python-dateutil==2.8.2 pytz==2023.3.post1 PyYAML==6.0.1 -referencing==0.30.2 +referencing==0.31.0 regex==2023.10.3 requests==2.31.0 requests-mock==1.11.0 -rpds-py==0.10.6 +rpds-py==0.13.1 rsa==4.9 -scikit-learn==1.3.1 -scipy==1.11.3 +scikit-learn==1.3.2 +scipy==1.11.4 shapely==2.0.2 six==1.16.0 sortedcontainers==2.4.0 soupsieve==2.5 -SQLAlchemy==1.4.49 +SQLAlchemy==1.4.50 sqlparse==0.4.4 tenacity==8.2.3 testcontainers==3.7.1 threadpoolctl==3.2.0 tqdm==4.66.1 typing_extensions==4.8.0 -tzlocal==5.1 -urllib3==2.0.7 +tzlocal==5.2 +urllib3==2.1.0 websocket-client==1.6.4 -wrapt==1.15.0 -zstandard==0.21.0 +wrapt==1.16.0 +zstandard==0.22.0 diff --git a/sdks/python/container/py38/base_image_requirements.txt b/sdks/python/container/py38/base_image_requirements.txt index 83580f65f2d4..f22bbfe7d719 100644 --- a/sdks/python/container/py38/base_image_requirements.txt +++ b/sdks/python/container/py38/base_image_requirements.txt @@ -26,10 +26,10 @@ backports.zoneinfo==0.2.1 beautifulsoup4==4.12.2 bs4==0.0.1 build==1.0.3 -cachetools==5.3.1 -certifi==2023.7.22 +cachetools==5.3.2 +certifi==2023.11.17 cffi==1.16.0 -charset-normalizer==3.3.0 +charset-normalizer==3.3.2 click==8.1.7 cloudpickle==2.2.1 crcmod==1.7 @@ -40,24 +40,24 @@ dill==0.3.1.1 dnspython==2.4.2 docker==6.1.3 docopt==0.6.2 -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 execnet==2.0.2 -fastavro==1.8.4 +fastavro==1.9.0 fasteners==0.19 freezegun==1.2.2 future==0.18.3 -google-api-core==2.12.0 -google-api-python-client==2.104.0 +google-api-core==2.14.0 +google-api-python-client==2.108.0 google-apitools==0.5.31 -google-auth==2.23.3 +google-auth==2.23.4 google-auth-httplib2==0.1.1 -google-cloud-aiplatform==1.35.0 -google-cloud-bigquery==3.12.0 +google-cloud-aiplatform==1.36.4 +google-cloud-bigquery==3.13.0 google-cloud-bigquery-storage==2.22.0 google-cloud-bigtable==2.21.0 google-cloud-core==2.3.3 google-cloud-datastore==2.18.0 -google-cloud-dlp==3.12.3 +google-cloud-dlp==3.13.0 google-cloud-language==2.11.1 google-cloud-profiler==4.1.0 google-cloud-pubsub==2.18.4 @@ -65,28 +65,28 @@ google-cloud-pubsublite==1.8.3 google-cloud-recommendations-ai==0.10.5 google-cloud-resource-manager==1.10.4 google-cloud-spanner==3.40.1 -google-cloud-storage==2.12.0 +google-cloud-storage==2.13.0 google-cloud-videointelligence==2.11.4 google-cloud-vision==3.4.5 google-crc32c==1.5.0 google-resumable-media==2.6.0 googleapis-common-protos==1.61.0 -greenlet==3.0.0 -grpc-google-iam-v1==0.12.6 -grpcio==1.59.0 -grpcio-status==1.59.0 -guppy3==3.1.4 +greenlet==3.0.1 +grpc-google-iam-v1==0.12.7 +grpcio==1.59.3 +grpcio-status==1.59.3 +guppy3==3.1.4.post1 hdfs==2.7.3 httplib2==0.22.0 -hypothesis==6.88.1 -idna==3.4 +hypothesis==6.91.0 +idna==3.6 importlib-metadata==6.8.0 -importlib-resources==6.1.0 +importlib-resources==6.1.1 iniconfig==2.0.0 joblib==1.3.2 Js2Py==0.74 -jsonschema==4.19.1 -jsonschema-specifications==2023.7.1 +jsonschema==4.20.0 +jsonschema-specifications==2023.11.1 mmh3==4.0.1 mock==5.1.0 nltk==3.8.1 @@ -94,7 +94,7 @@ nose==1.3.7 numpy==1.24.4 oauth2client==4.1.3 objsize==0.6.1 -orjson==3.9.9 +orjson==3.9.10 overrides==6.5.0 packaging==23.2 pandas==1.5.3 @@ -102,40 +102,40 @@ parameterized==0.9.0 pkgutil_resolve_name==1.3.10 pluggy==1.3.0 proto-plus==1.22.3 -protobuf==4.25.0 +protobuf==4.25.1 psycopg2-binary==2.9.9 -pyarrow==11.0.0 -pyarrow-hotfix==0.4 -pyasn1==0.5.0 +pyarrow==14.0.1 +pyarrow-hotfix==0.6 +pyasn1==0.5.1 pyasn1-modules==0.3.0 pycparser==2.21 pydot==1.4.2 -PyHamcrest==2.0.4 +PyHamcrest==2.1.0 pyjsparser==2.7.1 -pymongo==4.5.0 +pymongo==4.6.0 PyMySQL==1.1.0 pyparsing==3.1.1 pyproject_hooks==1.0.0 -pytest==7.4.2 +pytest==7.4.3 pytest-timeout==2.2.0 -pytest-xdist==3.3.1 +pytest-xdist==3.5.0 python-dateutil==2.8.2 python-snappy==0.6.1 pytz==2023.3.post1 PyYAML==6.0.1 -referencing==0.30.2 +referencing==0.31.0 regex==2023.10.3 requests==2.31.0 requests-mock==1.11.0 -rpds-py==0.10.6 +rpds-py==0.13.1 rsa==4.9 -scikit-learn==1.3.1 +scikit-learn==1.3.2 scipy==1.10.1 shapely==2.0.2 six==1.16.0 sortedcontainers==2.4.0 soupsieve==2.5 -SQLAlchemy==1.4.49 +SQLAlchemy==1.4.50 sqlparse==0.4.4 tenacity==8.2.3 testcontainers==3.7.1 @@ -143,10 +143,10 @@ threadpoolctl==3.2.0 tomli==2.0.1 tqdm==4.66.1 typing_extensions==4.8.0 -tzlocal==5.1 +tzlocal==5.2 uritemplate==4.1.1 -urllib3==2.0.7 +urllib3==2.1.0 websocket-client==1.6.4 -wrapt==1.15.0 +wrapt==1.16.0 zipp==3.17.0 -zstandard==0.21.0 +zstandard==0.22.0 diff --git a/sdks/python/container/py39/base_image_requirements.txt b/sdks/python/container/py39/base_image_requirements.txt index 1b8ad7a2e748..f42b6f8c2b45 100644 --- a/sdks/python/container/py39/base_image_requirements.txt +++ b/sdks/python/container/py39/base_image_requirements.txt @@ -25,38 +25,38 @@ attrs==23.1.0 beautifulsoup4==4.12.2 bs4==0.0.1 build==1.0.3 -cachetools==5.3.1 -certifi==2023.7.22 +cachetools==5.3.2 +certifi==2023.11.17 cffi==1.16.0 -charset-normalizer==3.3.0 +charset-normalizer==3.3.2 click==8.1.7 cloudpickle==2.2.1 crcmod==1.7 -cryptography==41.0.4 +cryptography==41.0.6 Cython==0.29.36 deprecation==2.1.0 dill==0.3.1.1 dnspython==2.4.2 docker==6.1.3 docopt==0.6.2 -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 execnet==2.0.2 -fastavro==1.8.4 +fastavro==1.9.0 fasteners==0.19 freezegun==1.2.2 future==0.18.3 -google-api-core==2.12.0 -google-api-python-client==2.104.0 +google-api-core==2.14.0 +google-api-python-client==2.108.0 google-apitools==0.5.31 -google-auth==2.23.3 +google-auth==2.23.4 google-auth-httplib2==0.1.1 -google-cloud-aiplatform==1.35.0 -google-cloud-bigquery==3.12.0 +google-cloud-aiplatform==1.36.4 +google-cloud-bigquery==3.13.0 google-cloud-bigquery-storage==2.22.0 google-cloud-bigtable==2.21.0 google-cloud-core==2.3.3 google-cloud-datastore==2.18.0 -google-cloud-dlp==3.12.3 +google-cloud-dlp==3.13.0 google-cloud-language==2.11.1 google-cloud-profiler==4.1.0 google-cloud-pubsub==2.18.4 @@ -64,27 +64,27 @@ google-cloud-pubsublite==1.8.3 google-cloud-recommendations-ai==0.10.5 google-cloud-resource-manager==1.10.4 google-cloud-spanner==3.40.1 -google-cloud-storage==2.12.0 +google-cloud-storage==2.13.0 google-cloud-videointelligence==2.11.4 google-cloud-vision==3.4.5 google-crc32c==1.5.0 google-resumable-media==2.6.0 googleapis-common-protos==1.61.0 -greenlet==3.0.0 -grpc-google-iam-v1==0.12.6 -grpcio==1.59.0 -grpcio-status==1.59.0 -guppy3==3.1.4 +greenlet==3.0.1 +grpc-google-iam-v1==0.12.7 +grpcio==1.59.3 +grpcio-status==1.59.3 +guppy3==3.1.4.post1 hdfs==2.7.3 httplib2==0.22.0 -hypothesis==6.88.1 -idna==3.4 +hypothesis==6.91.0 +idna==3.6 importlib-metadata==6.8.0 iniconfig==2.0.0 joblib==1.3.2 Js2Py==0.74 -jsonschema==4.19.1 -jsonschema-specifications==2023.7.1 +jsonschema==4.20.0 +jsonschema-specifications==2023.11.1 mmh3==4.0.1 mock==5.1.0 nltk==3.8.1 @@ -92,47 +92,47 @@ nose==1.3.7 numpy==1.24.4 oauth2client==4.1.3 objsize==0.6.1 -orjson==3.9.9 +orjson==3.9.10 overrides==6.5.0 packaging==23.2 pandas==1.5.3 parameterized==0.9.0 pluggy==1.3.0 proto-plus==1.22.3 -protobuf==4.25.0 +protobuf==4.25.1 psycopg2-binary==2.9.9 -pyarrow==11.0.0 -pyarrow-hotfix==0.4 -pyasn1==0.5.0 +pyarrow==14.0.1 +pyarrow-hotfix==0.6 +pyasn1==0.5.1 pyasn1-modules==0.3.0 pycparser==2.21 pydot==1.4.2 -PyHamcrest==2.0.4 +PyHamcrest==2.1.0 pyjsparser==2.7.1 -pymongo==4.5.0 +pymongo==4.6.0 PyMySQL==1.1.0 pyparsing==3.1.1 pyproject_hooks==1.0.0 -pytest==7.4.2 +pytest==7.4.3 pytest-timeout==2.2.0 -pytest-xdist==3.3.1 +pytest-xdist==3.5.0 python-dateutil==2.8.2 python-snappy==0.6.1 pytz==2023.3.post1 PyYAML==6.0.1 -referencing==0.30.2 +referencing==0.31.0 regex==2023.10.3 requests==2.31.0 requests-mock==1.11.0 -rpds-py==0.10.6 +rpds-py==0.13.1 rsa==4.9 -scikit-learn==1.3.1 -scipy==1.11.3 +scikit-learn==1.3.2 +scipy==1.11.4 shapely==2.0.2 six==1.16.0 sortedcontainers==2.4.0 soupsieve==2.5 -SQLAlchemy==1.4.49 +SQLAlchemy==1.4.50 sqlparse==0.4.4 tenacity==8.2.3 testcontainers==3.7.1 @@ -140,10 +140,10 @@ threadpoolctl==3.2.0 tomli==2.0.1 tqdm==4.66.1 typing_extensions==4.8.0 -tzlocal==5.1 +tzlocal==5.2 uritemplate==4.1.1 -urllib3==2.0.7 +urllib3==2.1.0 websocket-client==1.6.4 -wrapt==1.15.0 +wrapt==1.16.0 zipp==3.17.0 -zstandard==0.21.0 +zstandard==0.22.0 diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 1785cd75df80..cdd3ecb1994c 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -141,9 +141,21 @@ def cythonize(*args, **kwargs): # [BEAM-8181] pyarrow cannot be installed on 32-bit Windows platforms. if sys.platform == 'win32' and sys.maxsize <= 2**32: pyarrow_dependency = [''] +elif sys.platform == 'win32' or sys.platform == 'cygwin': + # https://github.com/apache/beam/issues/28410 - pyarrow>=13 seeing issues + # on windows with error + # C:\arrow\cpp\src\arrow\filesystem\s3fs.cc:2904: arrow::fs::FinalizeS3 was + # not called even though S3 was initialized. This could lead to a + # segmentation fault at exit. Keep pyarrow<13 until this is resolved. + pyarrow_dependency = [ + 'pyarrow>=3.0.0,<12.0.0', + # NOTE: We can remove this once Beam increases the pyarrow lower bound + # to a version that fixes CVE. + 'pyarrow-hotfix<1' + ] else: pyarrow_dependency = [ - 'pyarrow>=3.0.0,<12.0.0', + 'pyarrow>=3.0.0,<15.0.0', # NOTE(https://github.com/apache/beam/issues/29392): We can remove this # once Beam increases the pyarrow lower bound to a version that fixes CVE. 'pyarrow-hotfix<1'