From 8f1e86b7cb2fd09aa37ff4de6392f4b5786bd606 Mon Sep 17 00:00:00 2001 From: Sumit Dubey Date: Tue, 6 Aug 2024 07:30:40 -0400 Subject: [PATCH] ppc64le-support --- deployments/docker/ppc64le/docker-compose.yml | 61 +++++++++++++++++++ internal/core/conanfile.py | 4 +- internal/core/src/index/Utils.h | 4 +- internal/core/thirdparty/versions.txt | 2 +- tests/python_client/requirements.txt | 6 +- 5 files changed, 70 insertions(+), 7 deletions(-) create mode 100644 deployments/docker/ppc64le/docker-compose.yml diff --git a/deployments/docker/ppc64le/docker-compose.yml b/deployments/docker/ppc64le/docker-compose.yml new file mode 100644 index 0000000000000..0d03afa88ad06 --- /dev/null +++ b/deployments/docker/ppc64le/docker-compose.yml @@ -0,0 +1,61 @@ +version: '3.5' + +services: + etcd: + image: quay.io/coreos/etcd:v3.5.5 + environment: + - ETCD_AUTO_COMPACTION_MODE=revision + - ETCD_AUTO_COMPACTION_RETENTION=1000 + - ETCD_QUOTA_BACKEND_BYTES=4294967296 + - ETCD_SNAPSHOT_COUNT=50000 + volumes: + - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd + command: etcd -listen-peer-urls=http://127.0.0.1:2380 -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379,http://0.0.0.0:4001 -initial-advertise-peer-urls=http://127.0.0.1:2380 --initial-cluster default=http://127.0.0.1:2380 --data-dir /etcd + ports: + - "2379:2379" + - "2380:2380" + - "4001:4001" + + pulsar: + image: peiit/pulsar:3.0.0 + volumes: + - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/pulsar:/pulsar/data + environment: + # bin/apply-config-from-env.py script will modify the configuration file based on the environment variables + # nettyMaxFrameSizeBytes must be calculated from maxMessageSize + 10240 (padding) + - nettyMaxFrameSizeBytes=104867840 # this is 104857600 + 10240 (padding) + - defaultRetentionTimeInMinutes=10080 + - defaultRetentionSizeInMB=8192 + # maxMessageSize is missing from standalone.conf, must use PULSAR_PREFIX_ to get it configured + - PULSAR_PREFIX_maxMessageSize=104857600 + - PULSAR_GC=-XX:+UseG1GC + ports: + - "6650:6650" + - "8080:8080" + + minio: + image: minio/minio:latest + ports: + - "9000:9000" + - "9001:9001" + environment: + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin + volumes: + - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data + command: minio server /minio_data --console-address ":9001" + healthcheck: + test: + [ + "CMD", + "curl", + "-f", + "http://localhost:9000/minio/health/live" + ] + interval: 30s + timeout: 20s + retries: 3 + +networks: + default: + name: milvus_dev diff --git a/internal/core/conanfile.py b/internal/core/conanfile.py index ec552fb9cabeb..4850279850768 100644 --- a/internal/core/conanfile.py +++ b/internal/core/conanfile.py @@ -18,7 +18,7 @@ class MilvusConan(ConanFile): "aws-sdk-cpp/1.9.234", "googleapis/cci.20221108", "benchmark/1.7.0", - "gtest/1.13.0", + "gtest/1.14.0", "protobuf/3.21.4", "rapidxml/1.13", "yaml-cpp/0.7.0", @@ -72,6 +72,8 @@ class MilvusConan(ConanFile): "boost:without_locale": False, "glog:with_gflags": True, "glog:shared": True, + "openssl:shared": True, + "libunwind:shared": True, "prometheus-cpp:with_pull": False, "fmt:header_only": True, "onetbb:tbbmalloc": False, diff --git a/internal/core/src/index/Utils.h b/internal/core/src/index/Utils.h index 1444eeeac638d..21b4105d3ebcc 100644 --- a/internal/core/src/index/Utils.h +++ b/internal/core/src/index/Utils.h @@ -137,8 +137,8 @@ void AssembleIndexDatas(std::map& index_datas, std::unordered_map& result); -// On Linux, read() (and similar system calls) will transfer at most 0x7ffff000 (2,147,479,552) bytes once +// On Linux, read() (and similar system calls) will transfer at most 0x7fff0000 (2,147,418,112) bytes once void -ReadDataFromFD(int fd, void* buf, size_t size, size_t chunk_size = 0x7ffff000); +ReadDataFromFD(int fd, void* buf, size_t size, size_t chunk_size = 0x7fff0000); } // namespace milvus::index diff --git a/internal/core/thirdparty/versions.txt b/internal/core/thirdparty/versions.txt index 119d1a3a1fa8f..81787254acd4e 100644 --- a/internal/core/thirdparty/versions.txt +++ b/internal/core/thirdparty/versions.txt @@ -4,4 +4,4 @@ OPENTRACING_VERSION=v1.5.1 PROTOBUF_VERSION=3.9.0 LIBUNWIND_VERSION=1.6.2 GPERFTOOLS_VERSION=2.9.1 -MILVUS_JEMALLOC_BUILD_VERSION=5.2.1 +MILVUS_JEMALLOC_BUILD_VERSION=5.3.0 diff --git a/tests/python_client/requirements.txt b/tests/python_client/requirements.txt index 928f09bc0e489..e136e75594c1c 100644 --- a/tests/python_client/requirements.txt +++ b/tests/python_client/requirements.txt @@ -1,7 +1,7 @@ --extra-index-url https://test.pypi.org/simple/ pytest-cov==2.8.1 requests==2.26.0 -scikit-learn==1.1.3 +scikit-learn timeout_decorator==0.5.0 ujson==5.5.0 pytest==7.2.0 @@ -22,7 +22,7 @@ pyasn1 pytest-html==3.1.1 delayed-assert==0.3.5 kubernetes==17.17.0 -PyYAML==6.0 +PyYAML pytest-sugar==0.9.5 pytest-parallel pytest-random-order @@ -53,7 +53,7 @@ deepdiff==6.7.1 # for test result anaylszer prettytable==3.8.0 -pyarrow==14.0.1 +pyarrow==16.0.0 fastparquet==2023.7.0 # for bf16 datatype