ppc64le-support

milvus-io · Aug 6, 2024 · 8f1e86b · 8f1e86b
1 parent e937197
commit 8f1e86b
Show file tree

Hide file tree

Showing 5 changed files with 70 additions and 7 deletions.
diff --git a/deployments/docker/ppc64le/docker-compose.yml b/deployments/docker/ppc64le/docker-compose.yml
@@ -0,0 +1,61 @@
+version: '3.5'
+
+services:
+  etcd:
+    image: quay.io/coreos/etcd:v3.5.5
+    environment:
+      - ETCD_AUTO_COMPACTION_MODE=revision
+      - ETCD_AUTO_COMPACTION_RETENTION=1000
+      - ETCD_QUOTA_BACKEND_BYTES=4294967296
+      - ETCD_SNAPSHOT_COUNT=50000
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd
+    command: etcd -listen-peer-urls=http://127.0.0.1:2380 -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379,http://0.0.0.0:4001 -initial-advertise-peer-urls=http://127.0.0.1:2380 --initial-cluster default=http://127.0.0.1:2380 --data-dir /etcd
+    ports:
+      - "2379:2379"
+      - "2380:2380"
+      - "4001:4001"
+
+  pulsar:
+    image: peiit/pulsar:3.0.0
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/pulsar:/pulsar/data
+    environment:
+      # bin/apply-config-from-env.py script will modify the configuration file based on the environment variables
+      # nettyMaxFrameSizeBytes must be calculated from maxMessageSize + 10240 (padding)
+      - nettyMaxFrameSizeBytes=104867840 # this is 104857600 + 10240 (padding)
+      - defaultRetentionTimeInMinutes=10080
+      - defaultRetentionSizeInMB=8192
+      # maxMessageSize is missing from standalone.conf, must use PULSAR_PREFIX_ to get it configured
+      - PULSAR_PREFIX_maxMessageSize=104857600
+      - PULSAR_GC=-XX:+UseG1GC
+    ports:
+      - "6650:6650"
+      - "8080:8080"
+
+  minio:
+    image: minio/minio:latest
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    environment:
+      MINIO_ACCESS_KEY: minioadmin
+      MINIO_SECRET_KEY: minioadmin
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data
+    command: minio server /minio_data --console-address ":9001"
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "curl",
+          "-f",
+          "http://localhost:9000/minio/health/live"
+        ]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+
+networks:
+  default:
+    name: milvus_dev
diff --git a/internal/core/conanfile.py b/internal/core/conanfile.py
@@ -18,7 +18,7 @@ class MilvusConan(ConanFile):
         "aws-sdk-cpp/1.9.234",
         "googleapis/cci.20221108",
         "benchmark/1.7.0",
-        "gtest/1.13.0",
+        "gtest/1.14.0",
         "protobuf/3.21.4",
         "rapidxml/1.13",
         "yaml-cpp/0.7.0",
@@ -72,6 +72,8 @@ class MilvusConan(ConanFile):
         "boost:without_locale": False,
         "glog:with_gflags": True,
         "glog:shared": True,
+        "openssl:shared": True,
+        "libunwind:shared": True,
         "prometheus-cpp:with_pull": False,
         "fmt:header_only": True,
         "onetbb:tbbmalloc": False,

diff --git a/internal/core/src/index/Utils.h b/internal/core/src/index/Utils.h
@@ -137,8 +137,8 @@ void
 AssembleIndexDatas(std::map<std::string, FieldDataChannelPtr>& index_datas,
                    std::unordered_map<std::string, FieldDataPtr>& result);
 
-// On Linux, read() (and similar system calls) will transfer at most 0x7ffff000 (2,147,479,552) bytes once
+// On Linux, read() (and similar system calls) will transfer at most 0x7fff0000 (2,147,418,112) bytes once
 void
-ReadDataFromFD(int fd, void* buf, size_t size, size_t chunk_size = 0x7ffff000);
+ReadDataFromFD(int fd, void* buf, size_t size, size_t chunk_size = 0x7fff0000);
 
 }  // namespace milvus::index
diff --git a/internal/core/thirdparty/versions.txt b/internal/core/thirdparty/versions.txt
@@ -4,4 +4,4 @@ OPENTRACING_VERSION=v1.5.1
 PROTOBUF_VERSION=3.9.0
 LIBUNWIND_VERSION=1.6.2
 GPERFTOOLS_VERSION=2.9.1
-MILVUS_JEMALLOC_BUILD_VERSION=5.2.1
+MILVUS_JEMALLOC_BUILD_VERSION=5.3.0
diff --git a/tests/python_client/requirements.txt b/tests/python_client/requirements.txt
@@ -1,7 +1,7 @@
 --extra-index-url https://test.pypi.org/simple/
 pytest-cov==2.8.1
 requests==2.26.0
-scikit-learn==1.1.3
+scikit-learn
 timeout_decorator==0.5.0
 ujson==5.5.0
 pytest==7.2.0
@@ -22,7 +22,7 @@ pyasn1
 pytest-html==3.1.1
 delayed-assert==0.3.5
 kubernetes==17.17.0
-PyYAML==6.0
+PyYAML
 pytest-sugar==0.9.5
 pytest-parallel
 pytest-random-order
@@ -53,7 +53,7 @@ deepdiff==6.7.1
 
 # for test result anaylszer
 prettytable==3.8.0
-pyarrow==14.0.1
+pyarrow==16.0.0
 fastparquet==2023.7.0
 
 # for bf16 datatype