build: support building from aarch64

- include pre-compiled thirdparty and zetasql for aarch64 - fix python sdk naming under aarch64
4paradigm · Mar 14, 2024 · d09a1e7 · d09a1e7
1 parent db1c1f8
commit d09a1e7
Show file tree

Hide file tree

Showing 5 changed files with 29 additions and 17 deletions.
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -15,7 +15,7 @@
 
 FROM centos:7
 
-ARG ZETASQL_VERSION=0.3.4-b0
+ARG ZETASQL_VERSION=0.3.4
 ARG THIRDPARTY_VERSION=0.7.0
 ARG TARGETARCH
 

diff --git a/docker/setup_deps.sh b/docker/setup_deps.sh
@@ -88,11 +88,10 @@ if [[ "$ARCH" = "x86_64" ]]; then
     curl -Lo zetasql.tar.gz "https://github.com/4paradigm/zetasql/releases/download/v${ZETASQL_VERSION}/libzetasql-${ZETASQL_VERSION}-linux-gnu-x86_64-centos.tar.gz" && \
         echo "downloaed zetasql.tar.gz version $ZETASQL_VERSION for $ARCH"
 elif [[ "$ARCH" = "aarch64" ]]; then
-    # NOTE(aceforeverd): thirdparty for arm is out-of-date
-    curl -Lo thirdparty.tar.gz https://github.com/4paradigm/hybridsql-asserts/releases/download/v0.4.0/thirdparty-2021-08-03-linux-gnu-aarch64.tar.gz && \
-        echo "downloaded thirdparty.tar.gz version 0.4.0 for $ARCH"
-    curl -Lo zetasql.tar.gz "https://github.com/4paradigm/zetasql/releases/download/v0.2.6/libzetasql-0.2.6-linux-gnu-aarch64-centos.tar.gz" && \
-        echo "downloaed zetasql.tar.gz for version 0.2.6 $ARCH"
+    curl -Lo thirdparty.tar.gz "https://github.com/4paradigm/hybridsql-asserts/releases/download/v${THIRDPARTY_VERSION}/thirdparty-${THIRDPARTY_VERSION}-linux-gnu-${ARCH}.tar.gz" && \
+        echo "downloaded thirdparty.tar.gz version $THIRDPARTY_VERSION for $ARCH"
+    curl -Lo zetasql.tar.gz "https://github.com/4paradigm/zetasql/releases/download/v${ZETASQL_VERSION}/libzetasql-${ZETASQL_VERSION}-linux-gnu-${ARCH}.tar.gz" && \
+        echo "downloaed zetasql.tar.gz version $ZETASQL_VERSION for $ARCH"
 else
     echo "no pre-compiled deps for arch=$ARCH"
     exit 1

diff --git a/src/sdk/CMakeLists.txt b/src/sdk/CMakeLists.txt
@@ -130,7 +130,18 @@ if(SQL_PYSDK_ENABLE)
                 COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:sql_router_sdk> ${PROJECT_SOURCE_DIR}/python/openmldb_sdk/openmldb/native/
                 COMMAND echo "Do not strip library for MacOS, refer to https://github.com/4paradigm/OpenMLDB/issues/905")
     else()
-        set(PYTHON_PLATFORM manylinux1_x86_64)
+        # ref https://peps.python.org/pep-0600/ 
+        # centos7 is glibc 2.17, so 'manylinux2014' should preferred, 
+        # 'manylinux2014' is just legacy alias 'manylinux_2_17', we keep this legacy name 
+        # since there still installation issue on python 3.8 for new the name.
+        if (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(x86_64)")
+            set(PYTHON_PLATFORM manylinux1_x86_64)
+        elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "(arm64)|(ARM64)|(aarch64)|(AARCH64)")
+            set(PYTHON_PLATFORM manylinux2014_aarch64)
+        else()
+            message(FATAL_ERROR "unsupported architecture ${CMAKE_SYSTEM_PROCESSOR}")
+        endif()
+
         add_custom_target(strip_python_so ALL DEPENDS sql_router_sdk
             COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:sql_router_sdk> ${PROJECT_SOURCE_DIR}/python/openmldb_sdk/openmldb/native/
             COMMAND strip ${PROJECT_SOURCE_DIR}/python/openmldb_sdk/openmldb/native/_sql_router_sdk.so)

diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt
@@ -88,9 +88,8 @@ function(init_hybridsql_thirdparty_urls)
     get_linux_lsb_release_information()
 
     if(CMAKE_SYSTEM_PROCESSOR MATCHES "(arm64)|(ARM64)|(aarch64)|(AARCH64)")
-      # NOTE(#1138): pre-compiled thirdparty is out-of-date
-      set(HYBRIDSQL_ASSERTS_URL "https://github.com/4paradigm/hybridsql-asserts/releases/download/v0.4.0/thirdparty-2021-08-03-linux-gnu-aarch64.tar.gz" PARENT_SCOPE)
-      set(HYBRIDSQL_ASSERTS_HASH 128e2147903c6b4f5649284326c60ba048ff369f2925f9957f2973a515c65d75 PARENT_SCOPE)
+      set(HYBRIDSQL_ASSERTS_URL "${HYBRIDSQL_ASSERTS_HOME}/releases/download/v${HYBRIDSQL_ASSERTS_VERSION}/thirdparty-${HYBRIDSQL_ASSERTS_VERSION}-linux-gnu-aarch64.tar.gz" PARENT_SCOPE)
+      set(HYBRIDSQL_ASSERTS_HASH 323ca51573b46bb71b781a1a10a91a4112559fa54c18ae2d7bacc7e342f49f3d PARENT_SCOPE)
     else()
       if (LSB_RELEASE_ID_SHORT STREQUAL "centos")
         set(HYBRIDSQL_ASSERTS_URL "${HYBRIDSQL_ASSERTS_HOME}/releases/download/v${HYBRIDSQL_ASSERTS_VERSION}/thirdparty-${HYBRIDSQL_ASSERTS_VERSION}-linux-gnu-x86_64-centos.tar.gz" PARENT_SCOPE)

diff --git a/third-party/cmake/FetchZetasql.cmake b/third-party/cmake/FetchZetasql.cmake
@@ -13,16 +13,22 @@
 # limitations under the License.
 
 set(ZETASQL_HOME https://github.com/4paradigm/zetasql)
-set(ZETASQL_VERSION 0.3.4-b0)
-set(ZETASQL_HASH_DARWIN 0cd74b469396f877751cf7efd39685cc4fc544608aef4a901f3f640c3d8aa947)
-set(ZETASQL_HASH_LINUX_UBUNTU f94e97ad0b70b34d876cd3820f64756a164b061f3cb40a089e179f2997995bdc)
-set(ZETASQL_HASH_LINUX_CENTOS 8e853c4dd9b6638092357fa250ee29e634a1bc8b85ab8c829fbafac1b43ee8b8)
+set(ZETASQL_VERSION 0.3.4)
+set(ZETASQL_HASH_DARWIN a8d18f1595fa6a78d09c5feca813e9a332cd4d156a95124165f7c1a1ebcb86b2)
+set(ZETASQL_HASH_LINUX_UBUNTU 11988ad45de027566386a8e832e7385755f9367d78243b8d2aa6ef3493c0ee3d)
+set(ZETASQL_HASH_LINUX_CENTOS 92c9eb4d1ffb7d96fcb33150366906c52f27db4cf8dfb442c06e5fe5c738cd4a)
+set(ZETASQL_HASH_LINUX_AARCH64 e20e03938ec108b1c08203c238841ae781498dd66c5d791f4ecc326440287cdb)
 set(ZETASQL_TAG v${ZETASQL_VERSION})
 
 function(init_zetasql_urls)
   if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
     get_linux_lsb_release_information()
 
+    if (CMAKE_SYSTEM_PROCESSOR MATCHES "(arm64)|(ARM64)|(aarch64)|(AARCH64)")
+      set(ZETASQL_URL "${ZETASQL_HOME}/releases/download/v${ZETASQL_VERSION}/libzetasql-${ZETASQL_VERSION}-linux-gnu-aarch64.tar.gz" PARENT_SCOPE)
+      set(ZETASQL_HASH ${ZETASQL_HASH_LINUX_AARCH64} PARENT_SCOPE)
+    endif()
+
     if (LSB_RELEASE_ID_SHORT STREQUAL "centos")
       set(ZETASQL_URL "${ZETASQL_HOME}/releases/download/v${ZETASQL_VERSION}/libzetasql-${ZETASQL_VERSION}-linux-gnu-x86_64-centos.tar.gz" PARENT_SCOPE)
       set(ZETASQL_HASH ${ZETASQL_HASH_LINUX_CENTOS} PARENT_SCOPE)
@@ -42,9 +48,6 @@ endfunction()
 if (NOT BUILD_BUNDLED_ZETASQL)
   init_zetasql_urls()
 
-  if (CMAKE_SYSTEM_PROCESSOR MATCHES "(arm64)|(ARM64)|(aarch64)|(AARCH64)")
-    message(FATAL_ERROR "pre-compiled zetasql for arm64 not available, try compile zetasql from source by cmake flag: '-DBUILD_BUNDLED_ZETASQL=ON'")
-  endif()
   message(STATUS "Download pre-compiled zetasql from ${ZETASQL_URL}")
   # download pre-compiled zetasql from GitHub Release
   ExternalProject_Add(zetasql