diff --git a/.github/workflows/code-checker.yaml b/.github/workflows/code-checker.yaml index c5a49e19f6c96..306ed6db598fb 100644 --- a/.github/workflows/code-checker.yaml +++ b/.github/workflows/code-checker.yaml @@ -43,37 +43,37 @@ jobs: uses: actions/checkout@v2 - name: Cache CCache Volumes if: ${{ matrix.os == 'ubuntu-18.04' }} - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: .docker/amd64-ubuntu18.04-ccache key: ubuntu18.04-ccache-${{ hashFiles('internal/core/**') }} restore-keys: ubuntu18.04-ccache- - name: Cache Third Party if: ${{ matrix.os == 'ubuntu-18.04' }} - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: .docker/thirdparty - key: ubuntu18.04-thirdparty-${{ hashFiles('internal/core/**') }} + key: ubuntu18.04-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }} restore-keys: ubuntu18.04-thirdparty- - name: Mac Cache Third Party if: ${{ matrix.os == 'macos-11' }} - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: | /tmp/thirdparty - cmake_build/3rdparty_download - key: macos-11-thirdparty-${{ hashFiles('internal/core/**') }} + cmake_build/3rdparty_download/download + key: macos-11-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }} restore-keys: macos-11-thirdparty- - name: Mac Cache CCache Volumes if: ${{ matrix.os == 'macos-11' }} - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: /var/tmp/ccache key: macos-11-ccache-${{ hashFiles('internal/core/**') }} restore-keys: macos-11-ccache- - name: Cache Go Mod Volumes if: ${{ matrix.os == 'ubuntu-18.04' }} - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: .docker/amd64-ubuntu18.04-go-mod key: ubuntu18.04-go-mod-${{ hashFiles('**/go.sum') }} @@ -85,7 +85,7 @@ jobs: go-version: '~1.17.1' - name: Mac Cache Go Mod Volumes if: ${{ matrix.os == 'macos-11' }} - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: ~/go/pkg/mod key: macos-11-go-mod-${{ hashFiles('**/go.sum') }} @@ -110,7 +110,7 @@ jobs: mkdir -p /var/tmp/ccache fi brew install boost libomp ninja tbb openblas ccache - make check-proto-product && make verifiers + make check-proto-product && make verifiers && sh scripts/collect_arrow_dep.sh -f /tmp/thirdparty centos: name: Code Checker CentOS 7 # Run in centos docker @@ -120,19 +120,19 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Cache CCache Volumes - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: .docker/amd64-centos7-ccache key: centos7-ccache-${{ hashFiles('internal/core/**') }} restore-keys: centos7-ccache- - name: Cache Third Party - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: .docker/thirdparty - key: centos7-thirdparty-${{ hashFiles('internal/core/**') }} + key: centos7-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }} restore-keys: centos7-thirdparty- - name: Cache Go Mod Volumes - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: .docker/amd64-centos7-go-mod key: centos7-go-mod-${{ hashFiles('**/go.sum') }} @@ -161,7 +161,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Cache go - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | %LocalAppData%\go-build @@ -169,7 +169,7 @@ jobs: key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} restore-keys: ${{ runner.os }}-go- - name: Cache ccache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | ${{ github.workspace }}/.ccache diff --git a/scripts/collect_arrow_dep.sh b/scripts/collect_arrow_dep.sh new file mode 100755 index 0000000000000..fe518a3eb151d --- /dev/null +++ b/scripts/collect_arrow_dep.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash + +# Licensed to the LF AI & Data foundation under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# assume we are in milvus directory + +CUSTOM_THIRDPARTY_PATH= + +while getopts "f:h" arg; do + case $arg in + f) + CUSTOM_THIRDPARTY_PATH=$OPTARG + ;; + + h) # help + echo " + +parameter: +-f: custom paths of thirdparty downloaded files(default: NULL) +-h: help + +usage: +./collect_arrow_dep.sh -f\${CUSTOM_THIRDPARTY_PATH} [-h] + " + exit 0 + ;; + ?) + echo "ERROR! unknown argument" + exit 1 + ;; + esac +done + + +if [ -z "$CUSTOM_THIRDPARTY_PATH" ] +then + echo " + parameter: + -f: custom paths of thirdparty downloaded files(default: NULL) + -h: help + + usage: + ./core_build.sh -f\${CUSTOM_THIRDPARTY_PATH} [-h] + " + exit 0 +fi + +if [ ! -d "$CUSTOM_THIRDPARTY_PATH" ] +then + echo "${CUSTOM_THIRDPARTY_PATH} is not a directory" + exit 0 +fi + +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + +BUILD_OUTPUT_DIR="${SCRIPTS_DIR}/../cmake_build" +ARROW_DIR=${BUILD_OUTPUT_DIR}/thirdparty/arrow +ARROWBIN_DIR=${BUILD_OUTPUT_DIR}/thirdparty/arrow/arrow-bin + +echo ${ARROWBIN_DIR} + +pushd ${ARROWBIN_DIR} +targetNames=( +"jemalloc-5.2.1.tar.bz2" +"thrift-0.13.0.tar.gz" +"utf8proc-v2.7.0.tar.gz" +"xsimd-7d1778c3b38d63db7cec7145d939f40bc5d859d1.tar.gz" +"zstd-v1.5.1.tar.gz" +) + +srcNames=( +"jemalloc_ep-prefix/src/jemalloc-5.2.1.tar.bz2" +"thrift_ep-prefix/src/thrift-0.13.0.tar.gz" +"utf8proc_ep-prefix/src/v2.7.0.tar.gz" +"src/7d1778c3b38d63db7cec7145d939f40bc5d859d1.tar.gz" +"zstd_ep-prefix/src/v1.5.1.tar.gz" +) + +for i in "${!srcNames[@]}"; do + if test -f "${srcNames[i]}"; then + echo "${srcNames[i]} exists. start to copy to ${CUSTOM_THIRDPARTY_PATH}/${targetNames[i]}" + echo cp -f "${srcNames[i]}" "${CUSTOM_THIRDPARTY_PATH}/${targetNames[i]}" + cp -f "${srcNames[i]}" "${CUSTOM_THIRDPARTY_PATH}/${targetNames[i]}" + fi +done + +popd diff --git a/scripts/core_build.sh b/scripts/core_build.sh index b443c23ff4e8e..8567113428317 100755 --- a/scripts/core_build.sh +++ b/scripts/core_build.sh @@ -204,6 +204,30 @@ echo ${CMAKE_CMD} ${CMAKE_CMD} -G "${CMAKE_GENERATOR}" +# enable offline build of arrow dependency if files exist. +arrowDepKeys=( +"ARROW_JEMALLOC_URL" +"ARROW_THRIFT_URL" +"ARROW_UTF8PROC_URL" +"ARROW_XSIMD_URL" +"ARROW_ZSTD_URL" +) +arrowDepValues=( +"jemalloc-5.2.1.tar.bz2" +"thrift-0.13.0.tar.gz" +"utf8proc-v2.7.0.tar.gz" +"xsimd-7d1778c3b38d63db7cec7145d939f40bc5d859d1.tar.gz" +"zstd-v1.5.1.tar.gz" +) +for i in "${!arrowDepValues[@]}"; do + if test -f "${CUSTOM_THIRDPARTY_PATH}/${arrowDepValues[$i]}"; then + echo "${arrowDepValues[$i]} exists." + export ${arrowDepKeys[$i]}=${CUSTOM_THIRDPARTY_PATH}/${arrowDepValues[$i]} + fi +done + +set + if [[ ${RUN_CPPLINT} == "ON" ]]; then # cpplint check make lint @@ -233,4 +257,10 @@ else make -j ${jobs} install || exit 1 fi +if command -v ccache &> /dev/null +then + ccache -s + exit +fi + popd