From cac802ef7f5e5a695e7f9935251ecdab9ce3c54c Mon Sep 17 00:00:00 2001 From: PowderLi <135960789+PowderLi@users.noreply.github.com> Date: Thu, 30 Nov 2023 15:58:32 +0800 Subject: [PATCH] enhance: use already installed vcpkg (#28703) issue #28686 1. Update Builder gpu image changes, see changes #28505 2. update azure-identity-cpp from beta to release Signed-off-by: PowderLi --- .env | 8 +- .github/workflows/publish-builder.yaml | 82 +--------------- .github/workflows/publish-gpu-builder.yaml | 94 +++++++++++++++++++ docker-compose.yml | 4 + .../AzureBlobChunkManager.cpp | 15 +-- .../cmake-modules/AzureVcpkg.cmake | 12 ++- .../azure-blob-storage/test/CMakeLists.txt | 3 +- .../test/test_azure_blob_chunk_manager.cpp | 61 +++++++++++- .../src/storage/azure-blob-storage/vcpkg.json | 3 +- scripts/azure_build.sh | 8 ++ 10 files changed, 184 insertions(+), 106 deletions(-) create mode 100644 .github/workflows/publish-gpu-builder.yaml diff --git a/.env b/.env index bec9d1ac3545a..fb7448d2e05c8 100644 --- a/.env +++ b/.env @@ -1,10 +1,10 @@ IMAGE_REPO=milvusdb IMAGE_ARCH=amd64 OS_NAME=ubuntu20.04 -DATE_VERSION=20231024-4faba61 -LATEST_DATE_VERSION=20231024-4faba61 -GPU_DATE_VERSION=20231123-90997a5 -LATEST_GPU_DATE_VERSION=20231123-90997a5 +DATE_VERSION=20231123-90997a5 +LATEST_DATE_VERSION=20231123-90997a5 +GPU_DATE_VERSION=20231025-a951c75 +LATEST_GPU_DATE_VERSION=20231025-a951c75 MINIO_ADDRESS=minio:9000 PULSAR_ADDRESS=pulsar://pulsar:6650 ETCD_ENDPOINTS=etcd:2379 diff --git a/.github/workflows/publish-builder.yaml b/.github/workflows/publish-builder.yaml index dab2c6f3dd152..6e85a103c4d44 100644 --- a/.github/workflows/publish-builder.yaml +++ b/.github/workflows/publish-builder.yaml @@ -7,14 +7,12 @@ on: # file paths to consider in the event. Optional; defaults to all. paths: - 'build/docker/builder/cpu/**' - - 'build/docker/builder/gpu/**' - '.github/workflows/publish-builder.yaml' - '!**.md' pull_request: # file paths to consider in the event. Optional; defaults to all. paths: - 'build/docker/builder/cpu/**' - - 'build/docker/builder/gpu/**' - '.github/workflows/publish-builder.yaml' - '!**.md' @@ -55,7 +53,6 @@ jobs: docker buildx build --platform linux/amd64,linux/arm64 -t milvusdb/milvus-env:${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/cpu/${OS_NAME}/Dockerfile . - name: Docker Build&Push if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' - continue-on-error: true shell: bash run: | docker buildx ls @@ -64,7 +61,7 @@ jobs: docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/cpu/${OS_NAME}/Dockerfile . docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:${OS_NAME}-latest -f build/docker/builder/cpu/${OS_NAME}/Dockerfile . - name: Update Builder Image Changes - if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' + if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04' continue-on-error: true shell: bash run: | @@ -96,80 +93,3 @@ jobs: echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" - - publish-gpu-builder: - name: ${{ matrix.arch }} ${{ matrix.os }} - runs-on: ubuntu-latest - timeout-minutes: 500 - strategy: - fail-fast: false - matrix: - os: [ ubuntu20.04 ] - arch: [ amd64&arm64 ] - env: - OS_NAME: ${{ matrix.os }} - IMAGE_ARCH: ${{ matrix.arch }} - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Get version from system time after release step - id: extracter - run: | - echo "::set-output name=version::$(date +%Y%m%d)" - echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" - # - name: Setup upterm session - # uses: lhotari/action-upterm@v1 - - name: Docker Pull - shell: bash - run: | - docker run --rm --privileged tonistiigi/binfmt:latest --install arm64 - docker buildx ls - docker buildx create --use --name=milvus --driver docker-container - - name: Docker Build - if: success() && github.event_name == 'pull_request' && github.repository == 'milvus-io/milvus' - shell: bash - run: | - docker buildx ls - docker buildx build --platform linux/amd64,linux/arm64 -t milvusdb/milvus-env:gpu-${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/gpu/${OS_NAME}/Dockerfile . - - name: Docker Build&Push - if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' - continue-on-error: true - shell: bash - run: | - docker buildx ls - docker login -u ${{ secrets.DOCKERHUB_USER }} \ - -p ${{ secrets.DOCKERHUB_TOKEN }} - docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:gpu-${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/gpu/${OS_NAME}/Dockerfile . - docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:gpu-${OS_NAME}-latest -f build/docker/builder/gpu/${OS_NAME}/Dockerfile . - - name: Update Builder Image Changes - if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' - continue-on-error: true - shell: bash - run: | - sed -i "s#^GPU_DATE_VERSION=.*#GPU_DATE_VERSION=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" .env - sed -i "s#^LATEST_GPU_DATE_VERSION=.*#LATEST_GPU_DATE_VERSION=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" .env - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot]" - git add .env - git commit -m "Update Builder gpu image changes" - - name: Create Pull Request - id: cpr - if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04' - continue-on-error: true - uses: peter-evans/create-pull-request@v3 - with: - token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }} - author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> - signoff: false - branch: update_builder_${{ github.sha }} - delete-branch: true - title: '[automated] Update Builder gpu image changes' - body: | - Update Builder gpu image changes - See changes: https://github.com/milvus-io/milvus/commit/${{ github.sha }} - Signed-off-by: ${{ github.actor }} ${{ github.actor }}@users.noreply.github.com - - name: Check outputs - if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04' - run: | - echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" - echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" diff --git a/.github/workflows/publish-gpu-builder.yaml b/.github/workflows/publish-gpu-builder.yaml new file mode 100644 index 0000000000000..7778b17a2d8ea --- /dev/null +++ b/.github/workflows/publish-gpu-builder.yaml @@ -0,0 +1,94 @@ +name: Publish Gpu Builder +# TODO: do not trigger action for some document file update + +# This workflow is triggered on pushes or pull request to the repository. +on: + push: + # file paths to consider in the event. Optional; defaults to all. + paths: + - 'build/docker/builder/gpu/**' + - '.github/workflows/publish-gpu-builder.yaml' + - '!**.md' + pull_request: + # file paths to consider in the event. Optional; defaults to all. + paths: + - 'build/docker/builder/gpu/**' + - '.github/workflows/publish-gpu-builder.yaml' + - '!**.md' + +jobs: + publish-gpu-builder: + name: ${{ matrix.arch }} ${{ matrix.os }} + runs-on: ubuntu-latest + timeout-minutes: 500 + strategy: + fail-fast: false + matrix: + os: [ubuntu20.04] + arch: [amd64&arm64] + env: + OS_NAME: ${{ matrix.os }} + IMAGE_ARCH: ${{ matrix.arch }} + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Get version from system time after release step + id: extracter + run: | + echo "::set-output name=version::$(date +%Y%m%d)" + echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + # - name: Setup upterm session + # uses: lhotari/action-upterm@v1 + - name: Docker Pull + shell: bash + run: | + docker run --rm --privileged tonistiigi/binfmt:latest --install arm64 + docker buildx ls + docker buildx create --use --name=milvus --driver docker-container + - name: Docker Build + if: success() && github.event_name == 'pull_request' && github.repository == 'milvus-io/milvus' + shell: bash + run: | + docker buildx ls + docker buildx build --platform linux/amd64,linux/arm64 -t milvusdb/milvus-env:gpu-${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/gpu/${OS_NAME}/Dockerfile . + - name: Docker Build&Push + if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' + shell: bash + run: | + docker buildx ls + docker login -u ${{ secrets.DOCKERHUB_USER }} \ + -p ${{ secrets.DOCKERHUB_TOKEN }} + docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:gpu-${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/gpu/${OS_NAME}/Dockerfile . + docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:gpu-${OS_NAME}-latest -f build/docker/builder/gpu/${OS_NAME}/Dockerfile . + - name: Update Builder Image Changes + if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04' + continue-on-error: true + shell: bash + run: | + sed -i "s#^GPU_DATE_VERSION=.*#GPU_DATE_VERSION=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" .env + sed -i "s#^LATEST_GPU_DATE_VERSION=.*#LATEST_GPU_DATE_VERSION=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" .env + git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --local user.name "github-actions[bot]" + git add .env + git commit -m "Update Builder gpu image changes" + - name: Create Pull Request + id: cpr + if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04' + continue-on-error: true + uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }} + author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> + signoff: false + branch: update_gpu_builder_${{ github.sha }} + delete-branch: true + title: '[automated] Update Builder gpu image changes' + body: | + Update Builder gpu image changes + See changes: https://github.com/milvus-io/milvus/commit/${{ github.sha }} + Signed-off-by: ${{ github.actor }} ${{ github.actor }}@users.noreply.github.com + - name: Check outputs + if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04' + run: | + echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" + echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" diff --git a/docker-compose.yml b/docker-compose.yml index a7bce14e4889f..9f186963f40d5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,8 @@ services: build: context: . dockerfile: build/docker/builder/cpu/${OS_NAME}/Dockerfile + args: + TARGETARCH: ${IMAGE_ARCH} cache_from: - ${IMAGE_REPO}/milvus-env:${OS_NAME}-${LATEST_DATE_VERSION} platform: linux/${IMAGE_ARCH} @@ -52,6 +54,8 @@ services: build: context: . dockerfile: build/docker/builder/gpu/${OS_NAME}/Dockerfile + args: + TARGETARCH: ${IMAGE_ARCH} cache_from: - ${IMAGE_REPO}/milvus-env:gpu-${OS_NAME}-${LATEST_GPU_DATE_VERSION} # user: {{ CURRENT_ID }} diff --git a/internal/core/src/storage/azure-blob-storage/AzureBlobChunkManager.cpp b/internal/core/src/storage/azure-blob-storage/AzureBlobChunkManager.cpp index 644461d0ded27..2133c6af479bc 100644 --- a/internal/core/src/storage/azure-blob-storage/AzureBlobChunkManager.cpp +++ b/internal/core/src/storage/azure-blob-storage/AzureBlobChunkManager.cpp @@ -21,18 +21,6 @@ namespace azure { -std::string -GetTenantId() { - return std::getenv("AZURE_TENANT_ID"); -} -std::string -GetClientId() { - return std::getenv("AZURE_CLIENT_ID"); -} -std::string -GetTokenFilePath() { - return std::getenv("AZURE_FEDERATED_TOKEN_FILE"); -} std::string GetConnectionString(const std::string& access_key_id, const std::string& access_key_value, @@ -78,9 +66,10 @@ AzureBlobChunkManager::AzureBlobChunkManager( bool useIAM) { requestTimeoutMs_ = requestTimeoutMs; if (useIAM) { + Azure::Identity::WorkloadIdentityCredentialOptions options; auto workloadIdentityCredential = std::make_shared( - GetTenantId(), GetClientId(), GetTokenFilePath()); + options); client_ = std::make_shared( "https://" + access_key_id + ".blob." + address + "/", workloadIdentityCredential); diff --git a/internal/core/src/storage/azure-blob-storage/cmake-modules/AzureVcpkg.cmake b/internal/core/src/storage/azure-blob-storage/cmake-modules/AzureVcpkg.cmake index c49a433e59a45..1dc97632cff37 100644 --- a/internal/core/src/storage/azure-blob-storage/cmake-modules/AzureVcpkg.cmake +++ b/internal/core/src/storage/azure-blob-storage/cmake-modules/AzureVcpkg.cmake @@ -6,6 +6,7 @@ set(AZ_ROOT_DIR "${CMAKE_CURRENT_LIST_DIR}/..") macro(az_vcpkg_integrate) message("Vcpkg integrate step.") + # AUTO CMAKE_TOOLCHAIN_FILE: # User can call `cmake -DCMAKE_TOOLCHAIN_FILE="path_to_the_toolchain"` as the most specific scenario. # As the last alternative (default case), Azure SDK will automatically clone VCPKG folder and set toolchain from there. @@ -17,7 +18,7 @@ macro(az_vcpkg_integrate) message("AZURE_SDK_DISABLE_AUTO_VCPKG is not defined. Fetch a local copy of vcpkg.") # GET VCPKG FROM SOURCE # User can set env var AZURE_SDK_VCPKG_COMMIT to pick the VCPKG commit to fetch - set(VCPKG_COMMIT_STRING 71d875654e32ee216b0b7e0dc684e589dffa1b1c) # default SDK tested commit + set(VCPKG_COMMIT_STRING dc3c55f092c96fb3f1dcdff84e6a99f947ea4165) # default SDK tested commit if(DEFINED ENV{AZURE_SDK_VCPKG_COMMIT}) message("AZURE_SDK_VCPKG_COMMIT is defined. Using that instead of the default.") set(VCPKG_COMMIT_STRING "$ENV{AZURE_SDK_VCPKG_COMMIT}") # default SDK tested commit @@ -26,15 +27,15 @@ macro(az_vcpkg_integrate) include(FetchContent) FetchContent_Declare( vcpkg - GIT_REPOSITORY https://github.com/milvus-io/vcpkg.git + GIT_REPOSITORY https://github.com/microsoft/vcpkg.git GIT_TAG ${VCPKG_COMMIT_STRING} ) FetchContent_GetProperties(vcpkg) # make sure to pull vcpkg only once. - if(NOT vcpkg_POPULATED) + if(NOT vcpkg_POPULATED) FetchContent_Populate(vcpkg) endif() - # use the vcpkg source path + # use the vcpkg source path set(CMAKE_TOOLCHAIN_FILE "${vcpkg_SOURCE_DIR}/scripts/buildsystems/vcpkg.cmake" CACHE STRING "") endif() endif() @@ -43,6 +44,7 @@ macro(az_vcpkg_integrate) if(DEFINED ENV{VCPKG_DEFAULT_TRIPLET} AND NOT DEFINED VCPKG_TARGET_TRIPLET) set(VCPKG_TARGET_TRIPLET "$ENV{VCPKG_DEFAULT_TRIPLET}" CACHE STRING "") endif() + message("Vcpkg integrate step - DONE.") endmacro() macro(az_vcpkg_portfile_prep targetName fileName contentToRemove) @@ -166,4 +168,4 @@ macro(az_vcpkg_export targetName macroNamePart dllImportExportHeaderPath) # Export all the installs above as package. export(PACKAGE "${targetName}-cpp") -endmacro() +endmacro() \ No newline at end of file diff --git a/internal/core/src/storage/azure-blob-storage/test/CMakeLists.txt b/internal/core/src/storage/azure-blob-storage/test/CMakeLists.txt index bf666ef6f86bc..3dbbeb0b8458a 100644 --- a/internal/core/src/storage/azure-blob-storage/test/CMakeLists.txt +++ b/internal/core/src/storage/azure-blob-storage/test/CMakeLists.txt @@ -4,4 +4,5 @@ project(azure-blob-test) add_executable(azure-blob-test test_azure_blob_chunk_manager.cpp ../AzureBlobChunkManager.cpp) -target_link_libraries(azure-blob-test PRIVATE Azure::azure-identity Azure::azure-storage-blobs) +find_package(GTest CONFIG REQUIRED) +target_link_libraries(azure-blob-test PRIVATE Azure::azure-identity Azure::azure-storage-blobs GTest::gtest) diff --git a/internal/core/src/storage/azure-blob-storage/test/test_azure_blob_chunk_manager.cpp b/internal/core/src/storage/azure-blob-storage/test/test_azure_blob_chunk_manager.cpp index ce1b941ef07a8..79dcb55fa3c46 100644 --- a/internal/core/src/storage/azure-blob-storage/test/test_azure_blob_chunk_manager.cpp +++ b/internal/core/src/storage/azure-blob-storage/test/test_azure_blob_chunk_manager.cpp @@ -1,7 +1,60 @@ #include "../AzureBlobChunkManager.h" +#include +#include using namespace azure; +void +SetTenantId(const char* value) { + setenv("AZURE_TENANT_ID", value, 1); +} +void +SetClientId(const char* value) { + setenv("AZURE_CLIENT_ID", value, 1); +} +void +SetTokenFilePath(const char* value) { + setenv("AZURE_FEDERATED_TOKEN_FILE", value, 1); +} + +std::string +GetTenantId() { + return std::getenv("AZURE_TENANT_ID"); +} +std::string +GetClientId() { + return std::getenv("AZURE_CLIENT_ID"); +} +std::string +GetTokenFilePath() { + return std::getenv("AZURE_FEDERATED_TOKEN_FILE"); +} + +class AzureBlobChunkManagerTest : public testing::Test { + protected: + void + SetUp() override { + } + + // void TearDown() override {} +}; + +TEST(AzureBlobChunkManagerTest, Options) { + SetTenantId("tenant_id"); + SetClientId("client_id"); + SetTokenFilePath("token_file_path"); + + Azure::Identity::WorkloadIdentityCredentialOptions options; + Azure::Identity::WorkloadIdentityCredential const cred(options); + EXPECT_EQ(cred.GetCredentialName(), "WorkloadIdentityCredential"); + + EXPECT_EQ(options.TenantId, GetTenantId()); + EXPECT_EQ(options.TenantId, "tenant_id"); + EXPECT_EQ(options.ClientId, GetClientId()); + EXPECT_EQ(options.AuthorityHost, "https://login.microsoftonline.com/"); + EXPECT_EQ(options.TokenFilePath, GetTokenFilePath()); +} + void print(Azure::Core::Diagnostics::Logger::Level level, std::string const& message) { @@ -10,7 +63,7 @@ print(Azure::Core::Diagnostics::Logger::Level level, } int -main() { +main0() { const char* containerName = "default"; const char* blobName = "sample-blob"; AzureBlobChunkManager::InitLog("info", print); @@ -59,3 +112,9 @@ main() { chunkManager.DeleteBucket(containerName); exit(EXIT_SUCCESS); } + +int +main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/internal/core/src/storage/azure-blob-storage/vcpkg.json b/internal/core/src/storage/azure-blob-storage/vcpkg.json index ac0d797d5d9e5..797533209939e 100644 --- a/internal/core/src/storage/azure-blob-storage/vcpkg.json +++ b/internal/core/src/storage/azure-blob-storage/vcpkg.json @@ -3,6 +3,7 @@ "version-string": "1.0.0", "dependencies": [ "azure-identity-cpp", - "azure-storage-blobs-cpp" + "azure-storage-blobs-cpp", + "gtest" ] } diff --git a/scripts/azure_build.sh b/scripts/azure_build.sh index e494933fe68c0..60d2717feb447 100644 --- a/scripts/azure_build.sh +++ b/scripts/azure_build.sh @@ -39,6 +39,14 @@ fi AZURE_CMAKE_CMD="cmake -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ ${SOURCE_DIR}" + +if [[ -f "/opt/vcpkg/scripts/buildsystems/vcpkg.cmake" ]]; then + AZURE_CMAKE_CMD="cmake -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ + -DCMAKE_TOOLCHAIN_FILE=/opt/vcpkg/scripts/buildsystems/vcpkg.cmake \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ + ${SOURCE_DIR}" +fi + echo ${AZURE_CMAKE_CMD} ${AZURE_CMAKE_CMD}