enhance: use already installed vcpkg (#28703)

issue #28686 1. Update Builder gpu image changes, see changes #28505 2. update azure-identity-cpp from beta to release Signed-off-by: PowderLi <[email protected]>
milvus-io · Nov 30, 2023 · cac802e · cac802e
1 parent aae7e62
commit cac802e
Show file tree

Hide file tree

Showing 10 changed files with 184 additions and 106 deletions.
diff --git a/.env b/.env
@@ -1,10 +1,10 @@
 IMAGE_REPO=milvusdb
 IMAGE_ARCH=amd64
 OS_NAME=ubuntu20.04
-DATE_VERSION=20231024-4faba61
-LATEST_DATE_VERSION=20231024-4faba61
-GPU_DATE_VERSION=20231123-90997a5
-LATEST_GPU_DATE_VERSION=20231123-90997a5
+DATE_VERSION=20231123-90997a5
+LATEST_DATE_VERSION=20231123-90997a5
+GPU_DATE_VERSION=20231025-a951c75
+LATEST_GPU_DATE_VERSION=20231025-a951c75
 MINIO_ADDRESS=minio:9000
 PULSAR_ADDRESS=pulsar://pulsar:6650
 ETCD_ENDPOINTS=etcd:2379

diff --git a/.github/workflows/publish-builder.yaml b/.github/workflows/publish-builder.yaml
@@ -7,14 +7,12 @@ on:
     # file paths to consider in the event. Optional; defaults to all.
     paths:
       - 'build/docker/builder/cpu/**'
-      - 'build/docker/builder/gpu/**'
       - '.github/workflows/publish-builder.yaml'
       - '!**.md'
   pull_request:
     # file paths to consider in the event. Optional; defaults to all.
     paths:
       - 'build/docker/builder/cpu/**'
-      - 'build/docker/builder/gpu/**'
       - '.github/workflows/publish-builder.yaml'
       - '!**.md'
 
@@ -55,7 +53,6 @@ jobs:
           docker buildx build --platform linux/amd64,linux/arm64 -t milvusdb/milvus-env:${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/cpu/${OS_NAME}/Dockerfile .      
       - name: Docker Build&Push
         if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
-        continue-on-error: true
         shell: bash
         run: |
           docker buildx ls
@@ -64,7 +61,7 @@ jobs:
           docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/cpu/${OS_NAME}/Dockerfile .
           docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:${OS_NAME}-latest -f build/docker/builder/cpu/${OS_NAME}/Dockerfile .
       - name: Update Builder Image Changes
-        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
+        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
         continue-on-error: true
         shell: bash
         run: |
@@ -96,80 +93,3 @@ jobs:
           echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
           echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
 
-
-  publish-gpu-builder:
-    name: ${{ matrix.arch }} ${{ matrix.os }}
-    runs-on: ubuntu-latest
-    timeout-minutes: 500
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ ubuntu20.04 ]
-        arch: [ amd64&arm64 ]
-    env:
-      OS_NAME: ${{ matrix.os }}
-      IMAGE_ARCH: ${{ matrix.arch }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Get version from system time after release step
-        id: extracter
-        run: |
-          echo "::set-output name=version::$(date +%Y%m%d)"
-          echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
-      # - name: Setup upterm session
-      #   uses: lhotari/action-upterm@v1
-      - name: Docker Pull
-        shell: bash
-        run: |
-          docker run --rm --privileged tonistiigi/binfmt:latest --install arm64
-          docker buildx ls
-          docker buildx create --use --name=milvus --driver docker-container
-      - name: Docker Build
-        if: success() && github.event_name == 'pull_request' && github.repository == 'milvus-io/milvus'
-        shell: bash
-        run: |
-          docker buildx ls
-          docker buildx build --platform linux/amd64,linux/arm64 -t milvusdb/milvus-env:gpu-${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
-      - name: Docker Build&Push
-        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
-        continue-on-error: true
-        shell: bash
-        run: |
-          docker buildx ls
-          docker login -u ${{ secrets.DOCKERHUB_USER }} \
-                       -p ${{ secrets.DOCKERHUB_TOKEN }}
-          docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:gpu-${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
-          docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:gpu-${OS_NAME}-latest -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
-      - name: Update Builder Image Changes
-        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
-        continue-on-error: true
-        shell: bash
-        run: |
-          sed -i "s#^GPU_DATE_VERSION=.*#GPU_DATE_VERSION=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" .env
-          sed -i "s#^LATEST_GPU_DATE_VERSION=.*#LATEST_GPU_DATE_VERSION=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" .env
-          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git config --local user.name "github-actions[bot]"
-          git add .env
-          git commit -m "Update Builder gpu image changes"
-      - name: Create Pull Request
-        id: cpr
-        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
-        continue-on-error: true
-        uses: peter-evans/create-pull-request@v3
-        with:
-          token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}
-          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
-          signoff: false
-          branch: update_builder_${{ github.sha }}
-          delete-branch: true
-          title: '[automated] Update Builder gpu image changes'
-          body: |
-            Update Builder gpu image changes
-            See changes: https://github.com/milvus-io/milvus/commit/${{ github.sha }}
-            Signed-off-by: ${{ github.actor }} ${{ github.actor }}@users.noreply.github.com
-      - name: Check outputs
-        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
-        run: |
-          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
-          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
diff --git a/.github/workflows/publish-gpu-builder.yaml b/.github/workflows/publish-gpu-builder.yaml
@@ -0,0 +1,94 @@
+name: Publish Gpu Builder
+# TODO: do not trigger action for some document file update
+
+# This workflow is triggered on pushes or pull request to the repository.
+on:
+  push:
+    # file paths to consider in the event. Optional; defaults to all.
+    paths:
+      - 'build/docker/builder/gpu/**'
+      - '.github/workflows/publish-gpu-builder.yaml'
+      - '!**.md'
+  pull_request:
+    # file paths to consider in the event. Optional; defaults to all.
+    paths:
+      - 'build/docker/builder/gpu/**'
+      - '.github/workflows/publish-gpu-builder.yaml'
+      - '!**.md'
+
+jobs:
+  publish-gpu-builder:
+    name: ${{ matrix.arch }} ${{ matrix.os }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 500
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu20.04]
+        arch: [amd64&arm64]
+    env:
+      OS_NAME: ${{ matrix.os }}
+      IMAGE_ARCH: ${{ matrix.arch }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Get version from system time after release step
+        id: extracter
+        run: |
+          echo "::set-output name=version::$(date +%Y%m%d)"
+          echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+      # - name: Setup upterm session
+      #   uses: lhotari/action-upterm@v1
+      - name: Docker Pull
+        shell: bash
+        run: |
+          docker run --rm --privileged tonistiigi/binfmt:latest --install arm64
+          docker buildx ls
+          docker buildx create --use --name=milvus --driver docker-container
+      - name: Docker Build
+        if: success() && github.event_name == 'pull_request' && github.repository == 'milvus-io/milvus'
+        shell: bash
+        run: |
+          docker buildx ls
+          docker buildx build --platform linux/amd64,linux/arm64 -t milvusdb/milvus-env:gpu-${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
+      - name: Docker Build&Push
+        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
+        shell: bash
+        run: |
+          docker buildx ls
+          docker login -u ${{ secrets.DOCKERHUB_USER }} \
+                       -p ${{ secrets.DOCKERHUB_TOKEN }}
+          docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:gpu-${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
+          docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:gpu-${OS_NAME}-latest -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
+      - name: Update Builder Image Changes
+        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
+        continue-on-error: true
+        shell: bash
+        run: |
+          sed -i "s#^GPU_DATE_VERSION=.*#GPU_DATE_VERSION=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" .env
+          sed -i "s#^LATEST_GPU_DATE_VERSION=.*#LATEST_GPU_DATE_VERSION=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" .env
+          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git add .env
+          git commit -m "Update Builder gpu image changes"
+      - name: Create Pull Request
+        id: cpr
+        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
+        continue-on-error: true
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}
+          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
+          signoff: false
+          branch: update_gpu_builder_${{ github.sha }}
+          delete-branch: true
+          title: '[automated] Update Builder gpu image changes'
+          body: |
+            Update Builder gpu image changes
+            See changes: https://github.com/milvus-io/milvus/commit/${{ github.sha }}
+            Signed-off-by: ${{ github.actor }} ${{ github.actor }}@users.noreply.github.com
+      - name: Check outputs
+        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
+        run: |
+          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
+          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -14,6 +14,8 @@ services:
     build:
       context: .
       dockerfile: build/docker/builder/cpu/${OS_NAME}/Dockerfile
+      args:
+        TARGETARCH: ${IMAGE_ARCH}
       cache_from:
         - ${IMAGE_REPO}/milvus-env:${OS_NAME}-${LATEST_DATE_VERSION}
     platform: linux/${IMAGE_ARCH}
@@ -52,6 +54,8 @@ services:
     build:
       context: .
       dockerfile: build/docker/builder/gpu/${OS_NAME}/Dockerfile
+      args:
+        TARGETARCH: ${IMAGE_ARCH}
       cache_from:
         - ${IMAGE_REPO}/milvus-env:gpu-${OS_NAME}-${LATEST_GPU_DATE_VERSION}
     # user: {{ CURRENT_ID }}

diff --git a/internal/core/src/storage/azure-blob-storage/AzureBlobChunkManager.cpp b/internal/core/src/storage/azure-blob-storage/AzureBlobChunkManager.cpp
@@ -21,18 +21,6 @@
 
 namespace azure {
 
-std::string
-GetTenantId() {
-    return std::getenv("AZURE_TENANT_ID");
-}
-std::string
-GetClientId() {
-    return std::getenv("AZURE_CLIENT_ID");
-}
-std::string
-GetTokenFilePath() {
-    return std::getenv("AZURE_FEDERATED_TOKEN_FILE");
-}
 std::string
 GetConnectionString(const std::string& access_key_id,
                     const std::string& access_key_value,
@@ -78,9 +66,10 @@ AzureBlobChunkManager::AzureBlobChunkManager(
     bool useIAM) {
     requestTimeoutMs_ = requestTimeoutMs;
     if (useIAM) {
+        Azure::Identity::WorkloadIdentityCredentialOptions options;
         auto workloadIdentityCredential =
             std::make_shared<Azure::Identity::WorkloadIdentityCredential>(
-                GetTenantId(), GetClientId(), GetTokenFilePath());
+                options);
         client_ = std::make_shared<Azure::Storage::Blobs::BlobServiceClient>(
             "https://" + access_key_id + ".blob." + address + "/",
             workloadIdentityCredential);

diff --git a/internal/core/src/storage/azure-blob-storage/cmake-modules/AzureVcpkg.cmake b/internal/core/src/storage/azure-blob-storage/cmake-modules/AzureVcpkg.cmake
@@ -6,6 +6,7 @@ set(AZ_ROOT_DIR "${CMAKE_CURRENT_LIST_DIR}/..")
 
 macro(az_vcpkg_integrate)
   message("Vcpkg integrate step.")
+
   # AUTO CMAKE_TOOLCHAIN_FILE:
   #   User can call `cmake -DCMAKE_TOOLCHAIN_FILE="path_to_the_toolchain"` as the most specific scenario.
   #   As the last alternative (default case), Azure SDK will automatically clone VCPKG folder and set toolchain from there.
@@ -17,7 +18,7 @@ macro(az_vcpkg_integrate)
       message("AZURE_SDK_DISABLE_AUTO_VCPKG is not defined. Fetch a local copy of vcpkg.")
       # GET VCPKG FROM SOURCE
       #  User can set env var AZURE_SDK_VCPKG_COMMIT to pick the VCPKG commit to fetch
-      set(VCPKG_COMMIT_STRING 71d875654e32ee216b0b7e0dc684e589dffa1b1c) # default SDK tested commit
+      set(VCPKG_COMMIT_STRING dc3c55f092c96fb3f1dcdff84e6a99f947ea4165) # default SDK tested commit
       if(DEFINED ENV{AZURE_SDK_VCPKG_COMMIT})
         message("AZURE_SDK_VCPKG_COMMIT is defined. Using that instead of the default.")
         set(VCPKG_COMMIT_STRING "$ENV{AZURE_SDK_VCPKG_COMMIT}") # default SDK tested commit
@@ -26,15 +27,15 @@ macro(az_vcpkg_integrate)
       include(FetchContent)
       FetchContent_Declare(
           vcpkg
-          GIT_REPOSITORY      https://github.com/milvus-io/vcpkg.git
+          GIT_REPOSITORY      https://github.com/microsoft/vcpkg.git
           GIT_TAG             ${VCPKG_COMMIT_STRING}
           )
       FetchContent_GetProperties(vcpkg)
       # make sure to pull vcpkg only once.
-      if(NOT vcpkg_POPULATED) 
+      if(NOT vcpkg_POPULATED)
           FetchContent_Populate(vcpkg)
       endif()
-      # use the vcpkg source path 
+      # use the vcpkg source path
       set(CMAKE_TOOLCHAIN_FILE "${vcpkg_SOURCE_DIR}/scripts/buildsystems/vcpkg.cmake" CACHE STRING "")
     endif()
   endif()
@@ -43,6 +44,7 @@ macro(az_vcpkg_integrate)
   if(DEFINED ENV{VCPKG_DEFAULT_TRIPLET} AND NOT DEFINED VCPKG_TARGET_TRIPLET)
     set(VCPKG_TARGET_TRIPLET "$ENV{VCPKG_DEFAULT_TRIPLET}" CACHE STRING "")
   endif()
+  message("Vcpkg integrate step - DONE.")
 endmacro()
 
 macro(az_vcpkg_portfile_prep targetName fileName contentToRemove)
@@ -166,4 +168,4 @@ macro(az_vcpkg_export targetName macroNamePart dllImportExportHeaderPath)
 
   # Export all the installs above as package.
   export(PACKAGE "${targetName}-cpp")
-endmacro()
+endmacro()
diff --git a/internal/core/src/storage/azure-blob-storage/test/CMakeLists.txt b/internal/core/src/storage/azure-blob-storage/test/CMakeLists.txt
@@ -4,4 +4,5 @@
 project(azure-blob-test)
 
 add_executable(azure-blob-test test_azure_blob_chunk_manager.cpp ../AzureBlobChunkManager.cpp)
-target_link_libraries(azure-blob-test PRIVATE Azure::azure-identity Azure::azure-storage-blobs)
+find_package(GTest CONFIG REQUIRED)
+target_link_libraries(azure-blob-test PRIVATE Azure::azure-identity Azure::azure-storage-blobs GTest::gtest)