feat: [skip e2e] effective way to use cache (#34859)

pr: #34774 issue: #34876 Signed-off-by: Yellow Shine <[email protected]>
milvus-io · Jul 23, 2024 · 0951fe4 · 0951fe4
1 parent 6986dfd
commit 0951fe4
Show file tree

Hide file tree

Showing 3 changed files with 129 additions and 4 deletions.
diff --git a/.github/actions/cache-restore/action.yaml b/.github/actions/cache-restore/action.yaml
@@ -0,0 +1,49 @@
+name: 'Milvus Cache'
+description: ''
+inputs:
+  os:
+    description: 'OS name'
+    required: true
+    default: 'ubuntu20.04'
+  kind:
+    description: 'Cache kind'
+    required: false
+    default: 'all'
+runs:
+  using: "composite"
+  steps:
+  - name: Generate CCache Hash
+    env:
+      CORE_HASH: ${{ hashFiles( 'internal/core/**/*.cpp', 'internal/core/**/*.cc', 'internal/core/**/*.c', 'internal/core/**/*.h', 'internal/core/**/*.hpp', 'internal/core/**/CMakeLists.txt') }}
+    run: |
+      echo "corehash=${CORE_HASH}" >> $GITHUB_ENV
+      echo "Set CCache hash to ${CORE_HASH}"
+    shell: bash
+  - name: Cache CCache Volumes
+    if: ${{ inputs.kind == 'all' || inputs.kind == 'cpp' }}
+    uses: actions/cache/restore@v4
+    with:
+      path: .docker/amd64-${{ inputs.os }}-ccache
+      key: ${{ inputs.os }}-ccache-${{ env.corehash }}
+      restore-keys: ${{ inputs.os }}-ccache-
+  - name: Cache Conan Packages
+    if: ${{ inputs.kind == 'all' || inputs.kind == 'cpp' }}
+    uses: actions/cache/restore@v4
+    with:
+      path: .docker/amd64-${{ inputs.os }}-conan
+      key: ${{ inputs.os }}-conan-${{ hashFiles('internal/core/conanfile.*') }}
+      restore-keys: ${{ inputs.os }}-conan-
+  - name: Cache Third Party
+    if: ${{ inputs.kind == 'all' || inputs.kind == 'thirdparty' }}
+    uses: actions/cache/restore@v4
+    with:
+      path: .docker/thirdparty
+      key: ${{ inputs.os }}-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }}
+      restore-keys: ${{ inputs.os }}-thirdparty-
+  - name: Cache Go Mod Volumes
+    if: ${{ inputs.kind == 'all' || inputs.kind == 'go' }}
+    uses: actions/cache/restore@v4
+    with:
+      path: .docker/amd64-${{ inputs.os }}-go-mod
+      key: ${{ inputs.os }}-go-mod-${{ hashFiles('go.sum, */go.sum') }}
+      restore-keys: ${{ inputs.os }}-go-mod-
diff --git a/.github/actions/cache-save/action.yaml b/.github/actions/cache-save/action.yaml
@@ -0,0 +1,49 @@
+name: 'Milvus Cache'
+description: ''
+inputs:
+  os:
+    description: 'OS name'
+    required: true
+    default: 'ubuntu20.04'
+  kind:
+    description: 'Cache kind'
+    required: false
+    default: 'all'
+runs:
+  using: "composite"
+  steps:
+  - name: Generate CCache Hash
+    env:
+      CORE_HASH: ${{ hashFiles( 'internal/core/**/*.cpp', 'internal/core/**/*.cc', 'internal/core/**/*.c', 'internal/core/**/*.h', 'internal/core/**/*.hpp', 'internal/core/**/CMakeLists.txt') }}
+    run: |
+      echo "corehash=${CORE_HASH}" >> $GITHUB_ENV
+      echo "Set CCache hash to ${CORE_HASH}"
+    shell: bash
+  - name: Cache CCache Volumes
+    if: ${{ inputs.kind == 'all' || inputs.kind == 'cpp' }}
+    uses: actions/cache/save@v4
+    with:
+      path: .docker/amd64-${{ inputs.os }}-ccache
+      key: ${{ inputs.os }}-ccache-${{ env.corehash }}
+      restore-keys: ${{ inputs.os }}-ccache-
+  - name: Cache Conan Packages
+    if: ${{ inputs.kind == 'all' || inputs.kind == 'cpp' }}
+    uses: actions/cache/save@v4
+    with:
+      path: .docker/amd64-${{ inputs.os }}-conan
+      key: ${{ inputs.os }}-conan-${{ hashFiles('internal/core/conanfile.*') }}
+      restore-keys: ${{ inputs.os }}-conan-
+  - name: Cache Third Party
+    if: ${{ inputs.kind == 'all' || inputs.kind == 'thirdparty' }}
+    uses: actions/cache/save@v4
+    with:
+      path: .docker/thirdparty
+      key: ${{ inputs.os }}-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }}
+      restore-keys: ${{ inputs.os }}-thirdparty-
+  - name: Cache Go Mod Volumes
+    if: ${{ inputs.kind == 'all' || inputs.kind == 'go' }}
+    uses: actions/cache/save@v4
+    with:
+      path: .docker/amd64-${{ inputs.os }}-go-mod
+      key: ${{ inputs.os }}-go-mod-${{ hashFiles('go.sum, */go.sum') }}
+      restore-keys: ${{ inputs.os }}-go-mod-
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -90,7 +90,7 @@ jobs:
           echo "useasan=ON" >> $GITHUB_ENV
           echo "Setup USE_ASAN to true since cpp file(s) changed"
       - name: Download Caches
-        uses: ./.github/actions/cache
+        uses: ./.github/actions/cache-restore
         with:
           os: 'ubuntu22.04'
           kind: 'cpp'
@@ -104,6 +104,13 @@ jobs:
         with:
           name: code
           path: code.zip
+      - name: Save Caches
+        uses: ./.github/actions/cache-save
+        if: github.event_name != 'pull_request'
+        with:
+          os: 'ubuntu22.04'
+          kind: 'cpp'
+
   UT-Cpp:
     name: UT for Cpp
     needs: Build
@@ -127,7 +134,7 @@ jobs:
           unzip code.zip
           rm code.zip
       - name: Download Caches
-        uses: ./.github/actions/cache
+        uses: ./.github/actions/cache-restore
         with:
           os: 'ubuntu22.04'
           kind: 'cpp'
@@ -150,6 +157,12 @@ jobs:
             ./lcov_output.info
             *.info
             *.out
+      - name: Save Caches
+        uses: ./.github/actions/cache-save
+        if: github.event_name != 'pull_request'
+        with:
+          os: 'ubuntu22.04'
+          kind: 'cpp'
   UT-Go:
     name: UT for Go
     needs: Build
@@ -173,7 +186,7 @@ jobs:
           unzip code.zip
           rm code.zip
       - name: Download Caches
-        uses: ./.github/actions/cache
+        uses: ./.github/actions/cache-restore
         with:
           os: 'ubuntu22.04'
           kind: 'go'
@@ -195,6 +208,13 @@ jobs:
             ./lcov_output.info
             *.info
             *.out
+      - name: Save Caches
+        uses: ./.github/actions/cache-save
+        if: github.event_name != 'pull_request'
+        with:
+          os: 'ubuntu22.04'
+          kind: 'go'
+
   integration-test:
     name: Integration Test
     needs: Build
@@ -218,7 +238,7 @@ jobs:
           unzip code.zip
           rm code.zip
       - name: Download Caches
-        uses: ./.github/actions/cache
+        uses: ./.github/actions/cache-restore
         with:
           os: 'ubuntu22.04'
           kind: 'go'
@@ -239,6 +259,13 @@ jobs:
             ./it_coverage.txt
             *.info
             *.out
+      - name: Save Caches
+        uses: ./.github/actions/cache-save
+        if: github.event_name != 'pull_request'
+        with:
+          os: 'ubuntu22.04'
+          kind: 'go'
+
   codecov:
     name: Upload Code Coverage
     needs: [UT-Cpp, UT-Go, integration-test]