From dc8db45db4b594b94f2089f49245a2acd8934f60 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 18 Apr 2024 20:49:34 +0200 Subject: [PATCH] [GHA] Replace sscache with compressed ccache (#24039) ### Details: - use custom compressed action to restore/save cache, it should solve network related issued with sccache and speedup the build from cache (from 10 to 6 minutes) - ### Tickets: - *ticket-id* --- .github/workflows/windows.yml | 62 +++++------- .../windows_conditional_compilation.yml | 96 +++++++------------ 2 files changed, 60 insertions(+), 98 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index a457a43558f8db..0b9f8c688c4010 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -51,13 +51,11 @@ jobs: env: CMAKE_BUILD_TYPE: 'Release' CMAKE_GENERATOR: 'Ninja Multi-Config' - CMAKE_CXX_COMPILER_LAUNCHER: sccache - CMAKE_C_COMPILER_LAUNCHER: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: 1 - SCCACHE_SERVER_PORT: 35555 - SCCACHE_IDLE_TIMEOUT: 0 - SCCACHE_ERROR_LOG: "${{ github.workspace }}\\openvino\\sccache_log.txt" - SCCACHE_LOG: warn + CMAKE_CXX_COMPILER_LAUNCHER: ccache + CMAKE_C_COMPILER_LAUNCHER: ccache + CCACHE_REMOTE_DIR: "C:\\mount\\caches\\ccache\\windows2022_x86_64\\${{ github.base_ref || github.ref_name }}" + CCACHE_DIR: ${{ github.workspace }}\\ccache + CCACHE_MAXSIZE: 3G OPENVINO_REPO: "${{ github.workspace }}\\openvino" OPENVINO_CONTRIB_REPO: "${{ github.workspace }}\\openvino_contrib" INSTALL_DIR: "${{ github.workspace }}\\openvino_install" @@ -65,7 +63,6 @@ jobs: INSTALL_TEST_DIR: "${{ github.workspace }}\\tests_install" BUILD_DIR: "${{ github.workspace }}\\openvino_build" # TODO: specify version of compiler here - SCCACHE_AZURE_KEY_PREFIX: windows2022_x86_64_Release if: ${{ !needs.smart_ci.outputs.skip_workflow && github.event_name != 'merge_group' }} steps: @@ -124,10 +121,12 @@ jobs: # For getting rid of SSL issues during model downloading for unit tests python3 -m pip install certifi - - name: Install sccache - uses: mozilla-actions/sccache-action@v0.0.4 - with: - version: "v0.7.5" + - name: Download and install ccache + run: | + Invoke-WebRequest -Uri 'https://github.com/ccache/ccache/releases/download/v4.9.1/ccache-4.9.1-windows-x86_64.zip' -OutFile 'ccache.zip' + Expand-Archive -Path 'ccache.zip' -DestinationPath 'C:\temp\ccache' + Move-Item -Path 'C:\temp\ccache\*' -Destination 'C:\ccache' + Add-Content -Path $env:GITHUB_PATH -Value "C:\ccache" - name: Install build dependencies run: | @@ -140,6 +139,17 @@ jobs: # Build # + - name: Setup ccache + uses: ./openvino/.github/actions/cache + with: + save-always: ${{ github.event_name == 'push' && 'true' || 'false' }} + cleanup-always: ${{ github.event_name == 'push' && 'true' || 'false' }} + cache-path: ${{ env.CCACHE_REMOTE_DIR }} + path: ${{ env.CCACHE_DIR }} + key: ${{ runner.os }}-${{ runner.arch }}-ccache-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-ccache + - name: Configure Developer Command Prompt for Microsoft Visual C++ uses: ilammy/msvc-dev-cmd@v1 @@ -162,27 +172,14 @@ jobs: -S ${{ env.OPENVINO_REPO }} ` -B ${{ env.BUILD_DIR }} - - name: Clean sccache stats - run: '& "$Env:SCCACHE_PATH" --zero-stats' - - # to get more information on the issue - # described in the next step - - name: Show which network ports are used - run: netstat -ban - - # the case is the following: - # sccache: error: An attempt was made to access a socket in a way forbidden by its access permissions. (os error 10013) - # This looks like the attempt to use - # a port below 1024 or a port - # which is occupied by another app - - name: Stop sccache server just in case - run: '& "$Env:SCCACHE_PATH" --stop-server' + - name: Clean ccache stats + run: '& ccache --zero-stats' - name: Cmake build - OpenVINO run: cmake --build ${{ env.BUILD_DIR }} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --verbose - - name: Show sccache stats - run: '& "$Env:SCCACHE_PATH" --show-stats' + - name: Show ccache stats + run: '& ccache --show-stats' - name: Cmake install - OpenVINO run: | @@ -231,13 +228,6 @@ jobs: # # Upload build artifacts and logs # - - name: Upload build logs - uses: actions/upload-artifact@v4 - if: always() - with: - name: build_logs - path: ${{ env.SCCACHE_ERROR_LOG }} - if-no-files-found: 'ignore' - name: Upload openvino package uses: actions/upload-artifact@v4 diff --git a/.github/workflows/windows_conditional_compilation.yml b/.github/workflows/windows_conditional_compilation.yml index 722d58ee9e7804..4b77934694c6bf 100644 --- a/.github/workflows/windows_conditional_compilation.yml +++ b/.github/workflows/windows_conditional_compilation.yml @@ -46,7 +46,7 @@ jobs: Build: needs: Smart_CI - timeout-minutes: 180 + timeout-minutes: 300 # 5 hours without cache defaults: run: shell: pwsh @@ -54,13 +54,11 @@ jobs: env: CMAKE_BUILD_TYPE: 'Release' CMAKE_GENERATOR: 'Ninja Multi-Config' - CMAKE_CXX_COMPILER_LAUNCHER: sccache - CMAKE_C_COMPILER_LAUNCHER: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: 1 - SCCACHE_SERVER_PORT: 35555 - SCCACHE_IDLE_TIMEOUT: 0 - SCCACHE_ERROR_LOG: "${{ github.workspace }}\\openvino\\sccache_log.txt" - SCCACHE_LOG: warn + CMAKE_CXX_COMPILER_LAUNCHER: ccache + CMAKE_C_COMPILER_LAUNCHER: ccache + CCACHE_REMOTE_DIR: "C:\\mount\\caches\\ccache\\windows2022_x86_64_itt\\${{ github.base_ref || github.ref_name }}" + CCACHE_DIR: ${{ github.workspace }}\\ccache + CCACHE_MAXSIZE: 3G OPENVINO_REPO: "${{ github.workspace }}\\openvino" INSTALL_DIR: "${{ github.workspace }}\\openvino_install" INSTALL_TEST_DIR: "${{ github.workspace }}\\tests_install" @@ -68,7 +66,6 @@ jobs: MODELS_PATH: "${{ github.workspace }}\\testdata" SELECTIVE_BUILD_STAT_DIR: "${{ github.workspace }}\\selective_build_stat" # TODO: specify version of compiler here - SCCACHE_AZURE_KEY_PREFIX: windows2022_x86_64_itt_Release if: ${{ !needs.smart_ci.outputs.skip_workflow && github.event_name != 'merge_group' }} steps: @@ -104,10 +101,12 @@ jobs: should-setup-pip-paths: 'false' self-hosted-runner: 'false' - - name: Install sccache - uses: mozilla-actions/sccache-action@v0.0.4 - with: - version: "v0.7.5" + - name: Download and install ccache + run: | + Invoke-WebRequest -Uri 'https://github.com/ccache/ccache/releases/download/v4.9.1/ccache-4.9.1-windows-x86_64.zip' -OutFile 'ccache.zip' + Expand-Archive -Path 'ccache.zip' -DestinationPath 'C:\temp\ccache' + Move-Item -Path 'C:\temp\ccache\*' -Destination 'C:\ccache' + Add-Content -Path $env:GITHUB_PATH -Value "C:\ccache" - name: Install build dependencies run: | @@ -138,6 +137,17 @@ jobs: # Build # + - name: Setup ccache + uses: ./openvino/.github/actions/cache + with: + save-always: ${{ github.event_name == 'push' && 'true' || 'false' }} + cleanup-always: ${{ github.event_name == 'push' && 'true' || 'false' }} + cache-path: ${{ env.CCACHE_REMOTE_DIR }} + path: ${{ env.CCACHE_DIR }} + key: ${{ runner.os }}-${{ runner.arch }}-ccache-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-ccache + - name: Configure Developer Command Prompt for Microsoft Visual C++ uses: ilammy/msvc-dev-cmd@v1 @@ -159,29 +169,16 @@ jobs: -S ${{ env.OPENVINO_REPO }} ` -B ${{ env.BUILD_DIR }} - - name: Clean sccache stats - run: '& "$Env:SCCACHE_PATH" --zero-stats' - - # to get more information on the issue - # described in the next step - - name: Show which network ports are used - run: netstat -ban - - # the case is the following: - # sccache: error: An attempt was made to access a socket in a way forbidden by its access permissions. (os error 10013) - # This looks like the attempt to use - # a port below 1024 or a port - # which is occupied by another app - - name: Stop sccache server just in case - run: '& "$Env:SCCACHE_PATH" --stop-server' + - name: Clean ccache stats + run: '& ccache --zero-stats' - name: Cmake build - CC COLLECT run: | cmake --build ${{ env.BUILD_DIR }} --parallel 8 --config ${{ env.CMAKE_BUILD_TYPE }} && ` cmake --build ${{ env.BUILD_DIR }} --parallel 8 --config ${{ env.CMAKE_BUILD_TYPE }} --target sea_itt_lib - - name: Show sccache stats - run: '& "$Env:SCCACHE_PATH" --show-stats' + - name: Show ccache stats + run: '& ccache --show-stats' - name: Cmake install - OpenVINO run: cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }} -P ${{ env.BUILD_DIR }}/cmake_install.cmake @@ -242,13 +239,6 @@ jobs: # # Upload build artifacts and logs # - - name: Upload build logs - uses: actions/upload-artifact@v4 - if: always() - with: - name: build_logs - path: ${{ env.SCCACHE_ERROR_LOG }} - if-no-files-found: 'ignore' - name: Upload selective build statistics package if: ${{ always() }} @@ -275,16 +265,10 @@ jobs: runs-on: aks-win-8-cores-64gb env: CMAKE_BUILD_TYPE: 'Release' - CMAKE_CXX_COMPILER_LAUNCHER: sccache - CMAKE_C_COMPILER_LAUNCHER: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: 1 - SCCACHE_SERVER_PORT: 35555 - SCCACHE_IDLE_TIMEOUT: 0 OPENVINO_REPO: "${{ github.workspace }}\\openvino" BUILD_DIR: "${{ github.workspace }}\\openvino_build" MODELS_PATH: "${{ github.workspace }}\\testdata" SELECTIVE_BUILD_STAT_DIR: "${{ github.workspace }}\\selective_build_stat" - SCCACHE_AZURE_KEY_PREFIX: windows2022_x86_64_cc_Release if: "!needs.smart_ci.outputs.skip_workflow" steps: @@ -318,11 +302,6 @@ jobs: should-setup-pip-paths: 'false' self-hosted-runner: 'false' - - name: Install sccache - uses: mozilla-actions/sccache-action@v0.0.4 - with: - version: "v0.5.4" - - name: CMake configure - CC ON run: | cmake ` @@ -340,15 +319,9 @@ jobs: -S ${{ env.OPENVINO_REPO }} ` -B ${{ env.BUILD_DIR }} - - name: Clean sccache stats - run: '& "$Env:SCCACHE_PATH" --zero-stats' - - name: Cmake build - CC ON run: cmake --build ${{ env.BUILD_DIR }} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target benchmark_app - - name: Show sccache stats - run: '& "$Env:SCCACHE_PATH" --show-stats' - - name: List bin files shell: cmd run: dir ${{ env.OPENVINO_REPO }}\bin\ /s @@ -402,14 +375,13 @@ jobs: - name: Install python dependencies for run_parallel.py run: python3 -m pip install -r ${{ env.INSTALL_TEST_DIR }}/layer_tests_summary/requirements.txt - # Windows pipeline is in nightly mode, uncomment once there is a consistent cache creation - # - name: Restore tests execution time - # uses: actions/cache/restore@v3 - # with: - # path: ${{ env.PARALLEL_TEST_CACHE }} - # key: ${{ runner.os }}-tests-functional-cpu-stamp-${{ github.sha }} - # restore-keys: | - # ${{ runner.os }}-tests-functional-cpu-stamp + - name: Restore tests execution time + uses: actions/cache/restore@v4 + with: + path: ${{ env.PARALLEL_TEST_CACHE }} + key: ${{ runner.os }}-tests-functional-cpu-stamp-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-tests-functional-cpu-stamp - name: Intel CPU plugin func tests (parallel) shell: cmd