From 6abf878cea50847ea4321ee3dc9c7c52e0a18420 Mon Sep 17 00:00:00 2001 From: Georgi Petrov <32372905+G-D-Petrov@users.noreply.github.com> Date: Mon, 2 Dec 2024 15:44:44 +0200 Subject: [PATCH] Test new manylinux (#2025) #### Reference Issues/PRs #### What does this implement or fix? This includes: - fix for the Windows builds - fix for supporting Node 20 - Skipping the tests that use NFS storage - Removed the dataframe logging which is causing AHL problems with their logs #### Any other comments? #### Checklist
Checklist for code changes... - [ ] Have you updated the relevant docstrings, documentation and copyright notice? - [ ] Is this contribution tested against [all ArcticDB's features](../docs/mkdocs/docs/technical/contributing.md)? - [ ] Do all exceptions introduced raise appropriate [error messages](https://docs.arcticdb.io/error_messages/)? - [ ] Are API changes highlighted in the PR description? - [ ] Is the PR labelled as enhancement or bug so it appears in autogenerated release notes?
--------- Co-authored-by: Vasil Pashov Co-authored-by: Georgi Petrov --- .github/actions/setup_deps/action.yml | 43 +++ .github/workflows/analysis_workflow.yml | 312 +++++++++--------- .github/workflows/benchmark_commits.yml | 7 +- .github/workflows/build.yml | 2 +- .github/workflows/build_steps.yml | 27 +- cpp/CMakePresets.json | 10 +- .../x64-windows-static-msvc.cmake | 4 + python/arcticdb/version_store/_store.py | 14 +- python/tests/integration/arcticdb/test_s3.py | 2 + 9 files changed, 242 insertions(+), 179 deletions(-) create mode 100644 .github/actions/setup_deps/action.yml create mode 100644 cpp/custom-triplets/x64-windows-static-msvc.cmake diff --git a/.github/actions/setup_deps/action.yml b/.github/actions/setup_deps/action.yml new file mode 100644 index 0000000000..d79e91c7d6 --- /dev/null +++ b/.github/actions/setup_deps/action.yml @@ -0,0 +1,43 @@ +name: 'Install GCC related dependencies' +description: 'Installs GCC related dependencies for building ArcticDB' +runs: + using: "composite" + steps: + - name: Install deps + shell: bash -l {0} + run: | + dnf update -y + dnf remove -y 'gcc-toolset-13-*' + dnf install -y zip flex bison gcc-toolset-10 gcc-toolset-10-gdb gcc-toolset-10-libatomic-devel krb5-devel cyrus-sasl-devel openssl-devel \ + unzip tar epel-release jq wget libcurl-devel python3 \ + python3-devel python3-pip perl-IPC-Cmd + + dnf groupinstall -y 'Development Tools' + + echo "CC=/opt/rh/gcc-toolset-10/root/bin/gcc" | tee -a $GITHUB_ENV + echo "CMAKE_C_COMPILER=/opt/rh/gcc-toolset-10/root/bin/gcc" | tee -a $GITHUB_ENV + echo "CXX=/opt/rh/gcc-toolset-10/root/bin/g++" | tee -a $GITHUB_ENV + echo "CMAKE_CXX_COMPILER=/opt/rh/gcc-toolset-10/root/bin/g++" | tee -a $GITHUB_ENV + echo "LD_LIBRARY_PATH=/opt/rh/gcc-toolset-10/root/usr/lib64:/opt/rh/gcc-toolset-10/root/usr/lib:/opt/rh/gcc-toolset-10/root/usr/lib64/dyninst" | tee -a $GITHUB_ENV + echo "/opt/rh/devtoolset-10/root/usr/bin" | tee -a $GITHUB_PATH + + echo $GITHUB_ENV + + echo $GITHUB_PATH + + dnf install -y mono-complete + + dnf clean all + + # sccache + sccache_ver=`curl -sL -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/mozilla/sccache/releases/latest | jq -r ".tag_name"` + + wget -nv https://github.com/mozilla/sccache/releases/download/$sccache_ver/sccache-$sccache_ver-x86_64-unknown-linux-musl.tar.gz \ + -O /tmp/sccache.tar.gz + tar -xvf /tmp/sccache.tar.gz -C /tmp/ + cp /tmp/sccache-*/sccache /usr/local/bin/ + rm /tmp/sccache.tar.gz + + which gcc + which g++ diff --git a/.github/workflows/analysis_workflow.yml b/.github/workflows/analysis_workflow.yml index 95e1491755..ac2e93cef1 100644 --- a/.github/workflows/analysis_workflow.yml +++ b/.github/workflows/analysis_workflow.yml @@ -31,15 +31,9 @@ jobs: outputs: matrix: ${{ steps.get_tags.outputs.commits }} - cibw_docker_image: - uses: ./.github/workflows/cibw_docker_image.yml - permissions: {packages: write} - with: - cibuildwheel_ver: "2.12.1" - force_update: false benchmark_commits: - needs: [get_commits_to_benchmark, cibw_docker_image] + needs: [get_commits_to_benchmark] strategy: fail-fast: false matrix: @@ -49,7 +43,6 @@ jobs: secrets: inherit with: commit: ${{ matrix.commits }} - cibw_image_tag: ${{ needs.cibw_docker_image.outputs.tag }} run_all_benchmarks: ${{ inputs.run_all_benchmarks || false }} run_on_pr_head: ${{ github.event_name == 'pull_request_target' }} @@ -106,172 +99,175 @@ jobs: python -m asv publish -v python -m asv gh-pages -v --rewrite - code_coverage: - needs: [cibw_docker_image] - runs-on: "ubuntu-22.04" - container: - image: ${{needs.cibw_docker_image.outputs.tag}} - services: - mongodb: - image: mongo:4.4 - ports: - - 27017:27017 - env: - VCPKG_NUGET_USER: ${{secrets.VCPKG_NUGET_USER || github.repository_owner}} - VCPKG_NUGET_TOKEN: ${{secrets.VCPKG_NUGET_TOKEN || secrets.GITHUB_TOKEN}} - VCPKG_MAN_NUGET_USER: ${{secrets.VCPKG_MAN_NUGET_USER}} # For forks to download pre-compiled dependencies from the Man repo - VCPKG_MAN_NUGET_TOKEN: ${{secrets.VCPKG_MAN_NUGET_TOKEN}} - ARCTIC_CMAKE_PRESET: linux-debug - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - steps: - - uses: actions/checkout@v3.3.0 - with: - submodules: recursive - ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }} # Note: This is dangerous if we run automatic CI on external PRs + # code_coverage: + # runs-on: "ubuntu-22.04" + # container: + # image: quay.io/pypa/manylinux_2_28_x86_64:latest + # services: + # mongodb: + # image: mongo:4.4 + # ports: + # - 27017:27017 + # env: + # VCPKG_NUGET_USER: ${{secrets.VCPKG_NUGET_USER || github.repository_owner}} + # VCPKG_NUGET_TOKEN: ${{secrets.VCPKG_NUGET_TOKEN || secrets.GITHUB_TOKEN}} + # VCPKG_MAN_NUGET_USER: ${{secrets.VCPKG_MAN_NUGET_USER}} # For forks to download pre-compiled dependencies from the Man repo + # VCPKG_MAN_NUGET_TOKEN: ${{secrets.VCPKG_MAN_NUGET_TOKEN}} + # ARCTIC_CMAKE_PRESET: linux-debug + # ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + # steps: + # - uses: actions/checkout@v3.3.0 + # with: + # submodules: recursive + # ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }} # Note: This is dangerous if we run automatic CI on external PRs - - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v1.1.0 - id: cpu-cores + # - name: Get number of CPU cores + # uses: SimenB/github-actions-cpu-cores@v1.1.0 + # id: cpu-cores - - name: Extra envs - run: | - . build_tooling/vcpkg_caching.sh # Linux follower needs another call in CIBW - echo -e "VCPKG_BINARY_SOURCES=$VCPKG_BINARY_SOURCES - VCPKG_ROOT=$PLATFORM_VCPKG_ROOT" | tee -a $GITHUB_ENV - cmake -P cpp/CMake/CpuCount.cmake | sed 's/^-- //' | tee -a $GITHUB_ENV - echo "ARCTICDB_CODE_COVERAGE_BUILD=1" | tee -a $GITHUB_ENV - env: - CMAKE_BUILD_PARALLEL_LEVEL: ${{vars.CMAKE_BUILD_PARALLEL_LEVEL}} + # - name: Install deps + # uses: ./.github/actions/setup_deps + + # - name: Extra envs + # shell: bash -l {0} + # run: | + # . build_tooling/vcpkg_caching.sh # Linux follower needs another call in CIBW + # echo -e "VCPKG_BINARY_SOURCES=$VCPKG_BINARY_SOURCES + # VCPKG_ROOT=$PLATFORM_VCPKG_ROOT" | tee -a $GITHUB_ENV + # cmake -P cpp/CMake/CpuCount.cmake | sed 's/^-- //' | tee -a $GITHUB_ENV + # echo "ARCTICDB_CODE_COVERAGE_BUILD=1" | tee -a $GITHUB_ENV + # env: + # CMAKE_BUILD_PARALLEL_LEVEL: ${{vars.CMAKE_BUILD_PARALLEL_LEVEL}} - - name: Prepare C++ compilation env - run: . build_tooling/prep_cpp_build.sh + # - name: Prepare C++ compilation env + # run: . build_tooling/prep_cpp_build.sh - - name: CMake compile - # We are pinning the version to 10.6 because >= 10.7, use node20 which is not supported in the container - uses: lukka/run-cmake@v10.6 - with: - cmakeListsTxtPath: ${{github.workspace}}/cpp/CMakeLists.txt - configurePreset: ${{env.ARCTIC_CMAKE_PRESET}} - buildPreset: ${{env.ARCTIC_CMAKE_PRESET}} - env: - ARCTICDB_DEBUG_FIND_PYTHON: ${{vars.ARCTICDB_DEBUG_FIND_PYTHON}} - python_impl_name: 'cp311' + # - name: CMake compile + # # We are pinning the version to 10.6 because >= 10.7, use node20 which is not supported in the container + # uses: lukka/run-cmake@v10.6 + # with: + # cmakeListsTxtPath: ${{github.workspace}}/cpp/CMakeLists.txt + # configurePreset: ${{env.ARCTIC_CMAKE_PRESET}} + # buildPreset: ${{env.ARCTIC_CMAKE_PRESET}} + # env: + # ARCTICDB_DEBUG_FIND_PYTHON: ${{vars.ARCTICDB_DEBUG_FIND_PYTHON}} + # python_impl_name: 'cp311' - - name: Run C++ Tests - shell: bash -l {0} - run: | - cd cpp/out/linux-debug-build/ - ls arcticdb - make -j ${{ steps.cpu-cores.outputs.count }} arcticdb_rapidcheck_tests - make -j ${{ steps.cpu-cores.outputs.count }} test_unit_arcticdb - ctest + # - name: Run C++ Tests + # shell: bash -l {0} + # run: | + # cd cpp/out/linux-debug-build/ + # ls arcticdb + # make -j ${{ steps.cpu-cores.outputs.count }} arcticdb_rapidcheck_tests + # make -j ${{ steps.cpu-cores.outputs.count }} test_unit_arcticdb + # ctest - # We are chainging the python here because we want to use the default python to build (it is devel version) - # and this python for the rest of the testing - - name: Select Python (Linux) - run: echo /opt/python/cp36-cp36m/bin >> $GITHUB_PATH + # # We are chainging the python here because we want to use the default python to build (it is devel version) + # # and this python for the rest of the testing + # - name: Select Python (Linux) + # run: echo /opt/python/cp36-cp36m/bin >> $GITHUB_PATH - - name: Install local dependencies with pip - shell: bash - run: | - python -m pip install --upgrade pip - ARCTIC_CMAKE_PRESET=skip pip install -ve .[Testing] + # - name: Install local dependencies with pip + # shell: bash + # run: | + # python -m pip install --upgrade pip + # ARCTIC_CMAKE_PRESET=skip pip install -ve .[Testing] - # - name: Test with pytest - # uses: ./.github/actions/run_local_pytest - # with: - # build_type: debug - # threads: 1 - # fast_tests_only: 0 - # other_params: '-m coverage run ' + # # - name: Test with pytest + # # uses: ./.github/actions/run_local_pytest + # # with: + # # build_type: debug + # # threads: 1 + # # fast_tests_only: 0 + # # other_params: '-m coverage run ' - - name: Get python Coverage report - shell: bash -l {0} - run: | - cd python - python -m coverage report -m | tee output.txt - python -m coverage html - zip -r python_cov.zip htmlcov/ + # - name: Get python Coverage report + # shell: bash -l {0} + # run: | + # cd python + # python -m coverage report -m | tee output.txt + # python -m coverage html + # zip -r python_cov.zip htmlcov/ - echo "PYTHON_COV_PERCENT=$(cat output.txt | grep 'TOTAL' | awk '{print $NF}' | tr -d '%')" >> $GITHUB_ENV + # echo "PYTHON_COV_PERCENT=$(cat output.txt | grep 'TOTAL' | awk '{print $NF}' | tr -d '%')" >> $GITHUB_ENV - - name: Run Gcovr manually post-pytest - shell: bash -l {0} - run: | - cd cpp/out/linux-debug-build/ - python -m pip install gcovr - mkdir coverage - python -m gcovr --txt --html-details coverage/index.html -e vcpkg_installed/ -e proto/ -e ../../third_party -e ../../arcticdb/util/test/ -r ../.. --exclude-throw-branches --exclude-unreachable-branches -u --exclude-function-lines | tee output.txt - zip -r coverage.zip coverage/ + # - name: Run Gcovr manually post-pytest + # shell: bash -l {0} + # run: | + # cd cpp/out/linux-debug-build/ + # python -m pip install gcovr + # mkdir coverage + # python -m gcovr --txt --html-details coverage/index.html -e vcpkg_installed/ -e proto/ -e ../../third_party -e ../../arcticdb/util/test/ -r ../.. --exclude-throw-branches --exclude-unreachable-branches -u --exclude-function-lines | tee output.txt + # zip -r coverage.zip coverage/ - echo "CPP_COV_PERCENT=$(cat output.txt | grep 'TOTAL' | awk '{print $NF}' | tr -d '%')" >> $GITHUB_ENV + # echo "CPP_COV_PERCENT=$(cat output.txt | grep 'TOTAL' | awk '{print $NF}' | tr -d '%')" >> $GITHUB_ENV - - name: Upload Coverage - uses: actions/upload-artifact@v3.1.3 - with: - name: cpp-coverage-artifact - path: cpp/out/linux-debug-build/coverage.zip + # - name: Upload Coverage + # uses: actions/upload-artifact@v3.1.3 + # with: + # name: cpp-coverage-artifact + # path: cpp/out/linux-debug-build/coverage.zip - - name: Upload Python Coverage - uses: actions/upload-artifact@v3.1.3 - with: - name: python-coverage-artifact - path: python/python_cov.zip + # - name: Upload Python Coverage + # uses: actions/upload-artifact@v3.1.3 + # with: + # name: python-coverage-artifact + # path: python/python_cov.zip - - name: Restore cached CPP Coverage Percentage from the previous run - id: cache-cov-restore - uses: actions/cache/restore@v3.3.2 - with: - path: prev_coverage.txt - key: coverage + # - name: Restore cached CPP Coverage Percentage from the previous run + # id: cache-cov-restore + # uses: actions/cache/restore@v3.3.2 + # with: + # path: prev_coverage.txt + # key: coverage - - name: Get and compare coverage if cache was restored - run: | - # if cache was restored, compare coverage - if [ -f coverage.txt ]; then - PREV_COVERAGE=$(cat prev_coverage.txt | cut -d' ' -f2) - echo "Previous coverage: $PREV_COVERAGE" - CURR_COVERAGE=${{env.CPP_COV_PERCENT}} - echo "CPP_COV_PREV_PERCENT=$PREV_COVERAGE" >> $GITHUB_ENV - echo "Current coverage: $CURR_COVERAGE" - if [ $CURR_COVERAGE -gt $PREV_COVERAGE ]; then - echo "Coverage increased" - elif [ $CURR_COVERAGE -lt $PREV_COVERAGE ]; then - echo "Coverage decreased" - else - echo "Coverage unchanged" - fi - fi + # - name: Get and compare coverage if cache was restored + # run: | + # # if cache was restored, compare coverage + # if [ -f coverage.txt ]; then + # PREV_COVERAGE=$(cat prev_coverage.txt | cut -d' ' -f2) + # echo "Previous coverage: $PREV_COVERAGE" + # CURR_COVERAGE=${{env.CPP_COV_PERCENT}} + # echo "CPP_COV_PREV_PERCENT=$PREV_COVERAGE" >> $GITHUB_ENV + # echo "Current coverage: $CURR_COVERAGE" + # if [ $CURR_COVERAGE -gt $PREV_COVERAGE ]; then + # echo "Coverage increased" + # elif [ $CURR_COVERAGE -lt $PREV_COVERAGE ]; then + # echo "Coverage decreased" + # else + # echo "Coverage unchanged" + # fi + # fi - - name: Save CPP Coverage Percentage to file - run: | - echo "Coverage: ${{ env.CPP_COV_PERCENT }}" > current_coverage.txt + # - name: Save CPP Coverage Percentage to file + # run: | + # echo "Coverage: ${{ env.CPP_COV_PERCENT }}" > current_coverage.txt - - name: Save the current CPP Coverage Percentage to the cache - id: cache-cov-save - uses: actions/cache/save@v3.3.2 - with: - path: current_coverage.txt - key: coverage + # - name: Save the current CPP Coverage Percentage to the cache + # id: cache-cov-save + # uses: actions/cache/save@v3.3.2 + # with: + # path: current_coverage.txt + # key: coverage - - name: Check percentage and send Slack notification - if: ${{ env.CPP_COV_PREV_PERCENT && env.CPP_COV_PERCENT && env.CPP_COV_PERCENT < env.CPP_COV_PREV_PERCENT }} - uses: slackapi/slack-github-action@v1.24.0 - with: - # For posting a rich message using Block Kit - payload: | - { - "text": "The CPP Code Coverage has been reduced", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "The CPP Code Coverage from the current run(${{ env.CPP_COV_PERCENT }}%) is lower the previous one(${{ env.CPP_COV_PREV_PERCENT }}%)." - } - } - ] - } - env: - SLACK_WEBHOOK_URL: ${{ secrets.ARCTICDB_DEV_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + # - name: Check percentage and send Slack notification + # if: ${{ env.CPP_COV_PREV_PERCENT && env.CPP_COV_PERCENT && env.CPP_COV_PERCENT < env.CPP_COV_PREV_PERCENT }} + # uses: slackapi/slack-github-action@v1.24.0 + # with: + # # For posting a rich message using Block Kit + # payload: | + # { + # "text": "The CPP Code Coverage has been reduced", + # "blocks": [ + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "The CPP Code Coverage from the current run(${{ env.CPP_COV_PERCENT }}%) is lower the previous one(${{ env.CPP_COV_PREV_PERCENT }}%)." + # } + # } + # ] + # } + # env: + # SLACK_WEBHOOK_URL: ${{ secrets.ARCTICDB_DEV_WEBHOOK_URL }} + # SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK diff --git a/.github/workflows/benchmark_commits.yml b/.github/workflows/benchmark_commits.yml index 7580ef5614..a27177db93 100644 --- a/.github/workflows/benchmark_commits.yml +++ b/.github/workflows/benchmark_commits.yml @@ -4,7 +4,6 @@ on: inputs: run_all_benchmarks: {required: true, type: boolean, description: Run all benchmarks or just the one for the given commit} commit: {required: true, type: string, description: commit hash that will be benchmarked} - cibw_image_tag: {required: true, type: string, description: Linux only. As built by cibw_docker_image.yml workflow} run_on_pr_head: {required: false, default: false, type: boolean, description: Specifies if the benchmark should run on PR head branch} jobs: start_ec2_runner: @@ -20,7 +19,7 @@ jobs: always() && !cancelled() runs-on: ${{ needs.start_ec2_runner.outputs.label }} - container: ${{ inputs.cibw_image_tag}} + container: quay.io/pypa/manylinux_2_28_x86_64:latest env: # this is potentially overflowing the cache, so should be looked into after we address issue #1057 SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} # Setting this env var enables the caching @@ -44,9 +43,13 @@ jobs: with: version: "v0.4.0" + - name: Install deps + uses: ./.github/actions/setup_deps + # We are changing the python here because we want to use the default python to build (it is devel version) # and this python for the rest of the testing - name: Select Python (Linux) + shell: bash -el {0} run: | ls /opt/python echo /opt/python/cp36-cp36m/bin >> $GITHUB_PATH diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7097864ad..210a5813eb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,7 +52,7 @@ jobs: mongodb: image: "mongo:4.4" container: - image: ${{needs.cibw_docker_image.outputs.tag}} + image: quay.io/pypa/manylinux_2_28_x86_64 volumes: - /:/mnt windows_matrix: diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index 333ea47d63..d08399d171 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -87,11 +87,11 @@ jobs: maximum-size: 6GB disk-root: "D:" # This is also the checkout directory. Total size 12GB. continue-on-error: true - - - name: Enable Windows compiler commands - if: matrix.os == 'windows' - uses: ilammy/msvc-dev-cmd@v1.12.1 + - name: Install deps + if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' + uses: ./.github/actions/setup_deps + - name: Extra envs # This has to come after msvc-dev-cmd to overwrite the bad VCPKG_ROOT it sets run: | @@ -122,6 +122,23 @@ jobs: if: inputs.job_type != 'build-python-wheels' run: . build_tooling/prep_cpp_build.sh # Also applies to Windows + # When a GitHub Windows image gets update the MSVC compiler also can get updated. New compilers can have compilation errors in Arctic or in the VCPKG dependencies. + # We needd to pin a particular MSVC so that runner updates don't affect us. + # When the MSVC version is update custom-triplets/x64-windows-static-msvc.cmake must also be updated with the correct toolsed version. + - name: Install Required MSVC + if: matrix.os == 'windows' + run: | + choco install -y -f visualstudio2022buildtools --version=117.11.4 --params "--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 --installChannelUri https://aka.ms/vs/17/release/390666095_1317821361/channel" + choco install -y ninja + + - name: Enable Windows compiler commands + if: matrix.os == 'windows' + uses: TheMrMilchmann/setup-msvc-dev@v3 + with: + arch: x64 + toolset: 14.41 + vs-path: 'C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools' + - name: CMake compile if: inputs.job_type != 'build-python-wheels' # We are pinning the version to 10.6 because >= 10.7, use node20 which is not supported in the container @@ -255,7 +272,7 @@ jobs: ${{fromJSON(inputs.matrix)}} name: ${{matrix.type}}${{matrix.python_deps_id}} runs-on: ${{matrix.distro}} - container: ${{matrix.os == 'linux' && needs.compile.outputs.manylinux_image || null}} + container: ${{matrix.os == 'linux' && matrix.container || null}} defaults: run: {shell: bash} services: ${{matrix.test_services}} diff --git a/cpp/CMakePresets.json b/cpp/CMakePresets.json index 6368c19ba7..07d1fba724 100644 --- a/cpp/CMakePresets.json +++ b/cpp/CMakePresets.json @@ -63,7 +63,9 @@ "generator": "Ninja", "environment": { "cmakepreset_expected_host_system": "Windows" }, "cacheVariables": { - "ARCTICDB_USE_PCH": "ON" + "ARCTICDB_USE_PCH": "ON", + "VCPKG_OVERLAY_TRIPLETS": "custom-triplets", + "VCPKG_TARGET_TRIPLET": "x64-windows-static-msvc" } }, { @@ -80,8 +82,7 @@ }, "cacheVariables": { "CMAKE_C_COMPILER": "cl", - "CMAKE_CXX_COMPILER": "cl", - "VCPKG_TARGET_TRIPLET": "x64-windows-static" + "CMAKE_CXX_COMPILER": "cl" } }, { @@ -97,7 +98,8 @@ "installDir": "${sourceDir}/out/install", "cacheVariables": { "CMAKE_CXX_FLAGS": "/MP", - "VCPKG_TARGET_TRIPLET": "x64-windows-static", + "VCPKG_OVERLAY_TRIPLETS": "custom-triplets", + "VCPKG_TARGET_TRIPLET": "x64-windows-static-msvc", "ARCTICDB_PYTHON_EXPLICIT_LINK": "ON" } }, diff --git a/cpp/custom-triplets/x64-windows-static-msvc.cmake b/cpp/custom-triplets/x64-windows-static-msvc.cmake new file mode 100644 index 0000000000..58ade0255f --- /dev/null +++ b/cpp/custom-triplets/x64-windows-static-msvc.cmake @@ -0,0 +1,4 @@ +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE static) +set(VCPKG_LIBRARY_LINKAGE static) +set(VCPKG_PLATFORM_TOOLSET_VERSION 14.41) \ No newline at end of file diff --git a/python/arcticdb/version_store/_store.py b/python/arcticdb/version_store/_store.py index b13b577fd0..e0e6a8f2d2 100644 --- a/python/arcticdb/version_store/_store.py +++ b/python/arcticdb/version_store/_store.py @@ -365,18 +365,14 @@ def _try_normalize( ) except ArcticDbNotYetImplemented as ex: raise ArcticDbNotYetImplemented( - f"Not supported: normalizing\n" - f"symbol: {symbol}\n" - f"data:\n" - f"{dataframe}\n" - f"metadata:\n" - f"{metadata}\n" - f"Reason:\n" - f"{ex}\n" + f"Not supported: normalizing" + f"symbol: {symbol}" + f"Reason:" + f"{ex}" f"{norm_failure_options_msg}" ) except Exception as ex: - log.error("Error while normalizing symbol={}, data={}, metadata={}, {}", symbol, dataframe, metadata, ex) + log.error("Error while normalizing symbol={}, {}", symbol, ex) raise ArcticNativeException(str(ex)) if norm_meta is None: diff --git a/python/tests/integration/arcticdb/test_s3.py b/python/tests/integration/arcticdb/test_s3.py index d33e4205ac..22e5e44100 100644 --- a/python/tests/integration/arcticdb/test_s3.py +++ b/python/tests/integration/arcticdb/test_s3.py @@ -60,6 +60,7 @@ def test_s3_running_on_aws_fast_check(lib_name, s3_storage_factory, run_on_aws): assert lib_tool.inspect_env_variable("AWS_EC2_METADATA_DISABLED") == "true" +@pytest.mark.skip(reason="There is a flaky segfault in the test setup") def test_nfs_backed_s3_storage(lib_name, nfs_backed_s3_storage): # Given lib = nfs_backed_s3_storage.create_version_store_factory(lib_name)() @@ -93,6 +94,7 @@ def s3_storage_dots_in_path(request): yield g +@pytest.mark.skip(reason="There is a flaky segfault in the test setup") def test_read_path_with_dot(lib_name, s3_storage_dots_in_path): # Given factory = s3_storage_dots_in_path.create_version_store_factory(lib_name)