Refactor DataFrameNormalizer to improve performance #5848
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build with conda | |
| on: | |
| push: | |
| branches: | |
| - master | |
| # For Pull-Requests, this runs the CI on merge commit | |
| # of HEAD with the target branch instead on HEAD, allowing | |
| # testing against potential new states which might have | |
| # been introduced in the target branch last commits. | |
| # See: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request | |
| pull_request: | |
| jobs: | |
| linux: | |
| if: | | |
| always() && | |
| !cancelled() | |
| runs-on: ubuntu-22.04 | |
| env: | |
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | |
| services: | |
| mongodb: | |
| image: mongo:4.4 | |
| steps: | |
| - uses: actions/[email protected] | |
| # DONT use recursive submodules checkout to simulate conda feedstock build | |
| # with: | |
| # submodules: recursive | |
| - name: Get number of CPU cores | |
| uses: SimenB/[email protected] | |
| id: cpu-cores | |
| - name: Install Conda environment from environment_unix.yml | |
| uses: mamba-org/[email protected] | |
| with: | |
| environment-file: environment_unix.yml | |
| init-shell: >- | |
| bash | |
| cache-environment: true | |
| post-cleanup: 'all' | |
| - name: Build ArcticDB with conda (ARCTICDB_USING_CONDA=1) | |
| shell: bash -l {0} | |
| run: | | |
| # Protocol buffers compilation require not using build isolation. | |
| python -m pip install --no-build-isolation --no-deps -v -e . | |
| env: | |
| ARCTICDB_USING_CONDA: 1 | |
| ARCTICDB_BUILD_CPP_TESTS: 1 | |
| ARCTICDB_PROTOC_VERS: "4" | |
| - name: Build C++ Tests | |
| shell: bash -l {0} | |
| run: | | |
| cd cpp/out/linux-conda-release-build/ | |
| make -j ${{ steps.cpu-cores.outputs.count }} arcticdb_rapidcheck_tests | |
| make -j ${{ steps.cpu-cores.outputs.count }} test_unit_arcticdb | |
| - name: Run C++ Tests | |
| shell: bash -l {0} | |
| run: | | |
| cd cpp/out/linux-conda-release-build/ | |
| CTEST_OUTPUT_ON_FAILURE=1 make test | |
| # Note: mongo tests are skipped in the macos workflow | |
| # These steps are official: https://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-ubuntu/#install-mongodb-community-edition | |
| - name: Install mongo for mongod executable needed as a fallback server fixture | |
| shell: bash -l {0} | |
| run: | | |
| sudo apt-get install gnupg curl | |
| curl -fsSL https://www.mongodb.org/static/pgp/server-7.0.asc | sudo gpg -o /usr/share/keyrings/mongodb-server-7.0.gpg --dearmor | |
| echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-7.0.list | |
| sudo apt-get update | |
| sudo apt-get install -y mongodb-org | |
| - name: Install npm # Linux github runner image does not come with npm | |
| uses: actions/[email protected] | |
| with: | |
| node-version: '16' | |
| - name: Install azurite | |
| shell: bash -l {0} | |
| run: | | |
| npm install -g azurite | |
| - name: Test with pytest | |
| shell: bash -l {0} | |
| run: | | |
| cd python | |
| export ARCTICDB_RAND_SEED=$RANDOM | |
| python -m pytest --timeout=3600 -n ${{ steps.cpu-cores.outputs.count }} -v tests | |
| env: | |
| ARCTICDB_USING_CONDA: 1 | |
| # Use the Mongo created in the service container above to test against | |
| CI_MONGO_HOST: mongodb | |
| macos: | |
| strategy: | |
| matrix: | |
| include: | |
| - os: macos-12 | |
| fail-fast: false | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | |
| steps: | |
| - uses: actions/[email protected] | |
| # DONT use recursive submodules checkout to simulate conda feedstock build | |
| # with: | |
| # submodules: recursive | |
| - name: Get number of CPU cores | |
| uses: SimenB/[email protected] | |
| id: cpu-cores | |
| - name: Install Conda environment from environment_unix.yml | |
| uses: mamba-org/[email protected] | |
| with: | |
| environment-file: environment_unix.yml | |
| environment-name: arcticdb | |
| init-shell: >- | |
| bash | |
| cache-environment: true | |
| post-cleanup: 'all' | |
| - name: Build ArcticDB with conda (ARCTICDB_USING_CONDA=1) | |
| shell: bash -l {0} | |
| run: | | |
| # Protocol buffers compilation require not using build isolation. | |
| python -m pip install --no-build-isolation --no-deps -v -e . | |
| env: | |
| ARCTICDB_USING_CONDA: 1 | |
| ARCTICDB_BUILD_CPP_TESTS: 1 | |
| ARCTICDB_PROTOC_VERS: "4" | |
| - name: Build C++ Tests | |
| shell: bash -l {0} | |
| run: | | |
| cd cpp/out/darwin-conda-release-build/ | |
| make -j ${{ steps.cpu-cores.outputs.count }} arcticdb_rapidcheck_tests | |
| make -j ${{ steps.cpu-cores.outputs.count }} test_unit_arcticdb | |
| - name: Run C++ Tests | |
| shell: bash -l {0} | |
| run: | | |
| cd cpp/out/darwin-conda-release-build/ | |
| CTEST_OUTPUT_ON_FAILURE=1 make test | |
| - name: Install npm | |
| uses: actions/[email protected] | |
| with: | |
| node-version: '16' | |
| - name: Install azurite | |
| shell: bash -l {0} | |
| run: | | |
| npm install -g azurite | |
| - name: Test with pytest | |
| shell: bash -l {0} | |
| run: | | |
| cd python | |
| export ARCTICDB_RAND_SEED=$RANDOM | |
| python -m pytest --timeout=3600 -n ${{ steps.cpu-cores.outputs.count }} tests | |
| env: | |
| ARCTICDB_USING_CONDA: 1 | |