Skip to content

Building 1495/merge on pull_request by alexowens90 #6197

Building 1495/merge on pull_request by alexowens90

Building 1495/merge on pull_request by alexowens90 #6197

Workflow file for this run

name: Build and Test
on:
push:
tags: ["v**"]
branches: ["master"]
pull_request:
branches: ["**"]
workflow_dispatch:
inputs:
persistent_storage:
description: Run the persistent storage tests
type: boolean
pypi_publish:
type: boolean
publish_env:
description: Environment to publish to
type: environment
cmake_preset_type:
description: Override CMAKE preset type
type: choice
options: ["-", debug, release]
run-name: Building ${{github.ref_name}} on ${{github.event_name}} by ${{github.actor}}
concurrency:
group: ${{github.ref}}
cancel-in-progress: true
jobs:
cibw_docker_image:
uses: ./.github/workflows/cibw_docker_image.yml
permissions: {packages: write}
with:
cibuildwheel_ver: "2.12.1"
force_update: false
common_config:
needs: [cibw_docker_image]
runs-on: ubuntu-latest
strategy:
matrix:
include:
# Use a matrix to build the common hierarchical structure used by multiple matrix jobs in build_steps.yaml
# Please declare any key you added below in build_steps.yaml's dummy matrix as well to aid the linting tools
- linux_matrix:
- os: linux
distro: ubuntu-latest
cmake_preset_prefix: linux
cibw_build_suffix: manylinux_x86_64
build_dir: /tmp/cpp_build
vcpkg_packages_dir: ""
symbols: "*.gz" # Please update publish.yml when changing this!!
do_not_archive: ["*.so", "*.[ao]", "vcpkg_installed"]
test_services:
mongodb:
image: "mongo:4.4"
container:
image: ${{needs.cibw_docker_image.outputs.tag}}
volumes:
- /:/mnt
windows_matrix:
- os: windows
distro: windows-latest
cmake_preset_prefix: windows-cl
cibw_build_suffix: win_amd64
build_dir: C:/cpp_build
vcpkg_packages_dir: C:/vcpkg_packages
symbols: "*.pdb" # Please update publish.yml when changing this!!
do_not_archive: ["*.lib", "*.ilk", "*.pyd", "*.dll", "CMakeFiles/*.dir", "vcpkg_installed"]
test_services: {}
container: null
steps:
- run: |
if ${{startsWith(github.ref, 'refs/tags/v')}}; then
# Can only upload to Pypi once per version, so only auto upload on tag builds:
echo -e "PUBLISH_ENV=ProdPypi\nCMAKE_PRESET_TYPE=release\nPYPI_PUBLISH=1" | tee -a $GITHUB_ENV
elif $GITHUB_REF_PROTECTED || ${{github.ref == 'refs/heads/master'}} ; then
echo -e "PUBLISH_ENV=TestPypi\nCMAKE_PRESET_TYPE=release" | tee -a $GITHUB_ENV
else
echo -e "PUBLISH_ENV=${{vars.DEFAULT_PUBLISH_ENV}}\nCMAKE_PRESET_TYPE=$DEFAULT_PRESET" | tee -a $GITHUB_ENV
fi
env:
DEFAULT_PRESET: ${{startsWith(github.repository, 'man-group/ArcticDB') && 'release' || vars.DEFAULT_CMAKE_PRESET_TYPE || 'debug'}}
outputs:
cibuildwheel_ver: ${{needs.cibw_docker_image.outputs.cibuildwheel_ver}}
publish_env: ${{inputs.publish_env || env.PUBLISH_ENV || ''}}
pypi_publish: ${{inputs.pypi_publish || env.PYPI_PUBLISH}}
cmake_preset_type_resolved: ${{inputs.cmake_preset_type != '-' && inputs.cmake_preset_type || env.CMAKE_PRESET_TYPE}}
linux_matrix: ${{toJson(matrix.linux_matrix)}}
windows_matrix: ${{toJson(matrix.windows_matrix)}}
pre_seed_cleanup:
if: inputs.persistent_storage == true
name: Cleanup persistent storages
uses: ./.github/workflows/persistent_storage.yml
secrets: inherit
with:
job_type: cleanup
persistent_storage_seed_linux:
needs: [common_config, pre_seed_cleanup]
if: inputs.persistent_storage == true
strategy:
fail-fast: false
matrix:
python3: ${{fromJson(vars.LINUX_PYTHON_VERSIONS || '[6, 7, 8, 9, 10, 11]')}}
arcticdb_version: ["oldest", "latest"]
include:
- python_deps_ids: [""]
matrix_override: ${{fromJson(needs.common_config.outputs.linux_matrix)}}
name: Seed Persistent Storage for 3.${{matrix.python3}} Linux with ${{ matrix.arcticdb_version }} ArcticDB package version
uses: ./.github/workflows/persistent_storage.yml
secrets: inherit
permissions: {packages: write}
with:
job_type: seed
python3: ${{matrix.python3}}
arcticdb_version: ${{matrix.arcticdb_version}}
matrix: ${{toJson(matrix.matrix_override)}}
python_deps_ids: ${{toJson(matrix.python_deps_ids)}}
persistent_storage_seed_windows:
needs: [common_config, pre_seed_cleanup]
if: inputs.persistent_storage == true
strategy:
fail-fast: false
matrix:
python3: ${{fromJson(vars.WINDOWS_PYTHON_VERSIONS || '[7, 8, 9, 10, 11]')}}
arcticdb_version: ["oldest", "latest"]
include:
- python_deps_ids: [""]
matrix_override: ${{fromJson(needs.common_config.outputs.windows_matrix)}}
name: Seed Persistent Storage for 3.${{matrix.python3}} Windows with ${{ matrix.arcticdb_version }} ArcticDB package version
uses: ./.github/workflows/persistent_storage.yml
secrets: inherit
permissions: {packages: write}
with:
job_type: seed
python3: ${{matrix.python3}}
arcticdb_version: ${{matrix.arcticdb_version}}
matrix: ${{toJson(matrix.matrix_override)}}
python_deps_ids: ${{toJson(matrix.python_deps_ids)}}
cpp-test-linux:
needs: [cibw_docker_image, common_config]
name: Linux C++ Tests
uses: ./.github/workflows/build_steps.yml
secrets: inherit
with:
job_type: cpp-tests
cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}}
matrix: ${{needs.common_config.outputs.linux_matrix}}
build-python-wheels-linux:
# Then use the cached compilation artifacts to build other python versions concurrently in cibuildwheels
needs: [cibw_docker_image, common_config]
strategy:
fail-fast: false
matrix:
python3: ${{fromJson(vars.LINUX_PYTHON_VERSIONS || '[6, 7, 8, 9, 10, 11]')}}
include:
- python_deps_ids: [""]
matrix_override: ${{fromJson(needs.common_config.outputs.linux_matrix)}}
pytest_xdist_mode: "--dist worksteal"
- python3: 6
python_deps_ids: ["", -compat36]
matrix_override:
- ${{fromJson(needs.common_config.outputs.linux_matrix)[0]}}
- python_deps_id: -compat36
python_deps: requirements-compatibility-py36.txt
pytest_xdist_mode: "" # worksteal Not supported on Python 3.6
- python3: 8
python_deps_ids: ["", -compat38]
matrix_override:
- ${{fromJson(needs.common_config.outputs.linux_matrix)[0]}}
- python_deps_id: -compat38
python_deps: requirements-compatibility-py38.txt
name: 3.${{matrix.python3}} Linux
uses: ./.github/workflows/build_steps.yml
secrets: inherit
permissions: {packages: write}
with:
job_type: build-python-wheels
python3: ${{matrix.python3}}
cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}}
cibw_version: ${{needs.common_config.outputs.cibuildwheel_ver}}
cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
matrix: ${{toJson(matrix.matrix_override)}}
python_deps_ids: ${{toJson(matrix.python_deps_ids)}}
persistent_storage: ${{inputs.persistent_storage}}
pytest_xdist_mode: ${{matrix.pytest_xdist_mode}}
cpp-test-windows:
needs: [common_config]
name: Windows C++ Tests
uses: ./.github/workflows/build_steps.yml
secrets: inherit
with:
job_type: cpp-tests
cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
matrix: ${{needs.common_config.outputs.windows_matrix}}
build-python-wheels-windows:
needs: [common_config]
strategy:
fail-fast: false
matrix:
python3: ${{fromJson(vars.WINDOWS_PYTHON_VERSIONS || '[7, 8, 9, 10, 11]')}}
name: 3.${{matrix.python3}} Windows
uses: ./.github/workflows/build_steps.yml
secrets: inherit
permissions: {packages: write}
with:
job_type: build-python-wheels
python3: ${{matrix.python3}}
cibw_version: ${{needs.common_config.outputs.cibuildwheel_ver}}
cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
matrix: ${{needs.common_config.outputs.windows_matrix}}
persistent_storage: ${{ inputs.persistent_storage }}
persistent_storage_verify_linux:
needs: [common_config, build-python-wheels-linux, build-python-wheels-windows]
if: inputs.persistent_storage == true
strategy:
fail-fast: false
matrix:
python3: ${{fromJson(vars.LINUX_PYTHON_VERSIONS || '[6, 7, 8, 9, 10, 11]')}}
arcticdb_version: ["oldest", "latest"]
include:
- python_deps_ids: [""]
matrix_override: ${{fromJson(needs.common_config.outputs.linux_matrix)}}
name: Verify Persistent Storage for 3.${{matrix.python3}} Linux with ${{ matrix.arcticdb_version }} ArcticDB package version
uses: ./.github/workflows/persistent_storage.yml
secrets: inherit
permissions: {packages: write}
with:
job_type: verify
python3: ${{matrix.python3}}
arcticdb_version: ${{matrix.arcticdb_version}}
matrix: ${{toJson(matrix.matrix_override)}}
python_deps_ids: ${{toJson(matrix.python_deps_ids)}}
persistent_storage_verify_windows:
needs: [common_config, build-python-wheels-windows, build-python-wheels-linux]
if: inputs.persistent_storage == true
strategy:
fail-fast: false
matrix:
python3: ${{fromJson(vars.WINDOWS_PYTHON_VERSIONS || '[7, 8, 9, 10, 11]')}}
arcticdb_version: ["oldest", "latest"]
include:
- python_deps_ids: [""]
matrix_override: ${{fromJson(needs.common_config.outputs.windows_matrix)}}
name: Verify Persistent Storage for 3.${{matrix.python3}} Windows with ${{ matrix.arcticdb_version }} ArcticDB package version
uses: ./.github/workflows/persistent_storage.yml
secrets: inherit
permissions: {packages: write}
with:
job_type: verify
python3: ${{matrix.python3}}
arcticdb_version: ${{matrix.arcticdb_version}}
matrix: ${{toJson(matrix.matrix_override)}}
python_deps_ids: ${{toJson(matrix.python_deps_ids)}}
post_verify_cleanup:
needs: [persistent_storage_verify_windows, persistent_storage_verify_linux]
if: inputs.persistent_storage == true
name: Cleanup persistent storages
uses: ./.github/workflows/persistent_storage.yml
secrets: inherit
with:
job_type: cleanup
can_merge:
needs: [cpp-test-linux, cpp-test-windows, build-python-wheels-linux, build-python-wheels-windows, persistent_storage_verify_linux, persistent_storage_verify_windows]
if: |
always() &&
!failure() &&
!cancelled()
runs-on: ubuntu-latest
steps:
- run: echo Dummy job to simplify PR merge checks configuration
# FUTURE: add some test stats/reporting
publish:
needs: [common_config, can_merge]
if: |
always() &&
!failure() &&
!cancelled() &&
(needs.common_config.outputs.pypi_publish && needs.common_config.outputs.publish_env)
uses: ./.github/workflows/publish.yml
secrets: inherit
permissions: {contents: write}
with:
environment: ${{needs.common_config.outputs.publish_env}}