Building 4.0.4 on push by phoebusm #5774

Workflow file for this run

	name: Build and Test
	on:
	push:
	branches: ["**"]
	tags: [v**]
	paths:
	- .github/workflows/*.yml
	- build_tooling/**
	- cpp/**
	- python/**
	- setup.*
	- pyproject.toml
	workflow_dispatch:
	inputs:
	persistent_storage:
	description: Run the persistent storage tests
	type: boolean
	pypi_publish:
	type: boolean
	publish_env:
	description: Environment to publish to
	type: environment
	cmake_preset_type:
	description: Override CMAKE preset type
	type: choice
	options: ["-", debug, release]
	run-name: Building ${{github.ref_name}} on ${{github.event_name}} by ${{github.actor}}
	concurrency:
	group: ${{github.ref}}
	cancel-in-progress: true
	jobs:
	common_config:
	runs-on: ubuntu-latest
	strategy:
	matrix:
	include:
	# Use a matrix to build the common hierarchical structure used by multiple matrix jobs in build_steps.yaml
	# Please declare any key you added below in build_steps.yaml's dummy matrix as well to aid the linting tools
	- linux_matrix:
	- os: linux
	distro: ubuntu-latest
	cmake_preset_prefix: linux
	cibw_build_suffix: manylinux_x86_64
	build_dir: /tmp/cpp_build
	symbols: "*.gz" # Please update publish.yml when changing this!!
	do_not_archive: [".so", ".[ao]", "vcpkg_installed"]
	test_services:
	mongodb:
	image: "mongo:4.4"
	mongodb_2:
	image: "mongo:4.4"
	windows_matrix:
	- os: windows
	distro: windows-latest
	cmake_preset_prefix: windows-cl
	cibw_build_suffix: win_amd64
	build_dir: C:\cpp_build
	symbols: "*.pdb" # Please update publish.yml when changing this!!
	do_not_archive: [".lib", ".ilk", ".pyd", ".dll", "CMakeFiles/*.dir", "vcpkg_installed"]
	test_services: {}
	steps:
	- run: \|
	if ${{startsWith(github.ref, 'refs/tags/v')}}; then
	# Can only upload to Pypi once per version, so only auto upload on tag builds:
	echo -e "PUBLISH_ENV=ProdPypi\nCMAKE_PRESET_TYPE=release\nPYPI_PUBLISH=1" \| tee -a $GITHUB_ENV
	elif $GITHUB_REF_PROTECTED \|\| ${{github.ref == 'refs/heads/master'}} ; then
	echo -e "PUBLISH_ENV=TestPypi\nCMAKE_PRESET_TYPE=release" \| tee -a $GITHUB_ENV
	else
	echo -e "PUBLISH_ENV=${{vars.DEFAULT_PUBLISH_ENV}}\nCMAKE_PRESET_TYPE=$DEFAULT_PRESET" \| tee -a $GITHUB_ENV
	fi
	env:
	DEFAULT_PRESET: ${{startsWith(github.repository, 'man-group/ArcticDB') && 'release' \|\| vars.DEFAULT_CMAKE_PRESET_TYPE \|\| 'debug'}}
	outputs:
	cibuildwheel_ver: "2.12.1"
	publish_env: ${{inputs.publish_env \|\| env.PUBLISH_ENV \|\| ''}}
	pypi_publish: ${{inputs.pypi_publish \|\| env.PYPI_PUBLISH}}
	cmake_preset_type_resolved: ${{inputs.cmake_preset_type != '-' && inputs.cmake_preset_type \|\| env.CMAKE_PRESET_TYPE}}
	linux_matrix: ${{toJson(matrix.linux_matrix)}}
	windows_matrix: ${{toJson(matrix.windows_matrix)}}

	pre_seed_cleanup:
	if: inputs.persistent_storage == true
	name: Cleanup persistent storages
	uses: ./.github/workflows/persistent_storage.yml
	secrets: inherit
	with:
	job_type: cleanup

	persistent_storage_seed_linux:
	needs: [common_config, pre_seed_cleanup]
	if: inputs.persistent_storage == true
	strategy:
	fail-fast: false
	matrix:
	python3: ${{fromJson(vars.LINUX_PYTHON_VERSIONS \|\| '[6, 7, 8, 9, 10, 11]')}}
	include:
	- python_deps_ids: [""]
	matrix_override: ${{fromJson(needs.common_config.outputs.linux_matrix)}}
	name: Seed Persistent Storage for 3.${{matrix.python3}} Linux
	uses: ./.github/workflows/persistent_storage.yml
	secrets: inherit
	permissions: {packages: write}
	with:
	job_type: seed
	python3: ${{matrix.python3}}
	matrix: ${{toJson(matrix.matrix_override)}}
	python_deps_ids: ${{toJson(matrix.python_deps_ids)}}

	persistent_storage_seed_windows:
	needs: [common_config, pre_seed_cleanup]
	if: inputs.persistent_storage == true
	strategy:
	fail-fast: false
	matrix:
	python3: ${{fromJson(vars.WINDOWS_PYTHON_VERSIONS \|\| '[7, 8, 9, 10, 11]')}}
	include:
	- python_deps_ids: [""]
	matrix_override: ${{fromJson(needs.common_config.outputs.windows_matrix)}}
	name: Seed Persistent Storage for 3.${{matrix.python3}} Windows
	uses: ./.github/workflows/persistent_storage.yml
	secrets: inherit
	permissions: {packages: write}
	with:
	job_type: seed
	python3: ${{matrix.python3}}
	matrix: ${{toJson(matrix.matrix_override)}}
	python_deps_ids: ${{toJson(matrix.python_deps_ids)}}


	cibw_docker_image:
	needs: [common_config]
	uses: ./.github/workflows/cibw_docker_image.yml
	permissions: {packages: write}
	with:
	cibuildwheel_ver: ${{needs.common_config.outputs.cibuildwheel_ver}}
	force_update: false

	leader-compile-linux:
	# First do the C++ core compilation using one Python version to seed the compilation caches (and fail quicker)
	needs: [cibw_docker_image, common_config]
	name: Linux Compile
	uses: ./.github/workflows/build_steps.yml
	secrets: inherit
	permissions: {packages: write}
	with:
	job_type: leader-compile
	cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
	cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}}
	matrix: ${{needs.common_config.outputs.linux_matrix}}

	leader-cpp-test-linux:
	# Compile and run the C++ tests separately concurrently with the following job
	needs: [leader-compile-linux, cibw_docker_image, common_config]
	name: Linux C++ Tests
	uses: ./.github/workflows/build_steps.yml
	secrets: inherit
	with:
	job_type: cpp-tests
	cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
	cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}}
	matrix: ${{needs.common_config.outputs.linux_matrix}}

	follower-linux:
	# Then use the cached compilation artifacts to build other python versions concurrently in cibuildwheels
	needs: [leader-compile-linux, cibw_docker_image, common_config]
	strategy:
	fail-fast: false
	matrix:
	python3: ${{fromJson(vars.LINUX_PYTHON_VERSIONS \|\| '[6, 7, 8, 9, 10, 11]')}}
	include:
	- python_deps_ids: [""]
	matrix_override: ${{fromJson(needs.common_config.outputs.linux_matrix)}}
	- python3: 6
	python_deps_ids: ["", -compat36]
	matrix_override:
	- ${{fromJson(needs.common_config.outputs.linux_matrix)[0]}}
	- python_deps_id: -compat36
	python_deps: requirements-compatibility-py36.txt
	- python3: 8
	python_deps_ids: ["", -compat38]
	matrix_override:
	- ${{fromJson(needs.common_config.outputs.linux_matrix)[0]}}
	- python_deps_id: -compat38
	python_deps: requirements-compatibility-py38.txt
	name: 3.${{matrix.python3}} Linux
	uses: ./.github/workflows/build_steps.yml
	secrets: inherit
	permissions: {packages: write}
	with:
	job_type: follower
	python3: ${{matrix.python3}}
	cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}}
	cibw_version: ${{needs.common_config.outputs.cibuildwheel_ver}}
	cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
	matrix: ${{toJson(matrix.matrix_override)}}
	python_deps_ids: ${{toJson(matrix.python_deps_ids)}}
	persistent_storage: ${{inputs.persistent_storage}}

	leader-compile-windows:
	needs: [common_config]
	name: Windows Compile
	uses: ./.github/workflows/build_steps.yml
	secrets: inherit
	permissions: {packages: write}
	with:
	job_type: leader-compile
	cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
	matrix: ${{needs.common_config.outputs.windows_matrix}}

	leader-cpp-test-windows:
	needs: [leader-compile-windows, common_config]
	name: Windows C++ Tests
	uses: ./.github/workflows/build_steps.yml
	secrets: inherit
	with:
	job_type: cpp-tests
	cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
	matrix: ${{needs.common_config.outputs.windows_matrix}}

	follower-windows:
	needs: [leader-compile-windows, common_config]
	strategy:
	fail-fast: false
	matrix:
	python3: ${{fromJson(vars.WINDOWS_PYTHON_VERSIONS \|\| '[7, 8, 9, 10, 11]')}}
	name: 3.${{matrix.python3}} Windows
	uses: ./.github/workflows/build_steps.yml
	secrets: inherit
	permissions: {packages: write}
	with:
	job_type: follower
	python3: ${{matrix.python3}}
	cibw_version: ${{needs.common_config.outputs.cibuildwheel_ver}}
	cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
	matrix: ${{needs.common_config.outputs.windows_matrix}}
	persistent_storage: ${{ inputs.persistent_storage }}

	persistent_storage_verify_linux:
	needs: [common_config, follower-linux, follower-windows]
	if: inputs.persistent_storage == true
	strategy:
	fail-fast: false
	matrix:
	python3: ${{fromJson(vars.LINUX_PYTHON_VERSIONS \|\| '[6, 7, 8, 9, 10, 11]')}}
	include:
	- python_deps_ids: [""]
	matrix_override: ${{fromJson(needs.common_config.outputs.linux_matrix)}}
	name: Verify Persistent Storage for 3.${{matrix.python3}} Linux
	uses: ./.github/workflows/persistent_storage.yml
	secrets: inherit
	permissions: {packages: write}
	with:
	job_type: verify
	python3: ${{matrix.python3}}
	matrix: ${{toJson(matrix.matrix_override)}}
	python_deps_ids: ${{toJson(matrix.python_deps_ids)}}

	persistent_storage_verify_windows:
	needs: [common_config, follower-windows, follower-linux]
	if: inputs.persistent_storage == true
	strategy:
	fail-fast: false
	matrix:
	python3: ${{fromJson(vars.WINDOWS_PYTHON_VERSIONS \|\| '[7, 8, 9, 10, 11]')}}
	include:
	- python_deps_ids: [""]
	matrix_override: ${{fromJson(needs.common_config.outputs.windows_matrix)}}
	name: Verify Persistent Storage for 3.${{matrix.python3}} Windows
	uses: ./.github/workflows/persistent_storage.yml
	secrets: inherit
	permissions: {packages: write}
	with:
	job_type: verify
	python3: ${{matrix.python3}}
	matrix: ${{toJson(matrix.matrix_override)}}
	python_deps_ids: ${{toJson(matrix.python_deps_ids)}}

	post_verify_cleanup:
	needs: [persistent_storage_verify_windows, persistent_storage_verify_linux]
	if: inputs.persistent_storage == true
	name: Cleanup persistent storages
	uses: ./.github/workflows/persistent_storage.yml
	secrets: inherit
	with:
	job_type: cleanup

	can_merge:
	needs: [leader-cpp-test-linux, leader-cpp-test-windows, follower-linux, follower-windows, persistent_storage_verify_linux, persistent_storage_verify_windows]
	if: \|
	always() &&
	!failure() &&
	!cancelled()
	runs-on: ubuntu-latest
	steps:
	- run: echo Dummy job to simplify PR merge checks configuration
	# FUTURE: add some test stats/reporting

	publish:
	needs: [common_config, can_merge]
	if: \|
	always() &&
	!failure() &&
	!cancelled()
	uses: ./.github/workflows/publish.yml
	secrets: inherit
	# permissions: {contents: write}
	with:
	environment: ${{needs.common_config.outputs.publish_env}}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Building 4.0.4 on push by phoebusm #5774

Workflow file

Building 4.0.4 on push by phoebusm #5774

Jobs

Run details

Workflow file for this run