Stackable Build Pipeline #2331

Workflow file for this run

	# =============
	# This file is automatically generated from the templates in stackabletech/operator-templating
	# DON'T MANUALLY EDIT THIS FILE
	# =============
	---
	name: Stackable Build Pipeline

	on:
	push:
	branches:
	- main
	- staging
	- trying
	- "renovate/**"
	tags:
	- '[0-9][0-9].[0-9]+.[0-9]+'
	- '[0-9][0-9].[0-9]+.[0-9]+-rc[0-9]+'
	pull_request:
	merge_group:
	schedule:
	# Run every Saturday morning: https://crontab.guru/#15_3___6
	- cron: '15 3 * * 6'
	workflow_dispatch:

	env:
	CARGO_TERM_COLOR: always
	CARGO_INCREMENTAL: '0'
	CARGO_PROFILE_DEV_DEBUG: '0'
	RUST_TOOLCHAIN_VERSION: "1.82.0"
	RUSTFLAGS: "-D warnings"
	RUSTDOCFLAGS: "-D warnings"
	RUST_LOG: "info"
	DEV_REPO_HELM_URL: https://repo.stackable.tech/repository/helm-dev
	TEST_REPO_HELM_URL: https://repo.stackable.tech/repository/helm-test
	STABLE_REPO_HELM_URL: https://repo.stackable.tech/repository/helm-stable

	jobs:
	# Identify unused dependencies
	run_udeps:
	name: Run Cargo Udeps
	runs-on: ubuntu-latest
	env:
	RUSTC_BOOTSTRAP: 1
	steps:
	- name: Install host dependencies
	uses: awalsh128/cache-apt-pkgs-action@a6c3917cc929dd0345bfb2d3feaf9101823370ad # v1.4.2
	with:
	packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https
	version: ubuntu-latest
	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	with:
	submodules: recursive
	- uses: dtolnay/rust-toolchain@7b1c307e0dcbda6122208f10795a713336a9b35a
	with:
	toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
	- uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3
	with:
	key: udeps
	cache-all-crates: "true"
	- uses: stackabletech/cargo-install-action@cargo-udeps
	- run: cargo udeps --workspace --all-targets

	# This job evaluates the github environment to determine why this action is running and selects the appropriate
	# target repository for published Helm charts based on this.
	#
	# The following scenarios are identified:
	# - all pull requests land in the test repository:
	# condition: github.event_name == "pull_request"
	# repository: test
	#
	# - all tagged releases land in stable:
	# condition: github.event_name == 'push' & github.ref.startswith('refs/tags/')
	# repository: stable
	#
	# - all pushes to main (i.e. PR-merges) and all scheduled/manual workflow runs on main land in dev:
	# condition: ( github.event_name == 'push' \| github.event_name == 'schedule' \| github.event_name == 'workflow_dispatch' ) & github.ref == 'refs/heads/main'
	# repository: dev
	#
	# Any other scenarios (e.g. when a branch is created/pushed) will cause the publish step to be skipped, most commonly this is expected to happen for the
	# branches that the GitHub merge queue feature uses internally for which the checks need to run, but we do not want artifacts to be published.
	select_helm_repo:
	name: Select target helm repository based on action trigger
	runs-on: ubuntu-latest
	outputs:
	helm_repository: ${{ steps.selecthelmrepo.outputs.helm_repo }}
	steps:
	- id: selecthelmrepo
	env:
	TRIGGER: ${{ github.event_name }}
	GITHUB_REF: ${{ github.ref }}
	run: \|
	if [[ "$TRIGGER" == "pull_request" ]]; then
	echo "exporting test as target helm repo: ${{ env.TEST_REPO_HELM_URL }}"
	echo "helm_repo=${{ env.TEST_REPO_HELM_URL }}" >> "$GITHUB_OUTPUT"
	elif [[ ( "$TRIGGER" == "push" \|\| "$TRIGGER" == "schedule" \|\| "$TRIGGER" == "workflow_dispatch" ) && "$GITHUB_REF" == "refs/heads/main" ]]; then
	echo "exporting dev as target helm repo: ${{ env.DEV_REPO_HELM_URL }}"
	echo "helm_repo=${{ env.DEV_REPO_HELM_URL }}" >> "$GITHUB_OUTPUT"
	elif [[ "$TRIGGER" == "push" && $GITHUB_REF == refs/tags/* ]]; then
	echo "exporting stable as target helm repo: ${{ env.STABLE_REPO_HELM_URL }}"
	echo "helm_repo=${{ env.STABLE_REPO_HELM_URL }}" >> "$GITHUB_OUTPUT"
	else
	echo "Unknown trigger and ref combination encountered, skipping publish step: $TRIGGER $GITHUB_REF"
	echo "helm_repo=skip" >> "$GITHUB_OUTPUT"
	fi

	run_cargodeny:
	name: Run Cargo Deny
	runs-on: ubuntu-latest
	strategy:
	matrix:
	checks:
	- advisories
	- bans licenses sources

	# Prevent sudden announcement of a new advisory from failing ci:
	continue-on-error: ${{ matrix.checks == 'advisories' }}

	steps:
	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	with:
	submodules: recursive
	- uses: EmbarkStudios/cargo-deny-action@8371184bd11e21dcf8ac82ebf8c9c9f74ebf7268 # v2.0.1
	with:
	command: check ${{ matrix.checks }}

	run_rustfmt:
	name: Run Rustfmt
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	with:
	submodules: recursive
	- uses: dtolnay/rust-toolchain@7b1c307e0dcbda6122208f10795a713336a9b35a
	with:
	toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
	components: rustfmt
	- run: cargo fmt --all -- --check

	run_clippy:
	name: Run Clippy
	runs-on: ubuntu-latest
	steps:
	- name: Install host dependencies
	uses: awalsh128/cache-apt-pkgs-action@a6c3917cc929dd0345bfb2d3feaf9101823370ad # v1.4.2
	with:
	packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https
	version: ubuntu-latest
	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	with:
	submodules: recursive
	- uses: dtolnay/rust-toolchain@7b1c307e0dcbda6122208f10795a713336a9b35a
	with:
	toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
	components: clippy
	- uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3
	with:
	key: clippy
	cache-all-crates: "true"
	- name: Run clippy action to produce annotations
	uses: giraffate/clippy-action@13b9d32482f25d29ead141b79e7e04e7900281e0 # v1.0.1
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	if: env.GITHUB_TOKEN != null
	with:
	clippy_flags: --all-targets -- -D warnings
	reporter: 'github-pr-review'
	github_token: ${{ secrets.GITHUB_TOKEN }}
	- name: Run clippy manually without annotations
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	if: env.GITHUB_TOKEN == null
	run: cargo clippy --color never -q --all-targets -- -D warnings

	run_rustdoc:
	name: Run RustDoc
	runs-on: ubuntu-latest
	steps:
	- name: Install host dependencies
	uses: awalsh128/cache-apt-pkgs-action@a6c3917cc929dd0345bfb2d3feaf9101823370ad # v1.4.2
	with:
	packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https
	version: ubuntu-latest
	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	with:
	submodules: recursive
	- uses: dtolnay/rust-toolchain@7b1c307e0dcbda6122208f10795a713336a9b35a
	with:
	toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
	components: rustfmt
	- uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3
	with:
	key: doc
	cache-all-crates: "true"
	- run: cargo doc --document-private-items

	run_tests:
	name: Run Cargo Tests
	runs-on: ubuntu-latest
	steps:
	- name: Install host dependencies
	uses: awalsh128/cache-apt-pkgs-action@a6c3917cc929dd0345bfb2d3feaf9101823370ad # v1.4.2
	with:
	packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https
	version: ubuntu-latest
	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	with:
	submodules: recursive
	- uses: dtolnay/rust-toolchain@7b1c307e0dcbda6122208f10795a713336a9b35a
	with:
	toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
	- uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3
	with:
	key: test
	cache-all-crates: "true"
	- run: cargo test


	# Similar to check_charts, this tries to render the README, and see if there are unintended changes.
	# This will save us from merging changes to the wrong file (instead of the templated source), and from
	# forgetting to render out modifications to the README.
	check_readme:
	name: Check if committed README is the one we would render from the available parts
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	with:
	submodules: recursive
	- uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
	with:
	python-version: '3.12'
	- name: Install jinja2-cli
	run: pip install jinja2-cli==0.8.2
	- name: Regenerate charts
	run: make render-readme
	- name: Check if committed README were up to date
	run: git diff --exit-code
	- name: Git Diff showed uncommitted changes
	if: ${{ failure() }}
	uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
	with:
	script: \|
	core.setFailed('Committed README are not up to date, please make sure to apply them to the templated partials, and re-commit!')

	# This job cleans up the CRDs and Helm charts, followed by rebuilding them
	# It then runs a `git diff` and fails the entire workflow, if any difference is encountered.
	#
	# Since CRD files are generated during the 'cargo build' process we need to run this once after
	# removing the CRD files to ensure that the checked in versions match what the code expects.
	#
	# The reason for this step is, that developers are expected to check in up-to-date versions of charts
	# as we'd otherwise have to build these in CI and commit them back to the PR, which
	# creates all kinds of problems.
	# This failsafe simply aborts anything that has not had charts rebuilt before pushing.
	check_charts:
	name: Check if committed Helm charts are up to date
	runs-on: ubuntu-latest
	steps:
	- name: Install host dependencies
	uses: awalsh128/cache-apt-pkgs-action@a6c3917cc929dd0345bfb2d3feaf9101823370ad # v1.4.2
	with:
	packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https
	version: ubuntu-latest
	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	with:
	submodules: recursive
	- name: Set up Helm
	uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
	with:
	version: v3.16.1
	- name: Set up cargo
	uses: dtolnay/rust-toolchain@7b1c307e0dcbda6122208f10795a713336a9b35a
	with:
	toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
	- uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3
	with:
	key: charts
	cache-all-crates: "true"
	- name: Regenerate charts
	run: make regenerate-charts
	- name: Check if committed charts were up to date
	run: git diff --exit-code
	- name: Git Diff showed uncommitted changes
	if: ${{ failure() }}
	uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
	with:
	script: \|
	core.setFailed('Committed charts were not up to date, please regenerate and re-commit!')

	tests_passed:
	name: All tests passed
	needs:
	- run_udeps
	- run_cargodeny
	- run_clippy
	- run_rustfmt
	- run_rustdoc
	- run_tests
	- check_charts
	- check_readme
	runs-on: ubuntu-latest
	steps:
	- name: log
	run: echo All tests have passed!

	package_and_publish:
	name: Package Charts, Build Docker Image and publish them - ${{ matrix.runner }}
	needs:
	- tests_passed
	- select_helm_repo
	strategy:
	matrix:
	runner: ["ubuntu-latest", "ubicloud-standard-8-arm"]
	runs-on: ${{ matrix.runner }}
	timeout-minutes: 120
	permissions:
	id-token: write
	env:
	NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }}
	HELM_REPO: ${{ needs.select_helm_repo.outputs.helm_repository }}
	OCI_REGISTRY_SDP_PASSWORD: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }}
	OCI_REGISTRY_SDP_USERNAME: "robot$sdp+github-action-build"
	OCI_REGISTRY_SDP_CHARTS_PASSWORD: ${{ secrets.HARBOR_ROBOT_SDP_CHARTS_GITHUB_ACTION_BUILD_SECRET }}
	OCI_REGISTRY_SDP_CHARTS_USERNAME: "robot$sdp-charts+github-action-build"
	if: needs.select_helm_repo.outputs.helm_repository != 'skip'
	outputs:
	IMAGE_TAG: ${{ steps.printtag.outputs.IMAGE_TAG }}
	steps:
	- name: Install host dependencies
	uses: awalsh128/cache-apt-pkgs-action@a6c3917cc929dd0345bfb2d3feaf9101823370ad # v1.4.2
	with:
	packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https
	version: ${{ matrix.runner }}
	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	with:
	submodules: recursive
	- uses: cachix/install-nix-action@ba0dd844c9180cbf77aa72a116d6fbc515d0e87b # v27
	- uses: dtolnay/rust-toolchain@7b1c307e0dcbda6122208f10795a713336a9b35a
	with:
	toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
	components: rustfmt
	# This step checks if the current run was triggered by a push to a pr (or a pr being created).
	# If this is the case it changes the version of this project in all Cargo.toml files to include the suffix
	# "-pr<prnumber>" so that the published artifacts can be linked to this PR.
	- uses: stackabletech/cargo-install-action@main
	with:
	crate: cargo-edit
	bin: cargo-set-version
	- name: Update version if PR against main branch
	if: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'main' }}
	env:
	PR_NUMBER: ${{ github.event.pull_request.number }}
	run: \|
	PR_VERSION="0.0.0-pr${PR_NUMBER}"
	cargo set-version --offline --workspace "$PR_VERSION"
	- name: Update version if PR against non-main branch
	# For PRs to be merged against a release branch, use the version that has already been set in the calling script.
	if: ${{ github.event_name == 'pull_request' && startsWith(github.event.pull_request.base.ref, 'release-') }}
	env:
	PR_NUMBER: ${{ github.event.pull_request.number }}
	run: \|
	MANIFEST_VERSION=$(cargo metadata --format-version 1 --no-deps \| jq -r '.packages[0].version')
	PR_VERSION="${MANIFEST_VERSION}-pr${PR_NUMBER}"
	cargo set-version --offline --workspace "$PR_VERSION"

	# Recreate charts and publish charts and docker image. The "-e" is needed as we want to override the
	# default value in the makefile if called from this action, but not otherwise (i.e. when called locally).
	# This is needed for the HELM_REPO variable.
	- name: Install cosign
	uses: sigstore/cosign-installer@4959ce089c160fddf62f7b42464195ba1a56d382 # v3.6.0
	- name: Install syft
	uses: anchore/sbom-action/download-syft@61119d458adab75f756bc0b9e4bde25725f86a7a # v0.17.2
	- name: Build Docker image and Helm chart
	run: \|
	# Installing helm and yq on ubicloud-standard-8-arm only
	if [ "$(arch)" = "aarch64" ]; then
	curl https://baltocdn.com/helm/signing.asc \| gpg --dearmor \| sudo tee /usr/share/keyrings/helm.gpg > /dev/null
	echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" \| sudo tee /etc/apt/sources.list.d/helm-stable-debian.list
	sudo apt-get -y update
	sudo apt-get -y install helm
	sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_arm64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq
	fi

	make -e build
	- name: Publish Docker image and Helm chart
	if: ${{ !github.event.pull_request.head.repo.fork }}
	run: \|
	# We want to publish helmcharts only once as they have a common name, while still publishing both images with architecture specific tags
	if [ "$(uname -m)" = "x86_64" ]; then
	make -e publish
	else
	make -e docker-publish
	fi
	# Output the name of the published image to the Job output for later use
	- id: printtag
	name: Output image name and tag
	if: ${{ !github.event.pull_request.head.repo.fork }}
	run: echo "IMAGE_TAG=$(make -e print-docker-tag)" >> "$GITHUB_OUTPUT"

	create_manifest_list:
	name: Build and publish manifest list
	if: ${{ !github.event.pull_request.head.repo.fork }}
	needs:
	- package_and_publish
	runs-on: ubuntu-latest
	permissions:
	id-token: write
	env:
	NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }}
	OCI_REGISTRY_SDP_PASSWORD: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }}
	OCI_REGISTRY_SDP_USERNAME: "robot$sdp+github-action-build"
	OCI_REGISTRY_SDP_CHARTS_PASSWORD: ${{ secrets.HARBOR_ROBOT_SDP_CHARTS_GITHUB_ACTION_BUILD_SECRET }}
	OCI_REGISTRY_SDP_CHARTS_USERNAME: "robot$sdp-charts+github-action-build"
	steps:
	- name: Install cosign
	uses: sigstore/cosign-installer@4959ce089c160fddf62f7b42464195ba1a56d382 # v3.6.0
	- name: Checkout
	uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	with:
	submodules: recursive
	# This step checks if the current run was triggered by a push to a pr (or a pr being created).
	# If this is the case it changes the version of this project in all Cargo.toml files to include the suffix
	# "-pr<prnumber>" so that the published artifacts can be linked to this PR.
	- uses: stackabletech/cargo-install-action@main
	with:
	crate: cargo-edit
	bin: cargo-set-version
	- name: Update version if PR against main branch
	if: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'main' }}
	env:
	PR_NUMBER: ${{ github.event.pull_request.number }}
	run: \|
	PR_VERSION="0.0.0-pr${PR_NUMBER}"
	cargo set-version --offline --workspace "$PR_VERSION"
	- name: Update version if PR against non-main branch
	# For PRs to be merged against a release branch, use the version that has already been set in the calling script.
	if: ${{ github.event_name == 'pull_request' && startsWith(github.event.pull_request.base.ref, 'release-') }}
	env:
	PR_NUMBER: ${{ github.event.pull_request.number }}
	run: \|
	MANIFEST_VERSION=$(cargo metadata --format-version 1 --no-deps \| jq -r '.packages[0].version')
	PR_VERSION="${MANIFEST_VERSION}-pr${PR_NUMBER}"
	cargo set-version --offline --workspace "$PR_VERSION"
	- name: Build manifest list
	run: \|
	# Creating manifest list
	make -e docker-manifest-list-build
	# Pushing and signing manifest list
	make -e docker-manifest-list-publish

	openshift_preflight:
	name: Run the OpenShift Preflight check on the published images
	if: ${{ !github.event.pull_request.head.repo.fork }}
	needs:
	- create_manifest_list
	- package_and_publish
	runs-on: ubuntu-latest
	env:
	IMAGE_TAG: ${{ needs.package_and_publish.outputs.IMAGE_TAG }}
	steps:
	- name: Install preflight
	run: \|
	wget https://github.com/redhat-openshift-ecosystem/openshift-preflight/releases/download/1.10.0/preflight-linux-amd64
	chmod +x preflight-linux-amd64
	- name: Check container
	run: \|
	ARCH_FOR_PREFLIGHT="$(arch \| sed -e 's#x86_64#amd64#' \| sed -e 's#aarch64#arm64#')"
	./preflight-linux-amd64 check container "$IMAGE_TAG" --platform "${ARCH_FOR_PREFLIGHT}" > preflight.out
	- name: "Passed?"
	run: '[ "$(jq -r .passed < preflight.out)" == true ]'

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Stackable Build Pipeline #2331

Workflow file

Stackable Build Pipeline #2331

Jobs

Run details

Workflow file for this run