diff --git a/Makefile b/Makefile index 69f76593..c6e24209 100644 --- a/Makefile +++ b/Makefile @@ -12,11 +12,16 @@ TAG := $(shell git rev-parse --short HEAD) OPERATOR_NAME := stackable-cockpit VERSION := $(shell cargo metadata --format-version 1 | jq -r '.packages[] | select(.name=="stackable-cockpitd") | .version') +ARCH := $(shell uname -m | sed -e 's#x86_64#amd64#' | sed -e 's#aarch64#arm64#') DOCKER_REPO := docker.stackable.tech ORGANIZATION := stackable -# this will be overwritten by an environmental variable if called from the github action +OCI_REGISTRY_HOSTNAME := oci.stackable.tech +OCI_REGISTRY_PROJECT_IMAGES := sdp +OCI_REGISTRY_PROJECT_CHARTS := sdp-charts +# This will be overwritten by an environmental variable if called from the github action HELM_REPO := https://repo.stackable.tech/repository/helm-dev +HELM_CHART_NAME := ${OPERATOR_NAME} HELM_CHART_ARTIFACT := target/helm/${OPERATOR_NAME}-${VERSION}.tgz SHELL=/usr/bin/env bash -euo pipefail @@ -27,11 +32,84 @@ SHELL=/usr/bin/env bash -euo pipefail ## Docker related targets docker-build: - docker build --force-rm --build-arg VERSION=${VERSION} -t "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}" -f docker/Dockerfile . + docker build --force-rm --build-arg VERSION=${VERSION} -t "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}-${ARCH}" -f docker/Dockerfile . + docker tag "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}-${ARCH}" "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-${ARCH}" docker-publish: + # Push to Nexus echo "${NEXUS_PASSWORD}" | docker login --username github --password-stdin "${DOCKER_REPO}" - docker push --all-tags "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}" + DOCKER_OUTPUT=$$(docker push --all-tags "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}");\ + # Obtain the digest of the pushed image from the output of `docker push`, because signing by tag is deprecated and will be removed from cosign in the future\ + REPO_DIGEST_OF_IMAGE=$$(echo "$$DOCKER_OUTPUT" | awk '/^${VERSION}-${ARCH}: digest: sha256:[0-9a-f]{64} size: [0-9]+$$/ { print $$3 }');\ + if [ -z "$$REPO_DIGEST_OF_IMAGE" ]; then\ + echo 'Could not find repo digest for container image: ${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}-${ARCH}';\ + exit 1;\ + fi;\ + # This generates a signature and publishes it to the registry, next to the image\ + # Uses the keyless signing flow with Github Actions as identity provider\ + cosign sign -y "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}@$$REPO_DIGEST_OF_IMAGE";\ + # Generate the SBOM for the operator image, this leverages the already generated SBOM for the operator binary by cargo-cyclonedx\ + syft scan --output cyclonedx-json=sbom.json --select-catalogers "-cargo-auditable-binary-cataloger" --scope all-layers --source-name "${OPERATOR_NAME}" --source-version "${VERSION}" "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}@$$REPO_DIGEST_OF_IMAGE";\ + # Determine the PURL for the container image\ + PURL="pkg:docker/${ORGANIZATION}/${OPERATOR_NAME}@$$REPO_DIGEST_OF_IMAGE?repository_url=${DOCKER_REPO}";\ + # Get metadata from the image\ + IMAGE_DESCRIPTION=$$(docker inspect --format='{{.Config.Labels.description}}' "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}-${ARCH}");\ + IMAGE_NAME=$$(docker inspect --format='{{.Config.Labels.name}}' "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}-${ARCH}");\ + # Merge the SBOM with the metadata for the operator\ + jq -s '{"metadata":{"component":{"description":"'"$$IMAGE_NAME. $$IMAGE_DESCRIPTION"'","supplier":{"name":"Stackable GmbH","url":["https://stackable.tech/"]},"author":"Stackable GmbH","purl":"'"$$PURL"'","publisher":"Stackable GmbH"}}} * .[0]' sbom.json > sbom.merged.json;\ + # Attest the SBOM to the image\ + cosign attest -y --predicate sbom.merged.json --type cyclonedx "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}@$$REPO_DIGEST_OF_IMAGE" + + # Push to Harbor + # We need to use "value" here to prevent the variable from being recursively expanded by make (username contains a dollar sign, since it's a Harbor bot) + docker login --username '${value OCI_REGISTRY_SDP_USERNAME}' --password '${OCI_REGISTRY_SDP_PASSWORD}' '${OCI_REGISTRY_HOSTNAME}' + DOCKER_OUTPUT=$$(docker push --all-tags '${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}');\ + # Obtain the digest of the pushed image from the output of `docker push`, because signing by tag is deprecated and will be removed from cosign in the future\ + REPO_DIGEST_OF_IMAGE=$$(echo "$$DOCKER_OUTPUT" | awk '/^${VERSION}-${ARCH}: digest: sha256:[0-9a-f]{64} size: [0-9]+$$/ { print $$3 }');\ + if [ -z "$$REPO_DIGEST_OF_IMAGE" ]; then\ + echo 'Could not find repo digest for container image: ${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-${ARCH}';\ + exit 1;\ + fi;\ + # This generates a signature and publishes it to the registry, next to the image\ + # Uses the keyless signing flow with Github Actions as identity provider\ + cosign sign -y "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}@$$REPO_DIGEST_OF_IMAGE";\ + # Generate the SBOM for the operator image, this leverages the already generated SBOM for the operator binary by cargo-cyclonedx\ + syft scan --output cyclonedx-json=sbom.json --select-catalogers "-cargo-auditable-binary-cataloger" --scope all-layers --source-name "${OPERATOR_NAME}" --source-version "${VERSION}" "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}@$$REPO_DIGEST_OF_IMAGE";\ + # Determine the PURL for the container image\ + PURL="pkg:docker/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}@$$REPO_DIGEST_OF_IMAGE?repository_url=${OCI_REGISTRY_HOSTNAME}";\ + # Get metadata from the image\ + IMAGE_DESCRIPTION=$$(docker inspect --format='{{.Config.Labels.description}}' "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-${ARCH}");\ + IMAGE_NAME=$$(docker inspect --format='{{.Config.Labels.name}}' "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-${ARCH}");\ + # Merge the SBOM with the metadata for the operator\ + jq -s '{"metadata":{"component":{"description":"'"$$IMAGE_NAME. $$IMAGE_DESCRIPTION"'","supplier":{"name":"Stackable GmbH","url":["https://stackable.tech/"]},"author":"Stackable GmbH","purl":"'"$$PURL"'","publisher":"Stackable GmbH"}}} * .[0]' sbom.json > sbom.merged.json;\ + # Attest the SBOM to the image\ + cosign attest -y --predicate sbom.merged.json --type cyclonedx "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}@$$REPO_DIGEST_OF_IMAGE" + +# This assumes "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}-amd64 and "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}-arm64 are build and pushed +docker-manifest-list-build: + docker manifest create "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}" --amend "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}-amd64" --amend "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}-arm64" + docker manifest create "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}" --amend "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-amd64" --amend "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-arm64" + +docker-manifest-list-publish: + # Push to Nexus + echo "${NEXUS_PASSWORD}" | docker login --username github --password-stdin "${DOCKER_REPO}" + # `docker manifest push` directly returns the digest of the manifest list + # As it is an experimental feature, this might change in the future + # Further reading: https://docs.docker.com/reference/cli/docker/manifest/push/ + DIGEST_NEXUS=$$(docker manifest push "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}");\ + # Refer to image via its digest (oci.stackable.tech/sdp/airflow@sha256:0a1b2c...)\ + # This generates a signature and publishes it to the registry, next to the image\ + # Uses the keyless signing flow with Github Actions as identity provider\ + cosign sign -y "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}@$$DIGEST_NEXUS" + + # Push to Harbor + # We need to use "value" here to prevent the variable from being recursively expanded by make (username contains a dollar sign, since it's a Harbor bot) + docker login --username '${value OCI_REGISTRY_SDP_USERNAME}' --password '${OCI_REGISTRY_SDP_PASSWORD}' '${OCI_REGISTRY_HOSTNAME}' + DIGEST_HARBOR=$$(docker manifest push "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}");\ + # Refer to image via its digest (oci.stackable.tech/sdp/airflow@sha256:0a1b2c...);\ + # This generates a signature and publishes it to the registry, next to the image\ + # Uses the keyless signing flow with Github Actions as identity provider\ + cosign sign -y "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}@$$DIGEST_HARBOR" # TODO remove if not used/needed docker: docker-build docker-publish @@ -40,8 +118,25 @@ print-docker-tag: @echo "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}" helm-publish: + # Push to Nexus curl --fail -u "github:${NEXUS_PASSWORD}" --upload-file "${HELM_CHART_ARTIFACT}" "${HELM_REPO}/" + # Push to Harbor + # We need to use "value" here to prevent the variable from being recursively expanded by make (username contains a dollar sign, since it's a Harbor bot) + helm registry login --username '${value OCI_REGISTRY_SDP_CHARTS_USERNAME}' --password '${OCI_REGISTRY_SDP_CHARTS_PASSWORD}' '${OCI_REGISTRY_HOSTNAME}' + # Obtain the digest of the pushed artifact from the output of `helm push`, because signing by tag is deprecated and will be removed from cosign in the future\ + HELM_OUTPUT=$$(helm push '${HELM_CHART_ARTIFACT}' 'oci://${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_CHARTS}' 2>&1);\ + REPO_DIGEST_OF_ARTIFACT=$$(echo "$$HELM_OUTPUT" | awk '/^Digest: sha256:[0-9a-f]{64}$$/ { print $$2 }');\ + if [ -z "$$REPO_DIGEST_OF_ARTIFACT" ]; then\ + echo 'Could not find repo digest for helm chart: ${HELM_CHART_NAME}';\ + exit 1;\ + fi;\ + # Login to Harbor, needed for cosign to be able to push the signature for the Helm chart\ + docker login --username '${value OCI_REGISTRY_SDP_CHARTS_USERNAME}' --password '${OCI_REGISTRY_SDP_CHARTS_PASSWORD}' '${OCI_REGISTRY_HOSTNAME}';\ + # This generates a signature and publishes it to the registry, next to the chart artifact\ + # Uses the keyless signing flow with Github Actions as identity provider\ + cosign sign -y "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_CHARTS}/${HELM_CHART_NAME}@$$REPO_DIGEST_OF_ARTIFACT" + helm-package: mkdir -p target/helm && helm package --destination target/helm deploy/helm/${OPERATOR_NAME} @@ -62,9 +157,9 @@ config: cp -r deploy/config-spec/* "deploy/helm/${OPERATOR_NAME}/configs";\ fi -# crds: -# mkdir -p deploy/helm/"${OPERATOR_NAME}"/crds -# cargo run --bin "${OPERATOR_NAME}" -- crd | yq eval '.metadata.annotations["helm.sh/resource-policy"]="keep"' - > "deploy/helm/${OPERATOR_NAME}/crds/crds.yaml" +#crds: +# mkdir -p deploy/helm/"${OPERATOR_NAME}"/crds +# cargo run --bin stackable-"${OPERATOR_NAME}" -- crd | yq eval '.metadata.annotations["helm.sh/resource-policy"]="keep"' - > "deploy/helm/${OPERATOR_NAME}/crds/crds.yaml" chart-lint: compile-chart docker run -it -v $(shell pwd):/build/helm-charts -w /build/helm-charts quay.io/helmpack/chart-testing:v3.5.0 ct lint --config deploy/helm/ct.yaml @@ -72,19 +167,31 @@ chart-lint: compile-chart clean: chart-clean cargo clean docker rmi --force "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}" + docker rmi --force '${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}' regenerate-charts: chart-clean compile-chart -# The explicit '--extra-experimental-features flakes' parameter was added, -# because the command requires Flakes to be enabled, but most non-Nix users -# don't enable them via the global nix rc file. regenerate-nix: - nix run --extra-experimental-features flakes -f . regenerateNixLockfiles + nix run --extra-experimental-features "nix-command flakes" -f . regenerateNixLockfiles build: regenerate-charts regenerate-nix helm-package docker-build +# This target is used by the CI +# It doesn't make use of any nix dependencies and thus aviods building the +# operator unnecessarily often. +build-ci: regenerate-charts helm-package docker-build + publish: docker-publish helm-publish -run-dev: +check-nix: + @which nix || (echo "Error: 'nix' is not installed. Please install it to proceed."; exit 1) + +check-kubernetes: + @kubectl cluster-info > /dev/null 2>&1 || (echo "Error: Kubernetes is not running or kubectl is not properly configured."; exit 1) + +run-dev: check-nix check-kubernetes kubectl apply -f deploy/stackable-operators-ns.yaml - nix run -f. tilt -- up --port 5480 --namespace stackable-operators + nix run --extra-experimental-features "nix-command flakes" -f. tilt -- up --port 5480 --namespace stackable-operators + +stop-dev: check-nix check-kubernetes + nix run --extra-experimental-features "nix-command flakes" -f. tilt -- down