From 54e733c31e05cc168d74f679217998dc3c672626 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Wed, 11 Dec 2024 16:39:12 +0800 Subject: [PATCH 1/5] Attach migration file to task bundle STONEBLD-3026 * Introduce concepts of normal task (e.g. buildah) and extended task (e.g. buildah-20gb, buildah-min). * Refactor the code that processes task build and push. * For testing purpose, add a simple local cache in order to reduce the number of skopeo-inpsect runs for checking existence of task bundles. This could also be helpful to reduce the whole script execution during testing. * For testing purpose, script can be configured with environment variable TEST_TASKS to run and check the result for a small number of tasks. To test the attach: * Open a new branch. * Bump version of a task in the YAML definition, e.g. task/buildah/0.2 * Create task/buildah/0.2/migrations/.sh * Commit the changes * Run hack/build-and-push.sh by configuring it to push images to your own test image repository. * Find out the buildah task bundle. * Run `oras discover '. The attached migration file should be listed. --- hack/build-and-push.sh | 428 +++++++++++++++++++++++++++++++++-------- 1 file changed, 352 insertions(+), 76 deletions(-) diff --git a/hack/build-and-push.sh b/hack/build-and-push.sh index e92af09de6..f1fc9a55c3 100755 --- a/hack/build-and-push.sh +++ b/hack/build-and-push.sh @@ -1,10 +1,56 @@ #!/bin/bash +# There are two major types of tasks in build-definitions: +# +# - Normal tasks, which are written as Tekton Task resource. +# +# - kustomized tasks, which are customized by kustomization based on normal +# tasks. kustomized tasks can be customized either based on another normal +# task, e.g. task buildah-24gb is based on task buildah. This type of +# kustomized task inherits the interface without change. Or based on itself, +# e.g. task inspect image. +# +# Task are built and pushed to the registry as Tekton task bundles. There are +# two kinds of tags in a single task bundle repository. +# +# - floating tag: this is the version defined in the task directory path, +# e.g. 0.1 included in path task/buildah/0.1. It always points to +# the newest pushed bundle of the version. +# +# - fixed tag: it is in form - to identifies a +# single bundle. For normal tasks, the identifier is the git commit hash +# where the task gets update. For the kustomized tasks, the identifier is +# calculated from the task YAML content generated by kustomization. +# +# Configuration +# +# - ENABLE_CACHE: enable a local cache to reduce the number of skopeo-inspect +# runs for fetching image digest. This is useful for local tesing particually. +# Do not use it in the CI. +# +# - TEST_TASKS: script builds and pushes tasks only listed in this value. For +# testing purpose only. It is useful for checking the result task bundles +# generally. Note that, if used, the result pipelines are broken. + set -e -o pipefail VCS_URL=https://github.com/konflux-ci/build-definitions VCS_REF=$(git rev-parse HEAD) +declare -r ARTIFACT_TYPE_TEXT_XSHELLSCRIPT="text/x-shellscript" +declare -r ANNOTATION_TASK_MIGRATION="dev.konflux-ci.task.migration" +declare -r TEST_TASKS=${TEST_TASKS:-""} + +AUTH_JSON= +if [ -e "$XDG_RUNTIME_DIR/containers/auth.json" ]; then + AUTH_JSON="$XDG_RUNTIME_DIR/containers/auth.json" +elif [ -e "$HOME/.docker/config.json" ]; then + AUTH_JSON="$HOME/.docker/config.json" +else + echo "warning: cannot find registry authentication file." 1>&2 +fi +declare -r AUTH_JSON + function is_official_repo() { # match e.g. # redhat-appstudio-tekton-catalog @@ -37,20 +83,21 @@ SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd "$SCRIPTDIR/.." || exit 1 WORKDIR=$(mktemp -d --suffix "-$(basename "${BASH_SOURCE[0]}" .sh)") +declare -r WORKDIR -tkn_bundle_push() { +retry() { local status local retry=0 local -r interval=${RETRY_INTERVAL:-5} local -r max_retries=5 while true; do - tkn bundle push "$@" && break + "$@" && break status=$? ((retry+=1)) if [ $retry -gt $max_retries ]; then return $status fi - echo "Waiting for a while, then retry the tkn bundle push ..." + echo "info: Waiting for a while, then retry ..." 1>&2 sleep "$interval" done } @@ -133,95 +180,324 @@ if [ "$SKIP_BUILD" == "" ]; then fi -generated_pipelines_dir=$(mktemp -d -p "$WORKDIR" pipelines.XXXXXXXX) -oc kustomize --output "$generated_pipelines_dir" pipelines/ +GENERATED_PIPELINES_DIR=$(mktemp -d -p "$WORKDIR" pipelines.XXXXXXXX) +declare -r GENERATED_PIPELINES_DIR +oc kustomize --output "$GENERATED_PIPELINES_DIR" pipelines/ # Generate YAML files separately since pipelines for core services have same .metadata.name. -core_services_pipelines_dir=$(mktemp -d -p "$WORKDIR" core-services-pipelines.XXXXXXXX) -oc kustomize --output "$core_services_pipelines_dir" pipelines/core-services/ +CORE_SERVICES_PIPELINES_DIR=$(mktemp -d -p "$WORKDIR" core-services-pipelines.XXXXXXXX) +declare -r CORE_SERVICES_PIPELINES_DIR +oc kustomize --output "$CORE_SERVICES_PIPELINES_DIR" pipelines/core-services/ -# Build tasks -find task/*/*/ -maxdepth 0 -type d | awk -F '/' '{ print $0, $2, $3 }' | \ -while read -r task_dir task_name task_version -do - if should_skip_repo "$QUAY_NAMESPACE" "task-$task_name"; then - echo "NOTE: not pushing task-$task_name:$task_version to $QUAY_NAMESPACE; the repo does not exist and $QUAY_NAMESPACE is deprecated" - continue + +inject_bundle_ref_to_pipelines() { + local -r task_name=$1 + local -r task_version=$2 + local -r task_bundle_with_digest=$3 + sub_expr_1=" + (.spec.tasks[].taskRef | select(.name == \"${task_name}\" and .version == \"${task_version}\" )) + |= {\"resolver\": \"bundles\", \"params\": [ { \"name\": \"name\", \"value\": \"${task_name}\" } , { \"name\": \"bundle\", \"value\": \"${task_bundle_with_digest}\" }, { \"name\": \"kind\", \"value\": \"task\" }] } + " + sub_expr_2=" + (.spec.finally[].taskRef | select(.name == \"${task_name}\" and .version == \"${task_version}\" )) + |= {\"resolver\": \"bundles\", \"params\": [ { \"name\": \"name\", \"value\": \"${task_name}\" } , { \"name\": \"bundle\", \"value\": \"${task_bundle_with_digest}\" },{ \"name\": \"kind\", \"value\": \"task\" }] } + " + for filename in "$GENERATED_PIPELINES_DIR"/*.yaml "$CORE_SERVICES_PIPELINES_DIR"/*.yaml + do + yq e "$sub_expr_1" -i "${filename}" + yq e "$sub_expr_2" -i "${filename}" + done +} + +# Get task version from task definition rather than the version in the directory path. +# Arguments: task_file +# The version is output to stdout. +get_concrete_task_version() { + local -r task_file=$1 + # Ensure an empty string is output rather than string "null" if the version label is not present + yq '.metadata.labels."app.kubernetes.io/version"' "$task_file" | sed '/null/d' | tr -d '[:space:]' +} + +# Build and push a task as a Tekton bundle, that is tagged a floating tag and a fixed tag. +# The task bundle reference with digest is output to stdout in the last line, +# that is extracted from tkn-bundle-push. +build_push_task() { + local -r task_dir=$1 + local -r prepared_task_file=$2 + local -r task_bundle=$3 + local -r task_file_sha=$4 + # local -r extra_annotations=${5:-""} + + local -r task_description=$(yq e '.spec.description' "$prepared_task_file" | head -n 1) + + local -a ANNOTATIONS=() + ANNOTATIONS+=("org.opencontainers.image.source=${VCS_URL}") + ANNOTATIONS+=("org.opencontainers.image.revision=${VCS_REF}") + ANNOTATIONS+=("org.opencontainers.image.url=${VCS_URL}/tree/${VCS_REF}/${task_dir}") + ANNOTATIONS+=("org.opencontainers.image.version=$(get_concrete_task_version "$prepared_task_file")") + # yq will return null if the element is missing. + if [[ "${task_description}" != "null" ]]; then + ANNOTATIONS+=("org.opencontainers.image.description=${task_description}") + fi + if [ -f "${task_dir}/README.md" ]; then + ANNOTATIONS+=("org.opencontainers.image.documentation=${VCS_URL}/tree/${VCS_REF}/${task_dir}README.md") + fi + if [ -f "${task_dir}/TROUBLESHOOTING.md" ]; then + ANNOTATIONS+=("dev.tekton.docs.troubleshooting=${VCS_URL}/tree/${VCS_REF}/${task_dir}TROUBLESHOOTING.md") + fi + if [ -f "${task_dir}/USAGE.md" ]; then + ANNOTATIONS+=("dev.tekton.docs.usage=${VCS_URL}/tree/${VCS_REF}/${task_dir}USAGE.md") fi - prepared_task_file="${WORKDIR}/$task_name-${task_version}.yaml" - if [ -f "$task_dir/$task_name.yaml" ]; then - cp "$task_dir/$task_name.yaml" "$prepared_task_file" - task_file_sha=$(git log -n 1 --pretty=format:%H -- "$task_dir/$task_name.yaml") - elif [ -f "$task_dir/kustomization.yaml" ]; then - oc kustomize "$task_dir" > "$prepared_task_file" - task_file_sha=$(sha256sum "$prepared_task_file" | awk '{print $1}') - else - echo Unknown task in "$task_dir" - continue + local -a ANNOTATION_FLAGS=() + for annotation in "${ANNOTATIONS[@]}"; do + ANNOTATION_FLAGS+=("--annotate" "$(escape_tkn_bundle_arg "$annotation")") + done + + output=$( + retry tkn bundle push "${ANNOTATION_FLAGS[@]}" -f "$prepared_task_file" "$task_bundle" \ + | save_ref "$task_bundle" "$OUTPUT_TASK_BUNDLE_LIST" + ) + echo "$output" + + # copy task to new tag pointing to commit where the file was changed lastly, so that image persists + # even when original tag is updated + skopeo copy "docker://${task_bundle}" "docker://${task_bundle}-${task_file_sha}" + + # Ensure task bundle reference with digest is the last one so that function + # caller can extract it easily. + local -r task_bundle_with_digest="${output##*$'\n'}" + + cache_set "${task_bundle}-${task_file_sha}" "${task_bundle_with_digest#*@}" + echo "$task_bundle_with_digest" +} + +# Determine if a task is a normal task. 0 returns if it is, otherwise 1 is returned. +is_normal_task() { + local -r task_dir=$1 + local -r task_name=$2 + if [ -f "${task_dir}/${task_name}.yaml" ]; then + return 0 fi - repository=${TEST_REPO_NAME:-task-${task_name}} - tag=${TEST_REPO_NAME:+${task_name}-}${task_version} - task_bundle=quay.io/$QUAY_NAMESPACE/${repository}:${tag} - task_description=$(yq e '.spec.description' "$prepared_task_file" | head -n 1) + return 1 +} - if digest=$(skopeo inspect --no-tags --format='{{.Digest}}' docker://"${task_bundle}-${task_file_sha}" 2>/dev/null); then - task_bundle_with_digest=${task_bundle}@${digest} - else - ANNOTATIONS=() - ANNOTATIONS+=("org.opencontainers.image.source=${VCS_URL}") - ANNOTATIONS+=("org.opencontainers.image.revision=${VCS_REF}") - ANNOTATIONS+=("org.opencontainers.image.url=${VCS_URL}/tree/${VCS_REF}/${task_dir}") - # Ensure an empty string is set rather than string "null" if the version label is not present - concrete_task_version=$(yq '.metadata.labels."app.kubernetes.io/version"' "$prepared_task_file" | sed '/null/d') - ANNOTATIONS+=("org.opencontainers.image.version=${concrete_task_version}") - # yq will return null if the element is missing. - if [[ "${task_description}" != "null" ]]; then - ANNOTATIONS+=("org.opencontainers.image.description=${task_description}") - fi - if [ -f "${task_dir}/README.md" ]; then - ANNOTATIONS+=("org.opencontainers.image.documentation=${VCS_URL}/tree/${VCS_REF}/${task_dir}README.md") +# Determine if a task is a kustomized task. 0 returns if it is, otherwise 1 is returned. +is_kustomized_task() { + local -r task_dir=$1 + local -r task_name=$2 + local -r kt_config_file="$task_dir/kustomization.yaml" + local -r task_file="${task_dir}/${task_name}.yaml" + if [ -f "$kt_config_file" ] && [ ! -e "$task_file" ]; then + return 0 + fi + return 1 +} + +# Generates task bundle with tag. The result bundle reference can be configured +# by environment variable TEST_REPO_NAME for testing purpose. +# Arguments: task_name, task_version +# Task bundle reference is output to stdout. +generate_tagged_task_bundle() { + local -r task_name=$1 task_version=$2 + local -r repository=${TEST_REPO_NAME:-task-${task_name}} + local -r tag=${TEST_REPO_NAME:+${task_name}-}${task_version} + echo "quay.io/${QUAY_NAMESPACE}/${repository}:${tag}" +} + +# Generate build data for a normal task. The data is output to stdout as +# space-separated fields in a single line. +# Arguments: task_dir, task_name, task_version +generate_normal_task_build_data() { + local -r task_dir=$1 task_name=$2 task_version=$3 + local -r task_file="${task_dir}/${task_name}.yaml" + local -r prepared_task_file="${WORKDIR}/${task_name}-${task_version}.yaml" + cp "$task_file" "$prepared_task_file" + local -r task_file_sha=$(git log -n 1 --pretty=format:%H -- "$task_file") + local -r task_bundle=$(generate_tagged_task_bundle "$task_name" "$task_version") + echo "$prepared_task_file $task_file_sha $task_bundle" +} + +# Generate build data for a normal task. The data is output to stdout as +# space-separated fields in a single line. +# Arguments: task_dir, task_name, task_version +generate_kustomized_task_build_data() { + local -r task_dir=$1 task_name=$2 task_version=$3 + local -r task_file="$task_dir/$task_name.yaml" + local -r prepared_task_file="${WORKDIR}/${task_name}-${task_version}.yaml" + oc kustomize "$task_dir" >"$prepared_task_file" + local -r task_file_sha=$(sha256sum "$prepared_task_file" | awk '{print $1}') + local -r task_bundle=$(generate_tagged_task_bundle "$task_name" "$task_version") + echo "$prepared_task_file $task_file_sha $task_bundle" +} + +declare -r ENABLE_CACHE=${ENABLE_CACHE:-""} +CACHE_FILE= + +if [ -n "$ENABLE_CACHE" ]; then + CACHE_FILE="/tmp/build-definitions-build-and-push.cache" + declare -r CACHE_FILE +fi + +cache_get() { + local -r key=$1 + local value= + if [ -n "$ENABLE_CACHE" ]; then + # No match should not fail the script. It means no cached data. + read -r _ value < <(grep "^${key}" "$CACHE_FILE" 2>/dev/null) || : + fi + echo "$value" +} + +cache_set() { + local -r key=$1 value=$2 + if [ -n "$ENABLE_CACHE" ]; then + echo "${key} ${value}" >>"$CACHE_FILE" + fi +} + +# Fetch image digest. +# Arguments: image +# The digest is output to stdout. +fetch_image_digest() { + local -r image=$1 + local digest= + digest=$(cache_get "$image") + if [ -z "$digest" ]; then + digest=$(skopeo inspect --no-tags --format='{{.Digest}}' "docker://${image}" 2>/dev/null) + if [ -n "$digest" ]; then + cache_set "$image" "$digest" fi - if [ -f "${task_dir}/TROUBLESHOOTING.md" ]; then - ANNOTATIONS+=("dev.tekton.docs.troubleshooting=${VCS_URL}/tree/${VCS_REF}/${task_dir}TROUBLESHOOTING.md") + fi + echo "$digest" +} + +# Attach migration file to given task bundle. +# Arguments: task_dir, concrete_task_version, task_bundle +attach_migration_file() { + local -r task_dir=$1 + local -r concrete_task_version=$2 + local -r task_bundle=$3 + + local -r task_file="${task_dir}/${task_name}.yaml" + local -r migration_file="${task_dir}/migrations/${concrete_task_version}.sh" + + if [ ! -e "$migration_file" ]; then + return 0 + fi + + # Check if task bundle has an attached migration file. + local filename + local found= + local artifact_refs + + # List attached artifacts, that have specific artifact type and annotation. + # Then, find out the migration artifact. + artifact_refs=$( + retry oras discover "$task_bundle" --artifact-type "$ARTIFACT_TYPE_TEXT_XSHELLSCRIPT" --format json | \ + jq -r " + .manifests[] + | select(.annotations.\"${ANNOTATION_TASK_MIGRATION}\" == \"true\") + | .reference" + ) + while read -r artifact_ref; do + if [ -z "$artifact_ref" ]; then + continue fi - if [ -f "${task_dir}/USAGE.md" ]; then - ANNOTATIONS+=("dev.tekton.docs.usage=${VCS_URL}/tree/${VCS_REF}/${task_dir}USAGE.md") + filename=$( + retry oras pull --format json "$artifact_ref" | jq -r " + .files[] + | select(.annotations.\"org.opencontainers.image.title\" == \"${concrete_task_version}.sh\") + | .annotations.\"org.opencontainers.image.title\" + " + ) + + if [ -n "$filename" ]; then + if diff "$filename" "$migration_file" >/dev/null; then + found=true + break + else + echo "error: task bundle $task_bundle has migration artifact $artifact_ref, but the migration content is different: $filename" 1>&2 + exit 1 + fi fi + done <<<"$artifact_refs" - ANNOTATION_FLAGS=() - for annotation in "${ANNOTATIONS[@]}"; do - ANNOTATION_FLAGS+=("--annotate" "$(escape_tkn_bundle_arg "$annotation")") - done + if [ -n "$found" ]; then + return 0 + fi - output=$(tkn_bundle_push "${ANNOTATION_FLAGS[@]}" -f "$prepared_task_file" "$task_bundle" | save_ref "$task_bundle" "$OUTPUT_TASK_BUNDLE_LIST") - echo "$output" - task_bundle_with_digest="${output##*$'\n'}" + ( + cd "${migration_file%/*}" + retry oras attach \ + --registry-config "$AUTH_JSON" \ + --artifact-type "$ARTIFACT_TYPE_TEXT_XSHELLSCRIPT" \ + --distribution-spec v1.1-referrers-tag \ + --annotation "$ANNOTATION_TASK_MIGRATION=true" \ + "$task_bundle" "${migration_file##*/}" + ) - # copy task to new tag pointing to commit where the file was changed lastly, so that image persists - # even when original tag is updated - skopeo copy "docker://${task_bundle}" "docker://${task_bundle}-${task_file_sha}" - fi - # version placeholder is removed naturally by the substitution. - real_task_name=$(yq e '.metadata.name' "$prepared_task_file") - sub_expr_1=" - (.spec.tasks[].taskRef | select(.name == \"${real_task_name}\" and .version == \"${task_version}\" )) - |= {\"resolver\": \"bundles\", \"params\": [ { \"name\": \"name\", \"value\": \"${real_task_name}\" } , { \"name\": \"bundle\", \"value\": \"${task_bundle_with_digest}\" }, { \"name\": \"kind\", \"value\": \"task\" }] } - " - sub_expr_2=" - (.spec.finally[].taskRef | select(.name == \"${real_task_name}\" and .version == \"${task_version}\" )) - |= {\"resolver\": \"bundles\", \"params\": [ { \"name\": \"name\", \"value\": \"${real_task_name}\" } , { \"name\": \"bundle\", \"value\": \"${task_bundle_with_digest}\" },{ \"name\": \"kind\", \"value\": \"task\" }] } - " - for filename in "$generated_pipelines_dir"/*.yaml "$core_services_pipelines_dir"/*.yaml + echo + echo "Attached migration file ${migration_file} to ${task_bundle}" + + return 0 +} + +build_push_tasks() { + local build_data + local task_bundle_with_digest + local migration_file + local prepared_task_file + local task_file_sha + local task_bundle + + find task/*/* -maxdepth 0 -type d | awk -F '/' '{ print $0, $2, $3 }' | \ + while read -r task_dir task_name task_version do - yq e "$sub_expr_1" -i "${filename}" - yq e "$sub_expr_2" -i "${filename}" + if [ -n "$TEST_TASKS" ] && echo "$TEST_TASKS" | grep -v "$task_name" 2>/dev/null; then + continue + fi + + if should_skip_repo "$QUAY_NAMESPACE" "task-${task_name}"; then + echo "NOTE: not pushing task-$task_name:$task_version to $QUAY_NAMESPACE; the repo does not exist and $QUAY_NAMESPACE is deprecated" + continue + fi + + if is_normal_task "$task_dir" "$task_name"; then + build_data=$(generate_normal_task_build_data "$task_dir" "$task_name" "$task_version") + elif is_kustomized_task "$task_dir" "$task_name"; then + build_data=$(generate_kustomized_task_build_data "$task_dir" "$task_name" "$task_version") + else + continue + fi + + read -r prepared_task_file task_file_sha task_bundle <<<"$build_data" + digest=$(fetch_image_digest "${task_bundle}-${task_file_sha}") + + if [ -n "$digest" ]; then + task_bundle_with_digest=${task_bundle}@${digest} + else + task_bundle_with_digest=$( + build_push_task "$task_dir" "$prepared_task_file" "$task_bundle" "$task_file_sha" \ + | tail -n 1 + ) + fi + + concrete_task_version=$(get_concrete_task_version "$prepared_task_file") + attach_migration_file "$task_dir" "$concrete_task_version" "$task_bundle_with_digest" + + # version placeholder is removed naturally by the substitution. + real_task_name=$(yq e '.metadata.name' "$prepared_task_file") + inject_bundle_ref_to_pipelines "$real_task_name" "$task_version" "$task_bundle_with_digest" done -done +} + +build_push_tasks + # Used for build-definitions pull request CI only if [ -n "$ENABLE_SOURCE_BUILD" ]; then - for pipeline_yaml in "$generated_pipelines_dir"/*.yaml; do + for pipeline_yaml in "$GENERATED_PIPELINES_DIR"/*.yaml; do yq e '(.spec.params[] | select(.name == "build-source-image") | .default) = "true"' -i "$pipeline_yaml" done fi @@ -232,7 +508,7 @@ if [ "$QUAY_NAMESPACE" == redhat-appstudio-tekton-catalog ]; then fi # Build Pipeline bundle with pipelines pointing to newly built task bundles -for pipeline_yaml in "$generated_pipelines_dir"/*.yaml "$core_services_pipelines_dir"/*.yaml +for pipeline_yaml in "$GENERATED_PIPELINES_DIR"/*.yaml "$CORE_SERVICES_PIPELINES_DIR"/*.yaml do pipeline_name=$(yq e '.metadata.name' "$pipeline_yaml") pipeline_description=$(yq e '.spec.description' "$pipeline_yaml" | head -n 1) @@ -270,7 +546,7 @@ do ANNOTATION_FLAGS+=("--annotate" "$(escape_tkn_bundle_arg "$annotation")") done - tkn_bundle_push "${ANNOTATION_FLAGS[@]}" "$pipeline_bundle" -f "${pipeline_yaml}" | \ + retry tkn bundle push "${ANNOTATION_FLAGS[@]}" "$pipeline_bundle" -f "${pipeline_yaml}" | \ save_ref "$pipeline_bundle" "$OUTPUT_PIPELINE_BUNDLE_LIST" [ "$pipeline_name" == "docker-build" ] && docker_pipeline_bundle=$pipeline_bundle From 3aba067e540e617b535ccec866739464919e2329 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Fri, 13 Dec 2024 15:44:07 +0800 Subject: [PATCH 2/5] make bundle ref string more readable --- hack/build-and-push.sh | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/hack/build-and-push.sh b/hack/build-and-push.sh index f1fc9a55c3..f76de84dfb 100755 --- a/hack/build-and-push.sh +++ b/hack/build-and-push.sh @@ -194,19 +194,20 @@ inject_bundle_ref_to_pipelines() { local -r task_name=$1 local -r task_version=$2 local -r task_bundle_with_digest=$3 - sub_expr_1=" - (.spec.tasks[].taskRef | select(.name == \"${task_name}\" and .version == \"${task_version}\" )) - |= {\"resolver\": \"bundles\", \"params\": [ { \"name\": \"name\", \"value\": \"${task_name}\" } , { \"name\": \"bundle\", \"value\": \"${task_bundle_with_digest}\" }, { \"name\": \"kind\", \"value\": \"task\" }] } - " - sub_expr_2=" - (.spec.finally[].taskRef | select(.name == \"${task_name}\" and .version == \"${task_version}\" )) - |= {\"resolver\": \"bundles\", \"params\": [ { \"name\": \"name\", \"value\": \"${task_name}\" } , { \"name\": \"bundle\", \"value\": \"${task_bundle_with_digest}\" },{ \"name\": \"kind\", \"value\": \"task\" }] } - " - for filename in "$GENERATED_PIPELINES_DIR"/*.yaml "$CORE_SERVICES_PIPELINES_DIR"/*.yaml - do - yq e "$sub_expr_1" -i "${filename}" - yq e "$sub_expr_2" -i "${filename}" - done + local -r bundle_ref="{ + \"resolver\": \"bundles\", + \"params\": [ + {\"name\": \"name\", \"value\": \"${task_name}\"}, + {\"name\": \"bundle\", \"value\": \"${task_bundle_with_digest}\"}, + {\"name\": \"kind\", \"value\": \"task\"} + ] + }" + local -r task_selector="select(.name == \"${task_name}\" and .version == \"${task_version}\")" + find "$GENERATED_PIPELINES_DIR" "$CORE_SERVICES_PIPELINES_DIR" -maxdepth 1 -type f -name '*.yaml' | \ + while read -r pipeline_file; do + yq e "(.spec.tasks[].taskRef | ${task_selector}) |= ${bundle_ref}" -i "${pipeline_file}" + yq e "(.spec.finally[].taskRef | ${task_selector}) |= ${bundle_ref}" -i "${pipeline_file}" + done } # Get task version from task definition rather than the version in the directory path. From 1dc00df7e147e28cf8b6acd9086710495424a820 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Mon, 16 Dec 2024 16:51:23 +0800 Subject: [PATCH 3/5] Address review comments --- hack/build-and-push.sh | 48 +++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/hack/build-and-push.sh b/hack/build-and-push.sh index f76de84dfb..73525e4864 100755 --- a/hack/build-and-push.sh +++ b/hack/build-and-push.sh @@ -8,7 +8,7 @@ # tasks. kustomized tasks can be customized either based on another normal # task, e.g. task buildah-24gb is based on task buildah. This type of # kustomized task inherits the interface without change. Or based on itself, -# e.g. task inspect image. +# e.g. task inspect-image. # # Task are built and pushed to the registry as Tekton task bundles. There are # two kinds of tags in a single task bundle repository. @@ -227,7 +227,6 @@ build_push_task() { local -r prepared_task_file=$2 local -r task_bundle=$3 local -r task_file_sha=$4 - # local -r extra_annotations=${5:-""} local -r task_description=$(yq e '.spec.description' "$prepared_task_file" | head -n 1) @@ -241,13 +240,13 @@ build_push_task() { ANNOTATIONS+=("org.opencontainers.image.description=${task_description}") fi if [ -f "${task_dir}/README.md" ]; then - ANNOTATIONS+=("org.opencontainers.image.documentation=${VCS_URL}/tree/${VCS_REF}/${task_dir}README.md") + ANNOTATIONS+=("org.opencontainers.image.documentation=${VCS_URL}/tree/${VCS_REF}/${task_dir}/README.md") fi if [ -f "${task_dir}/TROUBLESHOOTING.md" ]; then - ANNOTATIONS+=("dev.tekton.docs.troubleshooting=${VCS_URL}/tree/${VCS_REF}/${task_dir}TROUBLESHOOTING.md") + ANNOTATIONS+=("dev.tekton.docs.troubleshooting=${VCS_URL}/tree/${VCS_REF}/${task_dir}/TROUBLESHOOTING.md") fi if [ -f "${task_dir}/USAGE.md" ]; then - ANNOTATIONS+=("dev.tekton.docs.usage=${VCS_URL}/tree/${VCS_REF}/${task_dir}USAGE.md") + ANNOTATIONS+=("dev.tekton.docs.usage=${VCS_URL}/tree/${VCS_REF}/${task_dir}/USAGE.md") fi local -a ANNOTATION_FLAGS=() @@ -255,22 +254,12 @@ build_push_task() { ANNOTATION_FLAGS+=("--annotate" "$(escape_tkn_bundle_arg "$annotation")") done - output=$( - retry tkn bundle push "${ANNOTATION_FLAGS[@]}" -f "$prepared_task_file" "$task_bundle" \ - | save_ref "$task_bundle" "$OUTPUT_TASK_BUNDLE_LIST" - ) - echo "$output" + retry tkn bundle push "${ANNOTATION_FLAGS[@]}" -f "$prepared_task_file" "$task_bundle" \ + | save_ref "$task_bundle" "$OUTPUT_TASK_BUNDLE_LIST" # copy task to new tag pointing to commit where the file was changed lastly, so that image persists # even when original tag is updated skopeo copy "docker://${task_bundle}" "docker://${task_bundle}-${task_file_sha}" - - # Ensure task bundle reference with digest is the last one so that function - # caller can extract it easily. - local -r task_bundle_with_digest="${output##*$'\n'}" - - cache_set "${task_bundle}-${task_file_sha}" "${task_bundle_with_digest#*@}" - echo "$task_bundle_with_digest" } # Determine if a task is a normal task. 0 returns if it is, otherwise 1 is returned. @@ -344,8 +333,7 @@ cache_get() { local -r key=$1 local value= if [ -n "$ENABLE_CACHE" ]; then - # No match should not fail the script. It means no cached data. - read -r _ value < <(grep "^${key}" "$CACHE_FILE" 2>/dev/null) || : + value=$(awk -v key="$key" '$1 == key { print $2 }' <"$CACHE_FILE") fi echo "$value" } @@ -380,7 +368,6 @@ attach_migration_file() { local -r concrete_task_version=$2 local -r task_bundle=$3 - local -r task_file="${task_dir}/${task_name}.yaml" local -r migration_file="${task_dir}/migrations/${concrete_task_version}.sh" if [ ! -e "$migration_file" ]; then @@ -433,7 +420,7 @@ attach_migration_file() { retry oras attach \ --registry-config "$AUTH_JSON" \ --artifact-type "$ARTIFACT_TYPE_TEXT_XSHELLSCRIPT" \ - --distribution-spec v1.1-referrers-tag \ + --distribution-spec v1.1-referrers-api \ --annotation "$ANNOTATION_TASK_MIGRATION=true" \ "$task_bundle" "${migration_file##*/}" ) @@ -451,11 +438,12 @@ build_push_tasks() { local prepared_task_file local task_file_sha local task_bundle + local output find task/*/* -maxdepth 0 -type d | awk -F '/' '{ print $0, $2, $3 }' | \ while read -r task_dir task_name task_version do - if [ -n "$TEST_TASKS" ] && echo "$TEST_TASKS" | grep -v "$task_name" 2>/dev/null; then + if [ -n "$TEST_TASKS" ] && echo "$TEST_TASKS" | grep -qv "$task_name" 2>/dev/null; then continue fi @@ -464,11 +452,14 @@ build_push_tasks() { continue fi + echo "info: build and push task $task_dir" 1>&2 + if is_normal_task "$task_dir" "$task_name"; then build_data=$(generate_normal_task_build_data "$task_dir" "$task_name" "$task_version") elif is_kustomized_task "$task_dir" "$task_name"; then build_data=$(generate_kustomized_task_build_data "$task_dir" "$task_name" "$task_version") else + echo "warning: skip handling task $task_dir since it does not follow a known task definition structure." 1>&2 continue fi @@ -477,17 +468,22 @@ build_push_tasks() { if [ -n "$digest" ]; then task_bundle_with_digest=${task_bundle}@${digest} + echo "info: use existing $task_bundle_with_digest" 1>&2 else - task_bundle_with_digest=$( - build_push_task "$task_dir" "$prepared_task_file" "$task_bundle" "$task_file_sha" \ - | tail -n 1 - ) + echo "info: push new bundle $task_bundle" 1>&2 + + output=$(build_push_task "$task_dir" "$prepared_task_file" "$task_bundle" "$task_file_sha") + + task_bundle_with_digest=$(grep -m 1 "^Pushed Tekton Bundle to" <<<"$output" 2>/dev/null) + task_bundle_with_digest=${task_bundle_with_digest##* } + cache_set "${task_bundle}-${task_file_sha}" "${task_bundle_with_digest#*@}" fi concrete_task_version=$(get_concrete_task_version "$prepared_task_file") attach_migration_file "$task_dir" "$concrete_task_version" "$task_bundle_with_digest" # version placeholder is removed naturally by the substitution. + echo "info: inject task bundle to pielines $task_bundle_with_digest" 1>&2 real_task_name=$(yq e '.metadata.name' "$prepared_task_file") inject_bundle_ref_to_pipelines "$real_task_name" "$task_version" "$task_bundle_with_digest" done From 02de4ebb5eb1856ca5976ffc046221b59d5bd3d6 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Mon, 16 Dec 2024 17:45:43 +0800 Subject: [PATCH 4/5] Ensure required version of oras is available --- appstudio-utils/Dockerfile | 4 ++++ hack/build-and-push.sh | 2 ++ 2 files changed, 6 insertions(+) diff --git a/appstudio-utils/Dockerfile b/appstudio-utils/Dockerfile index 50acc53f55..a125593696 100755 --- a/appstudio-utils/Dockerfile +++ b/appstudio-utils/Dockerfile @@ -10,6 +10,10 @@ RUN curl -L https://github.com/open-policy-agent/conftest/releases/download/v0.3 RUN curl -L https://github.com/enterprise-contract/ec-cli/releases/download/snapshot/ec_linux_amd64 -o /usr/bin/ec && chmod +x /usr/bin/ec && ec version RUN curl -L https://github.com/cli/cli/releases/download/v2.60.1/gh_2.60.1_linux_amd64.tar.gz | tar -xz -C /usr/bin --wildcards "gh_*/bin/gh" --strip-components=2 --no-same-owner +# 1.2.0 is the minimum required version +RUN curl -L https://github.com/oras-project/oras/releases/download/v1.2.1/oras_1.2.1_linux_amd64.tar.gz | \ + tar -xz --no-same-owner -C /usr/bin oras + RUN dnf -y --setopt=tsflags=nodocs install \ git \ skopeo \ diff --git a/hack/build-and-push.sh b/hack/build-and-push.sh index 73525e4864..8ab12334ec 100755 --- a/hack/build-and-push.sh +++ b/hack/build-and-push.sh @@ -381,6 +381,8 @@ attach_migration_file() { # List attached artifacts, that have specific artifact type and annotation. # Then, find out the migration artifact. + # + # Minimum version oras 1.2.0 is required for option --format artifact_refs=$( retry oras discover "$task_bundle" --artifact-type "$ARTIFACT_TYPE_TEXT_XSHELLSCRIPT" --format json | \ jq -r " From c071ad9f52df869e46c02e8051e397662ae7dc26 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Mon, 16 Dec 2024 20:36:03 +0800 Subject: [PATCH 5/5] Let oras choose distribution spec --- hack/build-and-push.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/hack/build-and-push.sh b/hack/build-and-push.sh index 8ab12334ec..a3086d6112 100755 --- a/hack/build-and-push.sh +++ b/hack/build-and-push.sh @@ -422,7 +422,6 @@ attach_migration_file() { retry oras attach \ --registry-config "$AUTH_JSON" \ --artifact-type "$ARTIFACT_TYPE_TEXT_XSHELLSCRIPT" \ - --distribution-spec v1.1-referrers-api \ --annotation "$ANNOTATION_TASK_MIGRATION=true" \ "$task_bundle" "${migration_file##*/}" )