-
Notifications
You must be signed in to change notification settings - Fork 52
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
refactor(RELEASE-1041): move and rename iib pipeline
this PR moves the IIB internal pipeline and tasks to the release-service-catalog and renames it to update-fbc-catalog. Signed-off-by: Leandro Mendes <[email protected]>
- Loading branch information
1 parent
1b1616f
commit 7cd8e49
Showing
3 changed files
with
397 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| # update-fbc-catalog pipeline | ||
|
|
||
| Tekton pipeline add/update FBC fragments to the FBC catalog by interacting with IIB service for File Based Catalogs | ||
|
|
||
| ## Parameters | ||
|
|
||
| | Name | Description | Optional | Default value | | ||
| |-------------------------|-----------------------------------------------------------------------------|----------|---------------------| | ||
| | iibServiceAccountSecret | Secret containing the credentials for IIB service | yes | iib-service-account | | ||
| | fbcFragment | FBC fragment built by HACBS | no | - | | ||
| | fromIndex | Index image (catalog of catalogs) the FBC fragment will be added to | no | - | | ||
| | buildTags | List of additional tags the internal index image copy should be tagged with | yes | '[]' | | ||
| | addArches | List of arches the index image should be built for | yes | '[]' | | ||
| | hotfix | Whether this build is a hotfix build | yes | false | | ||
| | stagedIndex | Whether this build is a staged index build | yes | false | | ||
| | buildTimeoutSeconds | IIB Build Service timeout seconds | no | - | |
80 changes: 80 additions & 0 deletions
80
internal/pipelines/update-fbc-catalog/update-fbc-catalog.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| --- | ||
| apiVersion: tekton.dev/v1 | ||
| kind: Pipeline | ||
| metadata: | ||
| name: update-fbc-catalog | ||
| labels: | ||
| app.kubernetes.io/version: "1.0.0" | ||
| annotations: | ||
| tekton.dev/pipelines.minVersion: "0.12.1" | ||
| tekton.dev/tags: fbc | ||
| spec: | ||
| description: >- | ||
| Tekton pipeline add/update FBC fragments to the FBC catalog by interacting with IIB service for File Based Catalogs | ||
| params: | ||
| - name: iibServiceAccountSecret | ||
| type: string | ||
| description: Secret containing the credentials for IIB service | ||
| default: iib-service-account | ||
| - name: fbcFragment | ||
| type: string | ||
| description: FBC fragment built by HACBS | ||
| - name: fromIndex | ||
| type: string | ||
| description: >- | ||
| Index image (catalog of catalogs) the FBC fragment will be added to | ||
| - name: buildTags | ||
| type: string | ||
| default: '[]' | ||
| description: >- | ||
| List of additional tags the internal index image copy should be | ||
| tagged with | ||
| - name: addArches | ||
| type: string | ||
| default: '[]' | ||
| description: List of arches the index image should be built for | ||
| - name: hotfix | ||
| type: string | ||
| default: "false" | ||
| description: Whether this build is a hotfix build | ||
| - name: stagedIndex | ||
| type: string | ||
| default: "false" | ||
| description: Whether this build is a staged index build | ||
| - name: buildTimeoutSeconds | ||
| type: string | ||
| description: IIB Build Service timeout seconds | ||
| tasks: | ||
| - name: update-fbc-catalog-task | ||
| taskRef: | ||
| name: update-fbc-catalog-task | ||
| params: | ||
| - name: iibServiceAccountSecret | ||
| value: $(params.iibServiceAccountSecret) | ||
| - name: fbcFragment | ||
| value: $(params.fbcFragment) | ||
| - name: fromIndex | ||
| value: $(params.fromIndex) | ||
| - name: buildTags | ||
| value: $(params.buildTags) | ||
| - name: addArches | ||
| value: $(params.addArches) | ||
| - name: hotfix | ||
| value: $(params.hotfix) | ||
| - name: stagedIndex | ||
| value: $(params.stagedIndex) | ||
| - name: buildTimeoutSeconds | ||
| value: $(params.buildTimeoutSeconds) | ||
| results: | ||
| - name: jsonBuildInfo | ||
| value: $(tasks.update-fbc-catalog-task.results.jsonBuildInfo) | ||
| - name: buildState | ||
| value: $(tasks.update-fbc-catalog-task.results.buildState) | ||
| - name: genericResult | ||
| value: $(tasks.update-fbc-catalog-task.results.genericResult) | ||
| - name: indexImageDigests | ||
| value: $(tasks.update-fbc-catalog-task.results.indexImageDigests) | ||
| - name: iibLog | ||
| value: $(tasks.update-fbc-catalog-task.results.iibLog) | ||
| - name: exitCode | ||
| value: $(tasks.update-fbc-catalog-task.results.exitCode) |
301 changes: 301 additions & 0 deletions
301
internal/tasks/update-fbc-catalog-task/update-fbc-catalog-task.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,301 @@ | ||
| --- | ||
| apiVersion: tekton.dev/v1 | ||
| kind: Task | ||
| metadata: | ||
| name: update-fbc-catalog | ||
| labels: | ||
| app.kubernetes.io/version: "1.0.0" | ||
| annotations: | ||
| tekton.dev/pipelines.minVersion: "0.12.1" | ||
| tekton.dev/tags: release | ||
| spec: | ||
| description: >- | ||
| Submit a build request to add operator bundles to an index image | ||
| params: | ||
| - name: fbcFragment | ||
| type: string | ||
| description: FBC fragment built by HACBS | ||
| - name: fromIndex | ||
| type: string | ||
| description: -> | ||
| Index image (catalog of catalogs) the FBC fragment will be added to | ||
| - name: buildTags | ||
| type: string | ||
| description: -> | ||
| List of additional tags the internal index image copy should be | ||
| tagged with. | ||
| - name: addArches | ||
| type: string | ||
| description: List of arches the index image should be built for. | ||
| - name: buildTimeoutSeconds | ||
| type: string | ||
| default: "300" | ||
| description: Timeout seconds to receive the build state | ||
| - name: iibServiceAccountSecret | ||
| type: string | ||
| description: Secret with IIB credentials to be used | ||
| - name: hotfix | ||
| type: string | ||
| default: "false" | ||
| description: Whether this build is a hotfix build | ||
| - name: stagedIndex | ||
| type: string | ||
| default: "false" | ||
| description: Whether this build is for a staged index build | ||
| results: | ||
| - name: jsonBuildInfo | ||
| description: JSON build information for the requested build | ||
| - name: buildState | ||
| description: IIB Service build state | ||
| - name: genericResult | ||
| description: Set the genericResult if FBC Fragment is Opt-In and should be published | ||
| - name: indexImageDigests | ||
| description: The digests for each arch for the manifest list of the index image | ||
| - name: iibLog | ||
| description: The link to the log from the IIB request | ||
| - name: exitCode | ||
| description: The exit code from the task | ||
| steps: | ||
| - name: update-fbc-catalog-prepare-and-call-iib-step | ||
| image: >- | ||
| quay.io/konflux-ci/release-service-utils:e633d51cd41d73e4b3310face21bb980af7a662f | ||
| env: | ||
| - name: IIB_SERVICE_URL | ||
| valueFrom: | ||
| secretKeyRef: | ||
| name: iib-services-config | ||
| key: url | ||
| - name: IIB_OVERWRITE_FROM_INDEX_USERNAME | ||
| valueFrom: | ||
| secretKeyRef: | ||
| name: iib-overwrite-fromimage-credentials | ||
| key: username | ||
| - name: IIB_OVERWRITE_FROM_INDEX_TOKEN | ||
| valueFrom: | ||
| secretKeyRef: | ||
| name: iib-overwrite-fromimage-credentials | ||
| key: token | ||
| - name: KRB5_CONF_CONTENT | ||
| valueFrom: | ||
| secretKeyRef: | ||
| name: iib-services-config | ||
| key: krb5.conf | ||
| script: | | ||
| #!/usr/bin/env bash | ||
| isFBCOptIn() { | ||
| TMPFILE=$(mktemp) | ||
| PYXIS_URL="https://pyxis.engineering.redhat.com/v1" | ||
| IFS="/" read -r REGISTRY REPO IMAGE <<< "${1}" | ||
| IFS=":" read -r IMAGE TAG <<< "${IMAGE}" | ||
| FETCH_URL="${PYXIS_URL}/repositories/registry/${REGISTRY}/repository/${REPO}/${IMAGE}/tag/${TAG}" | ||
| # strips the last "/tag" in case $TAG is not set | ||
| [ -z "${TAG}" ] && FETCH_URL=${FETCH_URL%/tag*} | ||
| curl --negotiate -u: "${FETCH_URL}" -o "${TMPFILE}" | ||
| # prints "false" in case .fbc_opt_in entry is missing | ||
| jq -e -r '.fbc_opt_in //false' "${TMPFILE}" && rm -f "${TMPFILE}" | ||
| } | ||
| # checks if there is any previous build for the same fbc_fragment. | ||
| # in case multiple builds are found, returns only the last one. | ||
| check_previous_build() { | ||
| user="${1}" | ||
| from_index="${2}" | ||
| fbc_fragment="${3}" | ||
| # fetch only builds in progress or completed | ||
| curl -s "${IIB_SERVICE_URL}/builds?user=${user}&from_index=${from_index}" | \ | ||
| jq --arg fbc_fragment "${fbc_fragment}" \ | ||
| '[.items[] |select(.fbc_fragment==$fbc_fragment and .state!="failed")][0] // empty' | ||
| } | ||
| # performs kerberos authentication. | ||
| base64 -d /mnt/service-account-secret/keytab > /tmp/keytab | ||
| KRB5_TEMP_CONF=$(mktemp) | ||
| KRB5_PRINCIPAL=$(cat /mnt/service-account-secret/principal) | ||
| echo "${KRB5_CONF_CONTENT}" > "${KRB5_TEMP_CONF}" | ||
| export KRB5_CONFIG="${KRB5_TEMP_CONF}" | ||
| export KRB5_TRACE=/dev/stderr | ||
| /usr/bin/kinit -V "${KRB5_PRINCIPAL}" -k -t /tmp/keytab | ||
| set -x | ||
| # check if this fbc fragment is opt-in | ||
| echo "Fetching the image bundle from $(params.fbcFragment)..." | ||
| PULL_SPEC_LIST=$(opm render "$(params.fbcFragment)" | jq -r \ | ||
| 'select(.schema == "olm.bundle") | "\(.image)" | split("@")[0]' |uniq) | ||
| fbcOptIn="true" | ||
| for PULL_SPEC in ${PULL_SPEC_LIST}; do | ||
| # make sure they query is done using the internal name instead of the public | ||
| PULL_SPEC="${PULL_SPEC//registry.redhat.io/registry.access.redhat.com}" | ||
| echo "Attempting to fetch from ${FETCH_URL} to check if fragment is \`fbc_opt_in==true\`..." | ||
| if [ "$(isFBCOptIn "${PULL_SPEC}")" = "false" ]; then | ||
| fbcOptIn="false" | ||
| break | ||
| fi | ||
| done | ||
| mustOverwriteFromIndexImage="${fbcOptIn}" | ||
| mustPublishIndexImage="${fbcOptIn}" | ||
| mustSignIndexImage="${fbcOptIn}" | ||
| if [ "$(params.hotfix)" == "true" ]; then | ||
| echo "Hotfix build" | ||
| mustOverwriteFromIndexImage="false" | ||
| mustSignIndexImage="true" | ||
| mustPublishIndexImage="true" | ||
| elif [ "$(params.stagedIndex)" == "true" ]; then | ||
| echo "Staged Index build" | ||
| mustOverwriteFromIndexImage="false" | ||
| mustSignIndexImage="false" | ||
| mustPublishIndexImage="false" | ||
| fi | ||
| echo "Fragment has \`fbc_opt_in==${fbcOptIn}\`" | ||
| echo " \`mustPublishIndexImage==${mustPublishIndexImage}\`" | ||
| echo " \`mustSignIndexImage==${mustSignIndexImage}\`" | ||
| # these results will be used by add-fbc-contribution to control | ||
| # signing and publishing of the built fragment | ||
| jq -n -c \ | ||
| --arg fbc_opt_in "${fbcOptIn}" \ | ||
| --arg publish_index_image "${mustPublishIndexImage}" \ | ||
| --arg sign_index_image "${mustSignIndexImage}" \ | ||
| '{ | ||
| "fbc_opt_in": $fbc_opt_in, | ||
| "publish_index_image": $publish_index_image, | ||
| "sign_index_image": $sign_index_image | ||
| } | tostring' | tee "$(results.genericResult.path)" | ||
| # if it finds a build which is completed or in progress, it should exit this step and jump to | ||
| # the next step `s-wait-for-build-state` which will watch the build until it is completed. | ||
| build=$(check_previous_build "${KRB5_PRINCIPAL}" "$(params.fromIndex)" "$(params.fbcFragment)") | ||
| if [ -n "${build}" ]; then | ||
| echo "=== A previous build for this fragment was found ===" | ||
| echo "${build}" |tee "$(results.jsonBuildInfo.path)" | ||
| exit 0 | ||
| fi | ||
| # adds the json request parameters to a file to be used as input data | ||
| # for curl and preventing shell expansion. | ||
| json_input=/tmp/$$.tmp | ||
| json_raw_input=/tmp/$$_raw.tmp | ||
| cat > $json_raw_input <<JSON | ||
| { | ||
| "fbc_fragment": "$(params.fbcFragment)", | ||
| "from_index": "$(params.fromIndex)", | ||
| "build_tags": $(params.buildTags), | ||
| "add_arches": $(params.addArches), | ||
| "overwrite_from_index": ${mustOverwriteFromIndexImage}, | ||
| "overwrite_from_index_token": "${IIB_OVERWRITE_FROM_INDEX_USERNAME}:${IIB_OVERWRITE_FROM_INDEX_TOKEN}" | ||
| } | ||
| JSON | ||
| # filtering out empty params | ||
| jq -r ' | ||
| if .overwrite_from_index == false then del(( .overwrite_from_index, .overwrite_from_index_token)) else . end | | ||
| if(.add_arches | length) == 0 then del(.add_arches) else . end | | ||
| if(.build_tags | length) == 0 then del(.build_tags) else . end' ${json_raw_input} > ${json_input} | ||
| echo "Calling IIB endpoint" > "$(results.buildState.path)" | ||
| # adds image to the index. | ||
| /usr/bin/curl -u : --negotiate -s -X POST -H "Content-Type: application/json" -d@${json_input} --insecure \ | ||
| "${IIB_SERVICE_URL}/builds/fbc-operations" |tee "$(results.jsonBuildInfo.path)" | ||
| # checks if the previous call returned an error. | ||
| ! jq -e -r ".error | select( . != null )" "$(results.jsonBuildInfo.path)" | ||
| volumeMounts: | ||
| - name: service-account-secret | ||
| mountPath: /mnt/service-account-secret | ||
| - name: update-fbc-catalog-wait-for-iib-build-step | ||
| image: >- | ||
| quay.io/konflux-ci/release-service-utils:e633d51cd41d73e4b3310face21bb980af7a662f | ||
| env: | ||
| - name: IIB_SERVICE_URL | ||
| valueFrom: | ||
| secretKeyRef: | ||
| name: iib-services-config | ||
| key: url | ||
| script: | | ||
| #!/usr/bin/env bash | ||
| # shellcheck disable=SC2317 # shellcheck calls all the commands in the function unreachable | ||
| set -x | ||
| watch_build_state() { | ||
| build_id="$(jq -r ".id" "$(results.jsonBuildInfo.path)")" | ||
| state="" | ||
| while true; do | ||
| # | ||
| # fetching build information. | ||
| build_info=$(curl -s "${IIB_SERVICE_URL}/builds/${build_id}") | ||
| # get state from the build information. | ||
| state="$(jq -r ".state" <<< "${build_info}")" | ||
| # remove the history as it breaks the results build up | ||
| jq -r 'del(.state_history)' <<< "${build_info}" | jq -c . > "$(results.jsonBuildInfo.path)" | ||
| url="$(jq -r ".logs.url" <<< "${build_info}")" | ||
| echo IIB log url is: "${url}" > "$(results.iibLog.path)" | ||
| case ${state} in | ||
| "complete") break ;; | ||
| "failed") break ;; | ||
| *) echo -en "."; sleep 30; continue ;; | ||
| esac | ||
| done | ||
| echo | ||
| jq -cr '{ "state": .state, "state_reason": .state_reason }' "$(results.jsonBuildInfo.path)" | jq -Rc \ | ||
| | tee "$(results.buildState.path)" | ||
| test "${state}" = "complete" && exit 0 || exit 1 | ||
| } | ||
| echo -en "waiting for build state to exit..." | ||
| # adding timeout here due to the Task timeout not accepting $(params.buildTimeoutSeconds) | ||
| # as parameter. | ||
| export -f watch_build_state | ||
| timeout "$(params.buildTimeoutSeconds)" bash -c watch_build_state | ||
| BUILDEXIT=$? | ||
| # it should continue only if the IIB build status is complete | ||
| if [ ${BUILDEXIT} -eq 0 ]; then | ||
| echo -n 0 > "$(results.exitCode.path)" | ||
| # get the manifest digests | ||
| indexImageCopy=$(jq -cr .internal_index_image_copy < "$(results.jsonBuildInfo.path)") | ||
| # Use this to obtain the manifest digests for each arch in manifest list | ||
| indexImageDigestsRaw=$(skopeo inspect --raw "docker://${indexImageCopy}") | ||
| # according the IIB team, | ||
| # "all index images will always be multi-arch with a manifest list" | ||
| # | ||
| indexImageDigests=$(echo "${indexImageDigestsRaw}" | \ | ||
| jq -r \ | ||
| '.manifests[]? | select(.mediaType=="application/vnd.docker.distribution.manifest.v2+json") | .digest') | ||
| echo -n "${indexImageDigests}" > "$(results.indexImageDigests.path)" | ||
| if [ -z "${indexImageDigests}" ] ; then | ||
| echo "Index image produced is not multi-arch with a manifest list" | ||
| echo -n 1 > "$(results.exitCode.path)" | ||
| fi | ||
| else | ||
| if [ ${BUILDEXIT} -eq 124 ]; then | ||
| echo "Timeout while waiting for the build to finish" | ||
| echo "Build timeout" > "$(results.buildState.path)" | ||
| fi | ||
| echo -n "" > "$(results.indexImageDigests.path)" | ||
| echo -n "$BUILDEXIT" > "$(results.exitCode.path)" | ||
| fi | ||
| # We don't put the log in a result because tekton results are too limited for what we can put | ||
| # to be useful, but still print it for debugging | ||
| curl -s "$(awk '{print $NF}' < "$(results.iibLog.path)")" | ||
| exit 0 | ||
| volumes: | ||
| - name: service-account-secret | ||
| secret: | ||
| secretName: $(params.iibServiceAccountSecret) | ||
| defaultMode: | ||
| 0400 |