diff --git a/tasks/rh-sign-image-cosign/rh-sign-image-cosign.yaml b/tasks/rh-sign-image-cosign/rh-sign-image-cosign.yaml index 816635e45..3a56b6b08 100644 --- a/tasks/rh-sign-image-cosign/rh-sign-image-cosign.yaml +++ b/tasks/rh-sign-image-cosign/rh-sign-image-cosign.yaml @@ -28,6 +28,10 @@ spec: description: Retry cosign N times. type: string default: "3" + - name: concurrentLimit + type: string + default: 5 + description: The maximum number of concurrent cosign signing jobs workspaces: - name: data description: Workspace to read and save files @@ -61,6 +65,11 @@ spec: name: $(params.secretName) key: REKOR_URL optional: true + - name: PUBLIC_KEY + valueFrom: + secretKeyRef: + name: $(params.secretName) + key: PUBLIC_KEY script: | #!/usr/bin/env bash set -eux @@ -73,21 +82,23 @@ spec: echo "No valid file was provided as signRegistryAccessPath." exit 1 fi + PUBLIC_KEY_FILE=$(mktemp) + echo -n "$PUBLIC_KEY" > "$PUBLIC_KEY_FILE" + RUNNING_JOBS="\j" # Bash parameter for number of jobs currently running run_cosign () { # Expected arguments are [digest_reference, tag_reference] - # Upload transparency log when rekor url is specified - if [ -v REKOR_URL ]; then - COSIGN_COMMON_ARGS="-y --rekor-url=$REKOR_URL --key $SIGN_KEY" - else - COSIGN_COMMON_ARGS="--tlog-upload=false --key $SIGN_KEY" - fi - echo "Signing manifest $1 ($2)" attempt=0 + backoff1=2 + backoff2=3 until [ "$attempt" -gt "$(params.retries)" ] ; do # 3 retries by default - cosign -t 3m0s sign\ - ${COSIGN_COMMON_ARGS}\ - --sign-container-identity "$2"\ - "$1" && break + cosign "$@" && break + sleep $backoff2 + + # Fibbonaci backoff + old_backoff1=$backoff1 + backoff1=$backoff2 + backoff2=$((old_backoff1 + backoff2)) + attempt=$((attempt+1)) done if [ "$attempt" -gt "$(params.retries)" ] ; then @@ -95,6 +106,45 @@ spec: exit 1 fi } + function check_existing_signatures() { + local identity=$1 + local reference=$2 + local digest=$3 + if [ -v REKOR_URL ]; then + COSIGN_REKOR_ARGS="--rekor-url=$REKOR_URL" + else + COSIGN_REKOR_ARGS="--insecure-ignore-tlog=true" + fi + verify_output=$(run_cosign verify "$COSIGN_REKOR_ARGS" "$reference") + found_signatures=$(echo "$verify_output" | jq -j '[.[]|select(.critical.image."docker-manifest-digest"' + '| contains("'"$digest"'"))| ' + 'select(.critical.identity."docker-reference"| contains("'"$identity"'"))]|length') + echo "$found_signatures" + } + function check_and_sign() { + local identity=$1 + local reference=$2 + local tag=$3 + local digest=$4 + found_signatures=$(check_existing_signatures "$identity" "$reference:$tag" "$digest") + if [ -z "$found_signatures" ]; then + found_signatures=0 + fi + echo "FOUND SIGNATURES: $found_signatures" + + if [ -v REKOR_URL ]; then + COSIGN_REKOR_ARGS="-y --rekor-url=$REKOR_URL" + else + COSIGN_REKOR_ARGS="--tlog-upload=false" + fi + + if [ "$found_signatures" -eq 0 ]; then + run_cosign -t 3m0s sign "$COSIGN_REKOR_ARGS" \ + --key "$SIGN_KEY" \ + --sign-container-identity "$identity" "$reference@$digest" + fi + } + for (( COMPONENTS_INDEX=0; COMPONENTS_INDEX= $(params.concurrentLimit) )); do + wait -n + done + check_and_sign "${REGISTRY_REF}:${TAG}" "${INTERNAL_CONTAINER_REF}" "${TAG}" "${MDIGEST}" & done done done @@ -138,8 +191,14 @@ spec: # Sign manifest list itself or manifest if it's not list for REGISTRY_REF in "${REGISTRY_REFERENCES[@]}"; do for TAG in $TAGS; do - run_cosign "${INTERNAL_CONTAINER_REF}@${DIGEST}" "${REGISTRY_REF}:${TAG}" + while (( ${RUNNING_JOBS@P} >= $(params.concurrentLimit) )); do + wait -n + done + check_and_sign "${REGISTRY_REF}:${TAG}" "${INTERNAL_CONTAINER_REF}" "${TAG}" "${DIGEST}" & done done done + while (( ${RUNNING_JOBS@P} > 0 )); do + wait -n + done echo "done" diff --git a/tasks/rh-sign-image-cosign/tests/mocks.sh b/tasks/rh-sign-image-cosign/tests/mocks.sh index f191b339e..3139a2390 100644 --- a/tasks/rh-sign-image-cosign/tests/mocks.sh +++ b/tasks/rh-sign-image-cosign/tests/mocks.sh @@ -137,14 +137,25 @@ function skopeo() { fi } function cosign () { - echo "$@" >> $(workspaces.data.path)/mock_cosign_calls - echo "running cosign: $@" - # mock cosign failing the first 3 calls for the retry test - if [[ "$@" == *":retry-tag"* ]] - then - if [[ $(cat $(workspaces.data.path)/mock_cosign_calls | wc -l) -le 3 ]] - then - echo "expected cosign call failure for retry test" + # check if call should end successfully + successfull_run=$(sed -n '1p' $(workspaces.data.path)/mock_cosign_success_calls && \ + sed -i '1d' $(workspaces.data.path)/mock_cosign_success_calls) + + if [ "$1" = "verify" ]; then + mock_existing_sig_file=$(echo "${*: -1}" | tr "/" "-") + echo "$@" >> $(workspaces.data.path)/mock_cosign_verify_calls + # if the call shouldn't end successfully, exit with error + if [ "$successfull_run" != "1" ]; then + return 1 + fi + cat "$(workspaces.data.path)/$mock_existing_sig_file" + else + echo "running cosign: $@" + echo "$@" >> "$(workspaces.data.path)/mock_cosign_sign_calls" + # if the call shouldn't end successfully, exit with error + if [ "$successfull_run" != "1" ]; then + >&2 echo "- SIMULATED ERROR -" + echo "- SIMULATED ERROR -" >> "$(workspaces.data.path)/mock_cosign_sign_calls" return 1 fi fi diff --git a/tasks/rh-sign-image-cosign/tests/pre-apply-task-hook.sh b/tasks/rh-sign-image-cosign/tests/pre-apply-task-hook.sh index e8768f00a..b2cbf684e 100755 --- a/tasks/rh-sign-image-cosign/tests/pre-apply-task-hook.sh +++ b/tasks/rh-sign-image-cosign/tests/pre-apply-task-hook.sh @@ -7,14 +7,16 @@ kubectl create secret generic test-cosign-secret\ --from-literal=AWS_DEFAULT_REGION=us-test-1\ --from-literal=AWS_ACCESS_KEY_ID=test-access-key\ --from-literal=AWS_SECRET_ACCESS_KEY=test-secret-access-key\ - --from-literal=SIGN_KEY=aws://arn:mykey + --from-literal=SIGN_KEY=aws://arn:mykey\ + --from-literal=PUBLIC_KEY=public_key kubectl create secret generic test-cosign-secret-rekor\ --from-literal=AWS_DEFAULT_REGION=us-test-1\ --from-literal=AWS_ACCESS_KEY_ID=test-access-key\ --from-literal=AWS_SECRET_ACCESS_KEY=test-secret-access-key\ --from-literal=SIGN_KEY=aws://arn:mykey\ - --from-literal=REKOR_URL=https://fake-rekor-server + --from-literal=REKOR_URL=https://fake-rekor-server\ + --from-literal=PUBLIC_KEY=public_key # Add mocks to the beginning of task step script TASK_PATH="$1" diff --git a/tasks/rh-sign-image-cosign/tests/test-rh-sign-image-cosign-multiple-components.yaml b/tasks/rh-sign-image-cosign/tests/test-rh-sign-image-cosign-multiple-components.yaml index 2c8a01186..2b9d7729b 100644 --- a/tasks/rh-sign-image-cosign/tests/test-rh-sign-image-cosign-multiple-components.yaml +++ b/tasks/rh-sign-image-cosign/tests/test-rh-sign-image-cosign-multiple-components.yaml @@ -48,6 +48,21 @@ spec: ] } EOF + REF11="quay.io/redhat-pending/test-product----test-image0:t1" + REF12="quay.io/redhat-pending/test-product----test-image0:t2" + REF21="quay.io/redhat-pending/test-product----test-image1:t1" + REF22="quay.io/redhat-pending/test-product----test-image1:t2" + + # create empty cosign verify mock files + touch "$(workspaces.data.path)/$(echo $REF11 | tr '/' '-')" + touch "$(workspaces.data.path)/$(echo $REF12 | tr '/' '-')" + touch "$(workspaces.data.path)/$(echo $REF21 | tr '/' '-')" + touch "$(workspaces.data.path)/$(echo $REF22 | tr '/' '-')" + + # setup cosign success calls - all calls should pass + for _ in $(seq 1 48); do + echo "1" >> "$(workspaces.data.path)/mock_cosign_success_calls" + done cat > "$(workspaces.data.path)/signRegistryAccess.txt" << EOF test-product/test-image0 @@ -62,6 +77,8 @@ spec: value: 'test-cosign-secret-rekor' - name: signRegistryAccessPath value: signRegistryAccess.txt + - name: concurrentLimit + value: 1 workspaces: - name: data workspace: tests-workspace @@ -93,7 +110,7 @@ spec: CALLS=$(cat "$(workspaces.data.path)/mock_skopeo_calls") test "$CALLS" = "$EXPECTED" - CALLS=$(cat "$(workspaces.data.path)/mock_cosign_calls") + CALLS=$(cat "$(workspaces.data.path)/mock_cosign_sign_calls") COSIGN_COMMON="-t 3m0s sign -y --rekor-url=https://fake-rekor-server --key aws://arn:mykey \ --sign-container-identity" EXPECTED=$(cat < "$CALLS_FILE" + echo "$EXPECTED" > "$EXPECTED_FILE" + diff -Naur "$EXPECTED_FILE" "$CALLS_FILE" + rm "$CALLS_FILE" "$EXPECTED_FILE" + exit 1 + fi runAfter: - run-task diff --git a/tasks/rh-sign-image-cosign/tests/test-rh-sign-image-cosign-retries.yaml b/tasks/rh-sign-image-cosign/tests/test-rh-sign-image-cosign-retries.yaml index c7d897862..fd5f4b0e4 100644 --- a/tasks/rh-sign-image-cosign/tests/test-rh-sign-image-cosign-retries.yaml +++ b/tasks/rh-sign-image-cosign/tests/test-rh-sign-image-cosign-retries.yaml @@ -31,12 +31,28 @@ spec: "repository": "quay.io/redhat-pending/test-product----test-image2", "rh-registry-repo": "registry.stage.redhat.io/test-product/test-image2", "registry-access-repo": "registry.access.stage.redhat.com/test-product/test-image2", - "tags": ["retry-tag"] + "tags": ["t1", "t2"] } ] } EOF + # create empty cosign verify mock files + REF1="quay.io/redhat-pending/test-product----test-image2:t1" + touch "$(workspaces.data.path)/$(echo $REF1 | tr '/' '-')" + REF2="quay.io/redhat-pending/test-product----test-image2:t2" + touch "$(workspaces.data.path)/$(echo $REF2 | tr '/' '-')" + + # first 5 cosign calls should end with success + for _ in $(seq 1 3); do + echo "1" >> "$(workspaces.data.path)/mock_cosign_success_calls" + done + # simulate cosign failure on 6th call + echo "0" >> "$(workspaces.data.path)/mock_cosign_success_calls" + + # after retrying, it should pass + echo "1" >> "$(workspaces.data.path)/mock_cosign_success_calls" + cat > "$(workspaces.data.path)/signRegistryAccess.txt" << EOF test-product/test-image0 EOF @@ -52,6 +68,8 @@ spec: value: signRegistryAccess.txt - name: retries value: 3 + - name: concurrentLimit + value: 1 workspaces: - name: data workspace: tests-workspace @@ -72,15 +90,25 @@ spec: _TEST_PUB_REPO="registry.stage.redhat.io/test-product/test-image2" _TEST_REPO="quay.io/redhat-pending/test-product----test-image2" - CALLS=$(cat "$(workspaces.data.path)/mock_cosign_calls") + CALLS=$(cat "$(workspaces.data.path)/mock_cosign_sign_calls") COSIGN_COMMON="-t 3m0s sign --tlog-upload=false --key aws://arn:mykey --sign-container-identity" EXPECTED=$(cat < "$CALLS_FILE" + echo "$EXPECTED" > "$EXPECTED_FILE" + diff -Naur "$EXPECTED_FILE" "$CALLS_FILE" + rm "$CALLS_FILE" "$EXPECTED_FILE" + exit 1 + fi runAfter: - run-task diff --git a/tasks/rh-sign-image-cosign/tests/test-rh-sign-image-cosign-single-component.yaml b/tasks/rh-sign-image-cosign/tests/test-rh-sign-image-cosign-single-component.yaml index 37d3b6cde..ae38d1d1f 100644 --- a/tasks/rh-sign-image-cosign/tests/test-rh-sign-image-cosign-single-component.yaml +++ b/tasks/rh-sign-image-cosign/tests/test-rh-sign-image-cosign-single-component.yaml @@ -36,6 +36,16 @@ spec: ] } EOF + # create empty cosign verify mock files + REF1="quay.io/redhat-pending/test-product----test-image2:t1" + touch "$(workspaces.data.path)/$(echo $REF1 | tr '/' '-')" + REF2="quay.io/redhat-pending/test-product----test-image2:t2" + touch "$(workspaces.data.path)/$(echo $REF2 | tr '/' '-')" + + # first 5 cosign calls should end with success + for _ in $(seq 1 8); do + echo "1" >> "$(workspaces.data.path)/mock_cosign_success_calls" + done cat > "$(workspaces.data.path)/signRegistryAccess.txt" << EOF test-product/test-image2 @@ -50,6 +60,8 @@ spec: value: 'test-cosign-secret' - name: signRegistryAccessPath value: signRegistryAccess.txt + - name: concurrentLimit + value: 1 workspaces: - name: data workspace: tests-workspace @@ -75,7 +87,7 @@ spec: EXPECTED="inspect --raw docker://quay.io/redhat-user-workloads/test-product/test-image2@sha256:2222" test "$CALLS" = "$EXPECTED" - CALLS=$(cat "$(workspaces.data.path)/mock_cosign_calls") + CALLS=$(cat "$(workspaces.data.path)/mock_cosign_sign_calls") COSIGN_COMMON="-t 3m0s sign --tlog-upload=false --key aws://arn:mykey --sign-container-identity" EXPECTED=$(cat < "$CALLS_FILE" + echo "$EXPECTED" > "$EXPECTED_FILE" + diff -Naur "$EXPECTED_FILE" "$CALLS_FILE" + rm "$CALLS_FILE" "$EXPECTED_FILE" + exit 1 + fi runAfter: - run-task