diff --git a/.tekton/test.yaml b/.tekton/test.yaml index 213f569..9e9e870 100644 --- a/.tekton/test.yaml +++ b/.tekton/test.yaml @@ -15,15 +15,19 @@ spec: params: - name: SNAPSHOT steps: - - image: registry.redhat.io/openshift4/ose-cli:latest + - name: test + image: registry.redhat.io/openshift4/ose-cli:latest env: - name: SNAPSHOT value: $(params.SNAPSHOT) script: | + #!/bin/bash echo -e "Grabbing a copy of yq" oc image extract --confirm quay.io/konflux-ci/yq:latest --path=/usr/bin/yq:/usr/bin/. && chmod +x /usr/bin/yq echo -e "Testing Snapshot:\n ${SNAPSHOT}" + TESTS_FAILED="false" + failure_num=0 IMAGE=$(echo ${SNAPSHOT} | yq -r '.components[].containerImage') echo -e "Found image ${IMAGE}" @@ -33,28 +37,233 @@ spec: oc image extract --confirm ${IMAGE} --path=/usr/bin/yq:/usr/bin/. && chmod +x /usr/bin/yq oc image extract --confirm ${IMAGE} --path=/usr/local/bin/retry:/usr/local/bin/. && chmod +x /usr/local/bin/retry oc image extract --confirm ${IMAGE} --path=/usr/local/bin/select-oci-auth:/usr/local/bin/. && chmod +x /usr/local/bin/select-oci-auth + oc image extract --confirm ${IMAGE} --path=/usr/local/bin/attach-helper:/usr/local/bin/. && chmod +x /usr/local/bin/attach-helper + oc image extract --confirm ${IMAGE} --path=/usr/local/bin/oras-options:/usr/local/bin/. && chmod +x /usr/local/bin/oras-options + oc image extract --confirm ${IMAGE} --path=/usr/local/bin/get-reference-base:/usr/local/bin/. && chmod +x /usr/local/bin/get-reference-base REPO=$(echo ${IMAGE} | awk -F '@' '{ print $1 }') TAG="$(echo ${IMAGE} | awk -F '@' '{print $2 }' | sed s/:/-/).test" + ## Test isolating the OCI object registry and repository + echo -n "quay.io/test/foo" > base_reference + echo -n "quay.io:443/test/foo" > base_reference_port + get-reference-base quay.io:443/test/foo:bar > test_base1 + get-reference-base quay.io:443/test/foo@sha256:aaaa > test_base2 + get-reference-base quay.io:443/test/foo:bar@sha256:aaaa > test_base3 + get-reference-base quay.io/test/foo:bar@sha256:aaaa > test_base4 + get-reference-base quay.io/test/foo:bar > test_base5 + get-reference-base quay.io/test/foo@sha256:aaaa > test_base6 + + if [[ $(cmp -s base_reference_port test_base1) -ne 0 ]]; then + echo "ERROR: Incorrect reference isolation with registry port and tag" + TESTS_FAILED="true" + failure_num=$((failure_num + 1)) + fi + if [[ $(cmp -s base_reference_port test_base2) -ne 0 ]]; then + echo "ERROR: Incorrect reference isolation with registry port and digest" + TESTS_FAILED="true" + failure_num=$((failure_num + 1)) + fi + if [[ $(cmp -s base_reference_port test_base3) -ne 0 ]]; then + echo "ERROR: Incorrect reference isolation with registry port, tag, and digest" + TESTS_FAILED="true" + failure_num=$((failure_num + 1)) + fi + if [[ $(cmp -s base_reference test_base4) -ne 0 ]]; then + echo "ERROR: Incorrect reference isolation with tag and digest" + TESTS_FAILED="true" + failure_num=$((failure_num + 1)) + fi + if [[ $(cmp -s base_reference test_base5) -ne 0 ]]; then + echo "ERROR: Incorrect reference isolation with tag" + TESTS_FAILED="true" + failure_num=$((failure_num + 1)) + fi + if [[ $(cmp -s base_reference test_base6) -ne 0 ]]; then + echo "ERROR: Incorrect reference isolation with digest" + TESTS_FAILED="true" + failure_num=$((failure_num + 1)) + fi + + ## Test isolating registry auth and pushing echo "Extracting relevant OCI auth for $REPO" select-oci-auth $REPO > auth.json + # Test pushing directly with oras echo "Pushing foo.txt to $REPO:$TAG" echo -n "hello world" > foo.txt oras push --no-tty --registry-config auth.json $REPO:$TAG foo.txt:text/plain - rm foo.txt + mv foo.txt check.txt + # Test pulling directly with oras, ensuring that the file content is unchanged echo "Pulling foo.txt to $REPO:$TAG" oras pull --no-tty --registry-config auth.json $REPO:$TAG OUTPUT=$(cat foo.txt) - echo "Expecting hello world" - echo "Received ${OUTPUT}" + diff foo.txt check.txt > diff.txt + if [ $? -eq 0 ]; then + echo "Recieved the expected output" + else + TESTS_FAILED="true" + failure_num=$((failure_num + 1)) + echo "ERROR: Expecting hello world" + echo "Received ${OUTPUT}" + fi + + ## Test attaching simple files + attach-helper --subject $REPO:$TAG --digestfile foo-digest.txt foo.txt + attach-helper --subject $REPO:$TAG --artifact-type "application/vnd.konflux-ci.test-artifact" --media-type-name "foobar" check.txt - if [ "$OUTPUT" == "hello world" ]; then - exit 0 + ## Ensure that the files are unmodified and that the digest is set properly + diff foo.txt check.txt > diff.txt + if [ ! $? -eq 0 ]; then + TESTS_FAILED="true" + failure_num=$((failure_num + 1)) + echo "ERROR: Files were modified when attaching." + fi + + ## Check to make sure that all attachments have happened properly. Looking at both the total number + ## and the number for each artifact type (one custom, one default) + mkdir discoveries + oras discover -v --format tree $REPO:$TAG | tee discoveries/all_attached + oras discover -v --format tree --artifact-type "application/vnd.konflux-ci.attached-artifact" $REPO:$TAG > discoveries/default_attached + oras discover -v --format tree --artifact-type "application/vnd.konflux-ci.test-artifact" $REPO:$TAG > discoveries/custom_attached + + if [[ "$(cat discoveries/all_attached | wc -l)" == "7" ]]; then + echo "Two artifacts attached" else + echo "ERROR: All attached artifacts not found" + TESTS_FAILED="true" + failure_num=$((failure_num + 1)) + fi + if [[ "$(cat discoveries/default_attached | wc -l)" == "4" ]]; then + echo "One artifact attached with type application/vnd.konflux-ci.attached-artifact" + else + echo "ERROR: Artifact attachment application/vnd.konflux-ci.attached-artifact not found" + TESTS_FAILED="true" + failure_num=$((failure_num + 1)) + fi + if [[ "$(cat discoveries/custom_attached | wc -l)" == "4" ]]; then + echo "One artifact attached with type application/vnd.konflux-ci.test-artifact" + else + echo "ERROR: Artifact attachment application/vnd.konflux-ci.test-artifact not found" + TESTS_FAILED="true" + failure_num=$((failure_num + 1)) + fi + + ## Check to make sure that we have found each of the media types used. One is custom, another is auto. + oras manifest fetch --pretty $REPO:$TAG + referenced_artifacts=$( oras discover --format json $REPO:$TAG | yq -e '.manifests[].reference') + found_type1="false" + found_type2="false" + echo "Looking at mediaType for all referenced artifacts" + for artifact in ${referenced_artifacts[@]}; do + oras manifest fetch --pretty $artifact + mediaType=$(oras manifest fetch --pretty $artifact | yq -e '.layers[].mediaType') + if [[ "$mediaType" == "application/vnd.konflux-ci.attached-artifact.foo+txt" ]]; then + found_type1="true" + fi + if [[ "$mediaType" == "application/vnd.konflux-ci.test-artifact.foobar" ]]; then + found_type2="true" + fi + done + if [[ "$found_type1" == "true" ]]; then + echo "Found one application/vnd.konflux-ci.attached-artifact.foo+txt mediaType" + else + echo "ERROR: Didn't find application/vnd.konflux-ci.attached-artifact.foo+txt mediaType" + TESTS_FAILED="true" + failure_num=$((failure_num + 1)) + fi + if [[ "$found_type2" == "true" ]]; then + echo "Found one application/vnd.konflux-ci.test-artifact.foobar mediaType" + else + echo "ERROR: Didn't find application/vnd.konflux-ci.test-artifact.foobar mediaType" + TESTS_FAILED="true" + failure_num=$((failure_num + 1)) + fi + + ## Test to make sure that digest matches + digest_pullspec=$(oras discover --format json --artifact-type "application/vnd.konflux-ci.attached-artifact" $REPO:$TAG | yq -e '.manifests[].reference') + digestfile_content=$(cat foo-digest.txt) + if [ "${digest_pullspec}" == "${REPO}@sha256:${digestfile_content}" ]; then + echo "Digestfile properly created" + else + echo "ERROR: Reported digest ${digestfile_content} doesn't match ${digest_pullspec}" + cat foo-digest.txt + TESTS_FAILED="true" + failure_num=$((failure_num + 1)) + fi + + # Test attaching directories + attach-helper --subject $REPO:$TAG --artifact-type "application/vnd.konflux-ci.test-directory" --digestfile discoveries-digest.txt discoveries + mv discoveries discoveries-reference + + ## Ensure that the the artifact (custom) and media (auto) types are as expected for directories + directory_digest=$(cat discoveries-digest.txt) + oras discover --format json --artifact-type "application/vnd.konflux-ci.test-directory" $REPO:$TAG | yq -e '.manifests[].reference' > referenced_directory_artifacts + if [ ! "$(cat referenced_directory_artifacts | wc -l)" == "1" ]; then + echo "ERROR: Improper number of referenced artifacts for type application/vnd.konflux-ci.test-directory" + cat referenced_directory_artifacts + TESTS_FAILED="true" + failure_num=$((failure_num + 1)) + fi + artifactType=$(oras manifest fetch --pretty $(cat referenced_directory_artifacts | head -n 1) | yq -e '.artifactType') + mediaType=$(oras manifest fetch --pretty $(cat referenced_directory_artifacts | head -n 1) | yq -e '.layers[].mediaType') + if [[ "$artifactType" == "application/vnd.konflux-ci.test-directory" ]]; then + echo "Directory artifactType matches" + else + echo "ERROR: Directory artifact type was ${artifactType}/nexpected: application/vnd.konflux-ci.test-directory" + TESTS_FAILED="true" + failure_num=$((failure_num + 1)) + fi + if [[ "$mediaType" == "application/vnd.konflux-ci.test-directory.discoveries" ]]; then + echo "Directory mediaType matches" + else + echo "ERROR: Directory media type was ${mediaType}/nexpected: application/vnd.konflux-ci.test-directory.discoveries" + TESTS_FAILED="true" + failure_num=$((failure_num + 1)) + fi + + # Ensure that the manifest digest matches for a directory + directory_shasum=$(oras manifest fetch $(cat referenced_directory_artifacts | head -n 1) | sha256sum | tr -d "[:space:]-") + if [ "${directory_shasum}" == "${directory_digest}" ]; then + echo "Directory blob digests match" + else + echo "ERROR: Directory blob digest does not match returned value" + TESTS_FAILED="true" + failure_num=$((failure_num + 1)) + fi + + ## Ensure that directory content matches + oras pull ${REPO}@sha256:${directory_digest} + diff discoveries discoveries-reference > dir_diff.txt + if [ $? -eq 0 ]; then + echo "Fetched directory matches" + else + TESTS_FAILED="true" + failure_num=$((failure_num + 1)) + echo "ERROR: Fetched directory does not match" + cat dir_diff.txt + fi + + ## No need to test this right now. If it doesn't work, the script will error out. If it does, we will support it! + # ## Test attaching multiple files + # echo "one" > one.txt + # echo "two" > two.txt + # attach-helper --subject $REPO:$TAG --artifact-type "application/vnd.konflux-ci.multiple-artifacts" one.txt two.txt 2>/dev/null + # if [ "$?" == "2" ]; then + # echo "Attaching multiple artifacts correctly failed." + # else + # echo "ERROR: We shouldn't be able to attach multiple artifacts" + # TESTS_FAILED="true" + # failure_num=$((failure_num + 1)) + # fi + + if [ "$TESTS_FAILED" == "true" ]; then + echo "$failure_num tests failed." exit 1 + else + echo "All tests passed, congrats!" + exit 0 fi diff --git a/Containerfile b/Containerfile index 752fa17..bb32f52 100644 --- a/Containerfile +++ b/Containerfile @@ -11,27 +11,32 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM brew.registry.redhat.io/rh-osbs/openshift-golang-builder:rhel_9_1.22 as builder +ARG ORASPKG=/oras + +FROM registry.access.redhat.com/ubi9/go-toolset:1.22.5 as builder ARG TARGETPLATFORM +ARG ORASPKG #RUN dnf -y install git make && dnf -y clean all -ENV ORASPKG /oras -ADD . ${ORASPKG} -WORKDIR ${ORASPKG}/oras +ADD --chown=default oras ${ORASPKG} +WORKDIR ${ORASPKG} RUN go mod vendor RUN make "build-$(echo $TARGETPLATFORM | sed s/\\/v8// | tr / -)" -RUN mv ${ORASPKG}/oras/bin/$(echo $TARGETPLATFORM | sed s/\\/v8//)/oras /usr/bin/oras -RUN mkdir /licenses && mv LICENSE /licenses/LICENSE +RUN mv ${ORASPKG}/bin/$(echo $TARGETPLATFORM | sed s/\\/v8//)/oras ${ORASPKG}/bin/oras FROM quay.io/konflux-ci/yq:latest@sha256:15a4bff3229069034b1fc7d6d3a7c9b06edf8c1c5f6f27d49bf4b31de823168a as yq FROM registry.access.redhat.com/ubi9:latest@sha256:1057dab827c782abcfb9bda0c3900c0966b5066e671d54976a7bcb3a2d1a5e53 +ARG ORASPKG RUN mkdir /licenses RUN useradd -r --uid=65532 --create-home --shell /bin/bash oras COPY --from=yq /usr/bin/yq /usr/bin/yq -COPY --from=builder /usr/bin/oras /usr/bin/oras -COPY --from=builder /licenses/LICENSE /licenses/LICENSE +COPY --from=builder ${ORASPKG}/bin/oras /usr/bin/oras +COPY --from=builder ${ORASPKG}/LICENSE /licenses/LICENSE +COPY hack/attach.sh /usr/local/bin/attach-helper +COPY hack/get-reference-base.sh /usr/local/bin/get-reference-base +COPY hack/oras-options.sh /usr/local/bin/oras-options COPY hack/retry.sh /usr/local/bin/retry COPY hack/select-oci-auth.sh /usr/local/bin/select-oci-auth diff --git a/hack/attach.sh b/hack/attach.sh new file mode 100755 index 0000000..35979e0 --- /dev/null +++ b/hack/attach.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# Helper function to use oras attach +# +# This script can be used to easily attach artifacts. It will reduce the authentication scope +# so that oras will work when there are repository-specific tokens. It also provides a template +# for specifying the artifactType when attaching. +# +# The --subject parameter is the subject to attach the artifact to, e.g. +# registry.local/org/repo. +# +# The --artifact-type (optional) parameter can be used to define an artifactType for the attached artifact. +# If absent, "application/vnd.konflux-ci.attached-artifact" will be used. +# +# The --media-type-name (optional) parameter can be used to define a mediaType for the attached artifact. +# The type will be appended to the artifact type as determined from the "--artifact-type" parameter. If +# absent, the filename+extension will be used. +# +# The --distribution-spec (optional) parameter can be used to use a specific distribution spec. Oras supports +# the values `v1.1-referrers-api` and `v1.1-referrers-tag`. If absent the system default will be used. +# +# The --digestfile (optional) parameter can be used to provide a file to store the digest for the pushed +# image manifest. This will NOT be the digest of the attached artifact blob itself. +# +# Positional parameters are artifacts that need to be attached. These are either relative or absolute path strings. +# Only one artifact can be attached per invocation. +# +# NOTE: if a directory is passed, timestamps are not modified in the gzipped directory. +# +# Example: +# attach.sh --subject quay.io:443/arewm/foo:bar local.file + +set -o errexit +set -o nounset +set -o pipefail + +# contains pairs of artifacts to attach and (optionally) paths to output the blob digest +artifacts=() +artifact_type="application/vnd.konflux-ci.attached-artifact" +# distribution_spec="v1.1-referrers-api" +distribution_spec="" +media_type_name="" +digest_file="/dev/null" + +while [[ $# -gt 0 ]]; do + case $1 in + --subject) + subject="$2" + shift + shift + ;; + --artifact-type) + artifact_type="$2" + shift + shift + ;; + --media-type-name) + media_type_name="$2" + shift + shift + ;; + --distribution-spec) + distribution_spec="$2" + shift + shift + ;; + --digestfile) + digest_file="$2" + shift + shift + ;; + -*) + >&2 echo "Unknown option $1" + exit 1 + ;; + *) + artifacts+=("$1") + shift + ;; + esac +done + +if [[ -z "${subject:-}" ]]; then + >&2 echo "ERROR: --subject cannot be empty when attaching OCI artifacts" + exit 1 +fi + +if [ ${#artifacts[@]} != 1 ]; then + >&2 echo "ERROR: Only one artifact can be attached: found ${#artifacts[@]}" + exit 2 +fi + +# read in any oras options +source oras-options + +artifact="${artifacts[0]}" +# change to the artifact directory so we don't have to use absolute paths +pushd "$(dirname ${artifact})" > /dev/null +media_type="${artifact_type}" +file_name="$(basename ${artifact})" +if [ -n "${media_type_name}" ]; then + media_type="${artifact_type}.${media_type_name}" +else + file_base="${file_name%.*}" + file_extension="${file_name##*.}" + type_descriptor="${file_base}" + if [[ "${file_base}" != "${file_extension}" ]]; then + type_descriptor="${file_base}+${file_extension}" + fi + media_type="${artifact_type}.${type_descriptor}" +fi +echo "attaching artifact:" +echo "${file_name}:${media_type}" +use_distribution_spec=() +if [ -n "${distribution_spec}" ]; then + use_distribution_spec+=(--distribution-spec ${distribution_spec}) +fi +oras attach "${oras_opts[@]}" --no-tty --registry-config <(select-oci-auth ${subject}) --artifact-type "${artifact_type}" \ + "${use_distribution_spec[@]}" "${subject}" "${file_name}:${media_type}" | tail -n 1 | cut -d: -f3 > "${digest_file}" +popd > /dev/null + +echo 'Artifacts attached' diff --git a/hack/get-reference-base.sh b/hack/get-reference-base.sh new file mode 100755 index 0000000..2fe1c3e --- /dev/null +++ b/hack/get-reference-base.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Outputs the registry and repository for an OCI object reference +# +# An OCI object reference can contain a registry port, tag, and digest in addition to the repository itself. +# Some scripts might handle the definition of an object reference differently and ignore various parts of the +# [specification](https://github.com/opencontainers/distribution-spec/blob/main/spec.md) +# +# Usage: +# get-reference-base.sh +# +# Example: +# get-reference-base.sh quay.io:443/arewm/foo:bar +# +set -o errexit +set -o nounset +set -o pipefail + +original_ref="$1" + +# Trim off digest +repo="$(echo -n $original_ref | cut -d@ -f1)" +if [[ $(echo -n "$repo" | tr -cd ":" | wc -c | tr -d '[:space:]') == 2 ]]; then + # format is now registry:port/repository:tag + # trim off everything after the last colon + repo=${repo%:*} +elif [[ $(echo -n "$repo" | tr -cd ":" | wc -c | tr -d '[:space:]') == 1 ]]; then + # we have either a port or a tag so inspect the content after + # the colon to determine if it is a valid tag. + # https://github.com/opencontainers/distribution-spec/blob/main/spec.md + # [a-zA-Z0-9_][a-zA-Z0-9._-]{0,127} is the regex for a valid tag + # If not a valid tag, leave the colon alone. + if [[ "$(echo -n "$repo" | cut -d: -f2 | tr -d '[:space:]')" =~ ^([a-zA-Z0-9_][a-zA-Z0-9._-]{0,127})$ ]]; then + # We match a tag so trim it off + repo=$(echo -n "$repo" | cut -d: -f1) + fi +fi + +echo -n "$repo" diff --git a/hack/oras-options.sh b/hack/oras-options.sh new file mode 100755 index 0000000..c643d4a --- /dev/null +++ b/hack/oras-options.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +oras_opts=(${ORAS_OPTIONS:-}) + +if [[ -v CA_FILE ]]; then + oras_opts+=(--ca-file=${CA_FILE}) +fi + +if [[ -v DEBUG ]]; then + oras_opts+=(--debug) +fi diff --git a/hack/select-oci-auth.sh b/hack/select-oci-auth.sh index e75dcc1..2a5476a 100755 --- a/hack/select-oci-auth.sh +++ b/hack/select-oci-auth.sh @@ -22,11 +22,8 @@ set -o pipefail original_ref="$1" -# Remove digest from image reference -ref="${original_ref/@*}" - -# Remove tag from image reference while making sure optional registry port is taken into account -ref="$(echo -n $ref | sed 's_/\(.*\):\(.*\)_/\1_g')" +# Get the OCI object reference without a tag and digest +ref="$(get-reference-base ${original_ref})" registry="${ref/\/*}"