Merge pull request #1208 from konflux-ci/s3-credentials

Add S3 bucket authentication to oci-copy
konflux-ci · Jul 29, 2024 · 4657318 · 4657318
2 parents d2618a8 + e3b8dd4
commit 4657318
Show file tree

Hide file tree

Showing 4 changed files with 114 additions and 21 deletions.
diff --git a/task/oci-copy-oci-ta/0.1/README.md b/task/oci-copy-oci-ta/0.1/README.md
@@ -5,6 +5,7 @@ Given a file in the user's source directory, copy content from arbitrary urls in
 ## Parameters
 |name|description|default value|required|
 |---|---|---|---|
+|AWS_SECRET_NAME|Name of a secret which will be made available to the build to construct Authorization headers for requests to Amazon S3 using v2 auth https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html. If specified, this will take precedence over BEARER_TOKEN_SECRET_NAME. The secret must contain two keys: `aws_access_key_id` and `aws_secret_access_key`. In the future, this will be reimplemented to use v4 auth: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html.|does-not-exist|false|
 |BEARER_TOKEN_SECRET_NAME|Name of a secret which will be made available to the build as an Authorization header. Note, the token will be sent to all servers found in the oci-copy.yaml file. If you do not wish to send the token to all servers, different taskruns and therefore different oci artifacts must be used.|does-not-exist|false|
 |IMAGE|Reference of the image we will push||true|
 |OCI_COPY_FILE|Path to the oci copy file.|./oci-copy.yaml|false|

diff --git a/task/oci-copy-oci-ta/0.1/oci-copy-oci-ta.yaml b/task/oci-copy-oci-ta/0.1/oci-copy-oci-ta.yaml
@@ -13,6 +13,15 @@ spec:
   description: Given a file in the user's source directory, copy content from
     arbitrary urls into the OCI registry.
   params:
+    - name: AWS_SECRET_NAME
+      description: 'Name of a secret which will be made available to the build
+        to construct Authorization headers for requests to Amazon S3 using
+        v2 auth https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html.
+        If specified, this will take precedence over BEARER_TOKEN_SECRET_NAME.
+        The secret must contain two keys: `aws_access_key_id` and `aws_secret_access_key`.
+        In the future, this will be reimplemented to use v4 auth: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html.'
+      type: string
+      default: does-not-exist
     - name: BEARER_TOKEN_SECRET_NAME
       description: Name of a secret which will be made available to the build
         as an Authorization header. Note, the token will be sent to all servers
@@ -69,6 +78,7 @@ spec:
       image: quay.io/konflux-ci/yq:latest@sha256:974dea6375ee9df561ffd3baf994db2b61777a71f3bcf0050c5dca91ac9b3430
       workingDir: /var/workdir
       script: |
+        #!/bin/bash
         set -eu
         set -o pipefail
 
@@ -104,17 +114,49 @@ spec:
               key: token
               name: $(params.BEARER_TOKEN_SECRET_NAME)
               optional: true
+        - name: AWS_ACCESS_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              key: aws_access_key_id
+              name: $(params.AWS_SECRET_NAME)
+              optional: true
+        - name: AWS_SECRET_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              key: aws_secret_access_key
+              name: $(params.AWS_SECRET_NAME)
+              optional: true
       script: |
+        #!/bin/bash
         set -e
         set -o pipefail
 
-        CURL_ARGS=()
-        if [ -n "${BEARER_TOKEN}" ]; then
-          echo "Found bearer token. Using it for authentication."
-          CURL_ARGS+=(-H "Authorization: Bearer ${BEARER_TOKEN}")
-        else
-          echo "Proceeding with anonymous requests"
-        fi
+        download() {
+          url="$1"
+          file="$2"
+          method="GET"
+
+          curl_args=(--fail --silent --show-error)
+          if [ -n "${AWS_ACCESS_KEY_ID}" ] && [ -n "${AWS_SECRET_ACCESS_KEY}" ]; then
+            echo "Found both aws credentials secret with both aws_access_key_id and aws_secret_access_key. Assuming S3 bucket"
+            # This implements v2 auth https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html.
+            # TODO - port to v4 auth https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html
+            path=$(echo "$url" | cut -d/ -f4-)
+            echo "Bucket path is $path"
+            date="$(date -u '+%a, %e %b %Y %H:%M:%S +0000')"
+            printf -v string_to_sign "%s\n\n\n%s\n%s" "$method" "$date" "/$path"
+            echo "String to sign is $string_to_sign"
+            signature=$(echo -n "$string_to_sign" | openssl dgst -sha1 -binary -hmac "${AWS_SECRET_ACCESS_KEY}" | openssl base64)
+            authorization="AWS ${AWS_ACCESS_KEY_ID}:${signature}"
+            curl "${curl_args[@]}" -H "Date: ${date}" -H "Authorization: ${authorization}" --location "$url" -o "$file"
+          elif [ -n "${BEARER_TOKEN}" ]; then
+            echo "Found bearer token. Using it for authentication."
+            curl "${curl_args[@]}" -H "Authorization: Bearer ${BEARER_TOKEN}" --location "$url" -o "$file"
+          else
+            echo "Proceeding with anonymous requests"
+            curl "${curl_args[@]}" --location "$url" -o "$file"
+          fi
+        }
 
         set -u
 
@@ -154,6 +196,7 @@ spec:
         for varfile in /var/workdir/vars/*; do
           echo
           echo "Reading $varfile"
+          # shellcheck source=/dev/null
           source $varfile
 
           echo "Checking to see if blob $OCI_ARTIFACT_DIGEST exists"
@@ -162,7 +205,7 @@ spec:
           else
             echo "Blob for ${OCI_FILENAME} does not yet exist in the registry at ${REPO}@sha256:${OCI_ARTIFACT_DIGEST}."
             echo "Downloading $OCI_SOURCE to $OCI_FILENAME"
-            curl "${CURL_ARGS[@]}" --fail --silent --show-error --location $OCI_SOURCE -o $OCI_FILENAME
+            download "$OCI_SOURCE" "$OCI_FILENAME"
 
             echo "Confirming that digest of $OCI_FILENAME matches expected $OCI_ARTIFACT_DIGEST"
             echo "$OCI_ARTIFACT_DIGEST $OCI_FILENAME" | sha256sum --check
@@ -210,6 +253,7 @@ spec:
       image: quay.io/konflux-ci/yq:latest@sha256:974dea6375ee9df561ffd3baf994db2b61777a71f3bcf0050c5dca91ac9b3430
       workingDir: /var/workdir
       script: |
+        #!/bin/bash
         cat >sbom-cyclonedx.json <<EOL
         {
             "\$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json",
@@ -222,6 +266,7 @@ spec:
 
         for varfile in /var/workdir/vars/*; do
           echo "Reading $varfile"
+          # shellcheck source=/dev/null
           source $varfile
 
           ENCODED_URL=$(echo "${OCI_SOURCE}" | python3 -c 'import sys; import urllib.parse; print(urllib.parse.quote(sys.stdin.read().strip(), safe=":/"))')
@@ -246,6 +291,7 @@ spec:
       image: quay.io/konflux-ci/yq:latest@sha256:974dea6375ee9df561ffd3baf994db2b61777a71f3bcf0050c5dca91ac9b3430
       workingDir: /var/workdir
       script: |
+        #!/bin/bash
         REPO=${IMAGE%:*}
         echo "Found that ${REPO} is the repository for ${IMAGE}"
         SBOM_DIGEST=$(sha256sum sbom-cyclonedx.json | awk '{ print $1 }')

diff --git a/task/oci-copy/0.1/README.md b/task/oci-copy/0.1/README.md
@@ -11,16 +11,16 @@ Note: the bearer token secret, if specified, will be sent to **all servers liste
 ## Parameters
 |name|description|default value|required|
 |---|---|---|---|
-|IMAGE|Reference of the image buildah will produce.||true|
+|IMAGE|Reference of the image we will push||true|
 |OCI_COPY_FILE|Path to the oci copy file.|./oci-copy.yaml|false|
-|BEARER_TOKEN_SECRET_NAME|Name of a secret which will be made available to the build as an Authorization header. Note, the token will be sent to all servers found in the oci-copy.yaml file. If you do not wish to send the token to all servers, different taskruns and therefore different oci artifacts must be used.|"does-not-exist"|false|
-
+|BEARER_TOKEN_SECRET_NAME|Name of a secret which will be made available to the build as an Authorization header. Note, the token will be sent to all servers found in the oci-copy.yaml file. If you do not wish to send the token to all servers, different taskruns and therefore different oci artifacts must be used.|does-not-exist|false|
+|AWS_SECRET_NAME|Name of a secret which will be made available to the build to construct Authorization headers for requests to Amazon S3 using v2 auth https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html. If specified, this will take precedence over BEARER_TOKEN_SECRET_NAME. The secret must contain two keys: `aws_access_key_id` and `aws_secret_access_key`. In the future, this will be reimplemented to use v4 auth: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html.|does-not-exist|false|
 
 ## Results
 |name|description|
 |---|---|
-|IMAGE_DIGEST|Digest of the image just built|
-|IMAGE_URL|Image repository where the built image was pushed|
+|IMAGE_DIGEST|Digest of the artifact just pushed|
+|IMAGE_URL|Repository where the artifact was pushed|
 |SBOM_BLOB_URL|Link to the SBOM blob pushed to the registry.|
 |IMAGE_REF|Image reference of the built image|
 

diff --git a/task/oci-copy/0.1/oci-copy.yaml b/task/oci-copy/0.1/oci-copy.yaml
@@ -25,6 +25,15 @@ spec:
         different taskruns and therefore different oci artifacts must be used.
       type: string
       default: "does-not-exist"
+    - name: AWS_SECRET_NAME
+      description: >-
+        Name of a secret which will be made available to the build to construct Authorization headers for requests to
+        Amazon S3 using v2 auth https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html.
+        If specified, this will take precedence over BEARER_TOKEN_SECRET_NAME. The secret must contain two keys:
+        `aws_access_key_id` and `aws_secret_access_key`. In the future, this will be reimplemented to use v4 auth:
+        https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html.
+      type: string
+      default: "does-not-exist"
   results:
     - description: Digest of the artifact just pushed
       name: IMAGE_DIGEST
@@ -47,6 +56,7 @@ spec:
     - name: prepare
       image: quay.io/konflux-ci/yq:latest@sha256:974dea6375ee9df561ffd3baf994db2b61777a71f3bcf0050c5dca91ac9b3430
       script: |
+        #!/bin/bash
         set -eu
         set -o pipefail
 
@@ -89,17 +99,49 @@ spec:
             name: $(params.BEARER_TOKEN_SECRET_NAME)
             key: token
             optional: true
+      - name: AWS_ACCESS_KEY_ID
+        valueFrom:
+          secretKeyRef:
+            name: $(params.AWS_SECRET_NAME)
+            key: aws_access_key_id
+            optional: true
+      - name: AWS_SECRET_ACCESS_KEY
+        valueFrom:
+          secretKeyRef:
+            name: $(params.AWS_SECRET_NAME)
+            key: aws_secret_access_key
+            optional: true
       script: |
+        #!/bin/bash
         set -e
         set -o pipefail
 
-        CURL_ARGS=()
-        if [ -n "${BEARER_TOKEN}" ]; then
-          echo "Found bearer token. Using it for authentication."
-          CURL_ARGS+=(-H "Authorization: Bearer ${BEARER_TOKEN}")
-        else
-          echo "Proceeding with anonymous requests"
-        fi
+        download() {
+          url="$1"
+          file="$2"
+          method="GET"
+
+          curl_args=(--fail --silent --show-error)
+          if [ -n "${AWS_ACCESS_KEY_ID}" ] && [ -n "${AWS_SECRET_ACCESS_KEY}" ]; then
+            echo "Found both aws credentials secret with both aws_access_key_id and aws_secret_access_key. Assuming S3 bucket"
+            # This implements v2 auth https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html.
+            # TODO - port to v4 auth https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html
+            path=$(echo "$url" | cut -d/ -f4-)
+            echo "Bucket path is $path"
+            date="$(date -u '+%a, %e %b %Y %H:%M:%S +0000')"
+            printf -v string_to_sign "%s\n\n\n%s\n%s" "$method" "$date" "/$path"
+            echo "String to sign is $string_to_sign"
+            signature=$(echo -n "$string_to_sign" | openssl dgst -sha1 -binary -hmac "${AWS_SECRET_ACCESS_KEY}" | openssl base64)
+            authorization="AWS ${AWS_ACCESS_KEY_ID}:${signature}"
+            curl "${curl_args[@]}" -H "Date: ${date}" -H "Authorization: ${authorization}" --location "$url" -o "$file"
+          elif [ -n "${BEARER_TOKEN}" ]; then
+            echo "Found bearer token. Using it for authentication."
+            curl "${curl_args[@]}" -H "Authorization: Bearer ${BEARER_TOKEN}" --location "$url" -o "$file"
+          else
+            echo "Proceeding with anonymous requests"
+            curl "${curl_args[@]}" --location "$url" -o "$file"
+          fi
+        }
 
         set -u
 
@@ -139,6 +181,7 @@ spec:
         for varfile in /var/workdir/vars/*; do
           echo
           echo "Reading $varfile"
+          # shellcheck source=/dev/null
           source $varfile
 
           echo "Checking to see if blob $OCI_ARTIFACT_DIGEST exists"
@@ -147,7 +190,7 @@ spec:
           else
             echo "Blob for ${OCI_FILENAME} does not yet exist in the registry at ${REPO}@sha256:${OCI_ARTIFACT_DIGEST}."
             echo "Downloading $OCI_SOURCE to $OCI_FILENAME"
-            curl "${CURL_ARGS[@]}" --fail --silent --show-error --location $OCI_SOURCE -o $OCI_FILENAME
+            download "$OCI_SOURCE" "$OCI_FILENAME"
 
             echo "Confirming that digest of $OCI_FILENAME matches expected $OCI_ARTIFACT_DIGEST"
             echo "$OCI_ARTIFACT_DIGEST $OCI_FILENAME" | sha256sum --check
@@ -188,6 +231,7 @@ spec:
     - name: sbom-generate
       image: quay.io/konflux-ci/yq:latest@sha256:974dea6375ee9df561ffd3baf994db2b61777a71f3bcf0050c5dca91ac9b3430
       script: |
+        #!/bin/bash
         cat >sbom-cyclonedx.json <<EOL
         {
             "\$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json",
@@ -200,6 +244,7 @@ spec:
 
         for varfile in /var/workdir/vars/*; do
           echo "Reading $varfile"
+          # shellcheck source=/dev/null
           source $varfile
 
           ENCODED_URL=$(echo "${OCI_SOURCE}" | python3 -c 'import sys; import urllib.parse; print(urllib.parse.quote(sys.stdin.read().strip(), safe=":/"))')
@@ -224,6 +269,7 @@ spec:
     - name: report-sbom-url
       image: quay.io/konflux-ci/yq:latest@sha256:974dea6375ee9df561ffd3baf994db2b61777a71f3bcf0050c5dca91ac9b3430
       script: |
+        #!/bin/bash
         REPO=${IMAGE%:*}
         echo "Found that ${REPO} is the repository for ${IMAGE}"
         SBOM_DIGEST=$(sha256sum sbom-cyclonedx.json | awk '{ print $1 }')