Add GCS and S3 using GO Cloud Dev

dev/pipeline/pipeline.yaml

@@ -74,8 +74,16 @@ spec:
                 value: $(workspaces.source.path)/cache
               - name: workingdir
                 value: $(workspaces.source.path)/repo
+              - name: awsCredentialFile
+                value: $(workspaces.cred.path)/credentials
+              - name: awsConfigFile
+                value: $(workspaces.cred.path)/config
               - name: cred-store
                 value: $(workspaces.cred.path)
+              - name: cred-store
+                value: $(workspaces.cred.path)
+              - name: googleCredentialsPath
+                value: $(workspaces.cred.path)/creds.json
 
           - name: run-go-build
             workingDir: $(workspaces.source.path)/repo
@@ -104,5 +112,11 @@ spec:
                 value: $(workspaces.source.path)/repo
               - name: cred-store
                 value: $(workspaces.cred.path)
+              - name: awsCredentialFile
+                value: $(workspaces.cred.path)/credentials
+              - name: awsConfigFile
+                value: $(workspaces.cred.path)/config
               - name: force-cache-upload
                 value: $(params.force-cache-upload)
+              - name: googleCredentialsPath
+                value: $(workspaces.cred.path)/creds.json

dev/pr/pipelinerun-gcs.yaml

@@ -0,0 +1,25 @@
+---
+apiVersion: tekton.dev/v1
+kind: PipelineRun
+metadata:
+  generateName: pipelinerun-s3-
+spec:
+  pipelineRef:
+    name: pipeline
+  params:
+    - name: repo_url
+      value: https://github.com/chmouel/go-helloworld
+    - name: revision
+      value: main
+    - name: registry
+      value: gs://tekton-cache
+    - name: buildCommand
+      value: go build -v ./
+    - name: image
+      value: golang:1.21
+  workspaces:
+    - name: cred
+      secret:
+        secretName: aws-cred
+    - name: source
+      emptyDir: {}

dev/step-action/cache-fetch.yaml

@@ -38,11 +38,18 @@ spec:
         The path where to find the google credentials. If left empty, it is ignored.
       type: string
       default: ""
-    - name: cred-store
+    - name: awsConfigFile
       description: |
-        The path where to find the creds to download cache files . If left empty, it is ignored.
+        The path to the aws config file. If left empty, it is ignored.
       type: string
       default: ""
+    - name: awsCredentialFile
+      description: |
+        The path to find the aws credentials file. If left empty, it is ignored.
+      type: string
+      default: ""
+    - name: BLOB_QUERY_PARAMS
+      value: $(params.blobQueryParams)
   results: # Any result to "publish" ?
     - name: fetched
       description: |
@@ -60,7 +67,12 @@ spec:
       value: $(params.googleCredentialsPath)
     - name: CRED_STORE
       value: $(params.cred-store)
-
+    - name: AWS_CONFIG_FILE
+      value: $(params.awsConfigFile)
+    - name: AWS_SHARED_CREDENTIALS_FILE
+      value: $(params.awsCredentialFile)
+    - name: BLOB_QUERY_PARAMS
+      value: $(params.blobQueryParams)
   # FIXME: use a released version once something is released :)
   image: ko://github.com/openshift-pipelines/tekton-caches/cmd/cache
   args: ["$(params.patterns[*])"]

dev/step-action/cache-upload.yaml

@@ -48,9 +48,19 @@ spec:
         The path where to find the google credentials. If left empty, it is ignored.
       type: string
       default: ""
-    - name: cred-store
+    - name: awsConfigFile
       description: |
-        The path where to find the creds to upload cache files . If left empty, it is ignored.
+        The path to the aws config file. If left empty, it is ignored.
+      type: string
+      default: ""
+    - name: awsCredentialFile
+      description: |
+        The path to find the aws credentials file. If left empty, it is ignored.
+      type: string
+      default: ""
+    - name: blobQueryParams
+      description: |
+        Blob Query Params to support configure s3, gcs and azure.
       type: string
       default: ""
   env:
@@ -68,8 +78,12 @@ spec:
       value: $(params.force-cache-upload)
     - name: GOOGLE_APPLICATION_CREDENTIALS
       value: $(params.googleCredentialsPath)
-    - name: CRED_STORE
-      value: $(params.cred-store)
+    - name: AWS_CONFIG_FILE
+      value: $(params.awsConfigFile)
+    - name: AWS_SHARED_CREDENTIALS_FILE
+      value: $(params.awsCredentialFile)
+    - name: BLOB_QUERY_PARAMS
+      value: $(params.blobQueryParams)
   # FIXME: use a released version once something is released :)
   image: ko://github.com/openshift-pipelines/tekton-caches/cmd/cache
   args: ["$(params.patterns[*])"]

go.mod

@@ -3,17 +3,12 @@ module github.com/openshift-pipelines/tekton-caches
 go 1.22
 
 require (
-	cloud.google.com/go/storage v1.43.0
-	github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher v0.0.0-20240628152042-7b3987f0b238
-	github.com/aws/aws-sdk-go-v2 v1.32.0
-	github.com/aws/aws-sdk-go-v2/config v1.27.39
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.64.1
 	github.com/codeclysm/extract/v3 v3.1.1
 	github.com/google/go-containerregistry v0.20.2
 	github.com/moby/patternmatcher v0.6.0
 	github.com/spf13/cobra v1.8.1
 	github.com/stretchr/testify v1.9.0
-	google.golang.org/api v0.199.0
+	gocloud.dev v0.39.0
 	gotest.tools/v3 v3.5.1
 )
 
@@ -23,9 +18,14 @@ require (
 	cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
 	cloud.google.com/go/compute/metadata v0.5.2 // indirect
 	cloud.google.com/go/iam v1.2.0 // indirect
+	cloud.google.com/go/storage v1.43.0 // indirect
+	github.com/aws/aws-sdk-go v1.55.5 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.32.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.39 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.37 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.10 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
@@ -34,6 +34,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.64.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.23.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.31.3 // indirect
@@ -50,10 +51,12 @@ require (
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/s2a-go v0.1.8 // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/google/wire v0.6.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
 	github.com/googleapis/gax-go/v2 v2.13.0 // indirect
 	github.com/h2non/filetype v1.1.3 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 // indirect
 	github.com/juju/loggo v1.0.0 // indirect
 	github.com/klauspost/compress v1.16.5 // indirect
@@ -80,6 +83,8 @@ require (
 	golang.org/x/sys v0.25.0 // indirect
 	golang.org/x/text v0.18.0 // indirect
 	golang.org/x/time v0.6.0 // indirect
+	golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect
+	google.golang.org/api v0.199.0 // indirect
 	google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect