Converted GCS and S3 to use VFS Library

GCS and S3 now uses vfs library
GCS also archive instead of doing per file operations.
This reduces class B operations for fetch and A operation
for upload.

dev/pipeline/pipeline.yaml

@@ -74,8 +74,16 @@ spec:
                 value: $(workspaces.source.path)/cache
               - name: workingdir
                 value: $(workspaces.source.path)/repo
+              - name: awsCredentialFile
+                value: $(workspaces.cred.path)/credentials
+              - name: awsConfigFile
+                value: $(workspaces.cred.path)/config
               - name: cred-store
                 value: $(workspaces.cred.path)
+              - name: cred-store
+                value: $(workspaces.cred.path)
+              - name: googleCredentialsPath
+                value: $(workspaces.cred.path)/creds.json
 
           - name: run-go-build
             workingDir: $(workspaces.source.path)/repo
@@ -104,5 +112,11 @@ spec:
                 value: $(workspaces.source.path)/repo
               - name: cred-store
                 value: $(workspaces.cred.path)
+              - name: awsCredentialFile
+                value: $(workspaces.cred.path)/credentials
+              - name: awsConfigFile
+                value: $(workspaces.cred.path)/config
               - name: force-cache-upload
                 value: $(params.force-cache-upload)
+              - name: googleCredentialsPath
+                value: $(workspaces.cred.path)/creds.json

dev/pr/pipelinerun-gcs.yaml

@@ -0,0 +1,25 @@
+---
+apiVersion: tekton.dev/v1
+kind: PipelineRun
+metadata:
+  generateName: pipelinerun-s3-
+spec:
+  pipelineRef:
+    name: pipeline
+  params:
+    - name: repo_url
+      value: https://github.com/chmouel/go-helloworld
+    - name: revision
+      value: main
+    - name: registry
+      value: gs://tekton-cache-assdfsdfdsvbdor0934o5
+    - name: buildCommand
+      value: go build -v ./
+    - name: image
+      value: golang:1.21
+  workspaces:
+    - name: cred
+      secret:
+        secretName: aws-cred
+    - name: source
+      emptyDir: {}

dev/step-action/cache-fetch.yaml

@@ -38,6 +38,16 @@ spec:
         The path where to find the google credentials. If left empty, it is ignored.
       type: string
       default: ""
+    - name: awsConfigFile
+      description: |
+        The path to the aws config file. If left empty, it is ignored.
+      type: string
+      default: ""
+    - name: awsCredentialFile
+      description: |
+        The path to find the aws credentials file. If left empty, it is ignored.
+      type: string
+      default: ""
     - name: cred-store
       description: |
         The path where to find the creds to download cache files . If left empty, it is ignored.
@@ -60,7 +70,10 @@ spec:
       value: $(params.googleCredentialsPath)
     - name: CRED_STORE
       value: $(params.cred-store)
-
+    - name: AWS_CONFIG_FILE
+      value: $(params.awsConfigFile)
+    - name: AWS_SHARED_CREDENTIALS_FILE
+      value: $(params.awsCredentialFile)
   # FIXME: use a released version once something is released :)
   image: ko://github.com/openshift-pipelines/tekton-caches/cmd/cache
   args: ["$(params.patterns[*])"]

dev/step-action/cache-upload.yaml

@@ -48,6 +48,16 @@ spec:
         The path where to find the google credentials. If left empty, it is ignored.
       type: string
       default: ""
+    - name: awsConfigFile
+      description: |
+        The path to the aws config file. If left empty, it is ignored.
+      type: string
+      default: ""
+    - name: awsCredentialFile
+      description: |
+        The path to find the aws credentials file. If left empty, it is ignored.
+      type: string
+      default: ""
     - name: cred-store
       description: |
         The path where to find the creds to upload cache files . If left empty, it is ignored.
@@ -68,8 +78,10 @@ spec:
       value: $(params.force-cache-upload)
     - name: GOOGLE_APPLICATION_CREDENTIALS
       value: $(params.googleCredentialsPath)
-    - name: CRED_STORE
-      value: $(params.cred-store)
+    - name: AWS_CONFIG_FILE
+      value: $(params.awsConfigFile)
+    - name: AWS_SHARED_CREDENTIALS_FILE
+      value: $(params.awsCredentialFile)
   # FIXME: use a released version once something is released :)
   image: ko://github.com/openshift-pipelines/tekton-caches/cmd/cache
   args: ["$(params.patterns[*])"]

internal/fetch/fetch.go

@@ -7,9 +7,8 @@ import (
 	"os"
 	"strings"
 
-	"github.com/openshift-pipelines/tekton-caches/internal/tar"
-
 	"github.com/openshift-pipelines/tekton-caches/internal/provider/s3"
+	"github.com/openshift-pipelines/tekton-caches/internal/provider/vfs"
 
 	"github.com/openshift-pipelines/tekton-caches/internal/provider/gcs"
 	"github.com/openshift-pipelines/tekton-caches/internal/provider/oci"
@@ -26,20 +25,24 @@ func Fetch(ctx context.Context, hash, target, folder string, insecure bool) erro
 	if err != nil {
 		return err
 	}
+	target = strings.ReplaceAll(target, "{{hash}}", hash)
 	source := strings.TrimPrefix(target, u.Scheme+"://")
-	source = strings.ReplaceAll(source, "{{hash}}", hash)
-	file, _ := os.CreateTemp("", "cache.tar")
 
 	switch u.Scheme {
 	case "oci":
 		return oci.Fetch(ctx, hash, source, folder, insecure)
 	case "s3":
-		if err := s3.Fetch(ctx, source, file.Name()); err != nil {
+		remoteFile, err := s3.File(ctx, target)
+		if err != nil {
 			return err
 		}
-		return tar.Untar(ctx, file, folder)
+		return vfs.Fetch(folder, remoteFile)
 	case "gs":
-		return gcs.Fetch(ctx, hash, source, folder)
+		remoteFile, err := gcs.File(ctx, target)
+		if err != nil {
+			return err
+		}
+		return vfs.Fetch(folder, remoteFile)
 	default:
 		return fmt.Errorf("unknown schema: %s", target)
 	}

internal/provider/gcs/vfs_file.go

@@ -0,0 +1,18 @@
+package gcs
+
+import (
+	"context"
+
+	"github.com/c2fo/vfs/v6"
+	"github.com/c2fo/vfs/v6/backend"
+	"github.com/c2fo/vfs/v6/backend/gs"
+	"github.com/c2fo/vfs/v6/vfssimple"
+)
+
+func File(ctx context.Context, file string) (vfs.File, error) {
+	bucketAuth := gs.NewFileSystem()
+	bucketAuth = bucketAuth.WithContext(ctx)
+	backend.Register("gs://", bucketAuth)
+
+	return vfssimple.NewFile(file)
+}