Add e2e test for GCS

This adds an e2e test for GCS using storage emulator by passing env
STORAGE_EMULATOR_HOST. Reference:
https://cloud.google.com/go/docs/reference/cloud.google.com/go/storage/latest

Makefile

@@ -8,6 +8,8 @@ GOFUMPT=gofumpt
 e2e-coverage: ## run e2e tests with coverage
 	@go test -failfast -count=1 -tags=$(E2E_TAG) ./tests -coverpkg=./... -coverprofile /tmp/coverage.out
 	@go tool cover -func /tmp/coverage.out
+	tests/e2e.sh
+
 e2e: e2e-coverage
 
 e2e-docker: ## run e2e tests with a docker registry started

tests/e2e.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -x
+
+ROOT="$(git rev-parse --show-toplevel)"
+
+# Apply the GCS emulator configuration
+kubectl apply -f "${ROOT}/tests/gcs-emulator.yaml"
+
+# Wait for the deployment to be ready
+echo "Waiting for GCS emulator deployment to be ready..."
+kubectl wait --for=condition=available --timeout=300s deployment/gcs-emulator
+
+# Check the deployment status
+if [ $? -eq 0 ]; then
+    echo "GCS emulator deployment is ready"
+else
+    echo "Error: GCS emulator deployment failed to become ready within the timeout period"
+    exit 1
+fi
+
+
+ko apply -R -f ${ROOT}/dev/step-action/
+
+namespace=$(kubectl config view --minify --output 'jsonpath={..namespace}')
+sed -i "s/gcs-emulator.default.svc.cluster.local:9000/gcs-emulator.${namespace}.svc.cluster.local:9000/g" ${ROOT}/tests/test-pipeline-gcs.yaml
+
+openssl rand -base64 20 > /tmp/test
+kubectl create secret generic creds   --from-literal=GCP_APPLICATION_CREDENTIALS=/tmp/test
+
+ko create -f ${ROOT}/tests/test-pipeline-gcs.yaml
+
+# Check the pipelinerun status
+echo "Waiting for pipelinerun to complete..."
+kubectl wait --for=condition=succeeded --timeout=900s pipelinerun --all
+
+if [ $? -eq 0 ]; then
+    echo "Pipelinerun completed successfully"
+    exit 0
+else
+    echo "Error: Pipelinerun failed or timed out"
+    kubectl get taskrun
+    kubectl describe taskrun
+    kubectl get pipelinerun
+    kubectl describe pipelinerun
+    exit 1
+fi
+

tests/gcs-emulator.yaml

@@ -0,0 +1,48 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: gcs-emulator
+spec:
+  selector:
+    matchLabels:
+      run: gcs-emulator
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        run: gcs-emulator
+    spec:
+      containers:
+      - name: gcs-emulator
+        image: quay.io/khrm/gcse-emulator:latest
+        volumeMounts:
+          - mountPath: /data
+            name: data
+        ports:
+        - containerPort: 9000
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_BIND_SERVICE
+            drop:
+            - ALL
+          runAsNonRoot: true
+          seccompProfile:
+            type: RuntimeDefault
+      volumes:
+        - name: data
+          emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: gcs-emulator
+  labels:
+    run: gcs-emulator
+spec:
+  ports:
+  - port: 9000
+    protocol: TCP
+  selector:
+    run: gcs-emulator

tests/test-pipeline-gcs.yaml

@@ -0,0 +1,177 @@
+apiVersion: tekton.dev/v1
+kind: PipelineRun
+metadata:
+  generateName: pipelinerun-gcs-
+spec:
+  pipelineSpec:
+    params:
+      - name: repo_url
+        type: string
+      - name: revision
+        type: string
+      - name: registry
+        type: string
+      - name: buildCommand
+        type: string
+        default: go build -v .
+      - name: cachePatterns
+        type: array
+        default: [ "**go.mod", "**go.sum" ]
+      - name: image
+        type: string
+        default: golang:latest
+      - name: force-cache-upload
+        type: string
+        default: "false"
+    workspaces:
+      - name: source
+      - name: cred
+      - name: test
+    tasks:
+      - displayName: Build go application
+        name: build-task
+        workspaces:
+          - name: source
+            workspace: source
+          - name: test
+            workspace: test
+        taskSpec:
+          workspaces:
+            - name: source
+            - name: cred
+            - name: test
+          params:
+            - name: buildCommand
+              default: $(params.buildCommand)
+            - name: cachePatterns
+              default: $(params.cachePatterns)
+            - name: image
+              default: $(params.image)
+          stepTemplate:
+            env:
+            - name: "STORAGE_EMULATOR_HOST"
+              value: "gcs-emulator.default.svc.cluster.local:9000"
+          steps:
+            - name: create-repo
+              image: $(params.image)
+              script: |
+                mkdir -p $(workspaces.source.path)/repo
+                chmod 777 $(workspaces.source.path)/repo
+            - name: fetch-repo
+              ref:
+                resolver: http
+                params:
+                  - name: url
+                    value: https://raw.githubusercontent.com/tektoncd/catalog/main/stepaction/git-clone/0.1/git-clone.yaml
+              params:
+                - name: output-path
+                  value: $(workspaces.source.path)/repo
+                - name: url
+                  value: $(params.repo_url)
+                - name: revision
+                  value: $(params.revision)
+            - name: cache-fetch
+              ref:
+                name: cache-fetch
+              params:
+                - name: patterns
+                  value: $(params.cachePatterns)
+                - name: source
+                  value: $(params.registry)/cache-go:{{hash}}
+                - name: cachePath
+                  value: $(workspaces.source.path)/cache
+                - name: workingdir
+                  value: $(workspaces.source.path)/repo
+                - name: awsCredentialFile
+                  value: $(workspaces.cred.path)/credentials
+                - name: awsConfigFile
+                  value: $(workspaces.cred.path)/config
+                - name: googleCredentialsPath
+                  value: $(workspaces.cred.path)/creds.json
+
+            - name: run-go-build
+              workingDir: $(workspaces.source.path)/repo
+              image: $(params.image)
+              env:
+                - name: GOCACHE
+                  value: $(workspaces.source.path)/cache/gocache
+                - name: GOMODCACHE
+                  value: $(workspaces.source.path)/cache/gomodcache
+              script: |
+                set -x
+                git config --global --add safe.directory $(workspaces.source.path)/repo
+                $(params.buildCommand)
+                echo "Cache size is $(du -sh $(workspaces.source.path)/cache)"
+            - name: cache-upload
+              ref:
+                name: cache-upload
+              params:
+                - name: patterns
+                  value: $(params.cachePatterns)
+                - name: target
+                  value: $(params.registry)/cache-go:{{hash}}
+                - name: cachePath
+                  value: $(workspaces.source.path)/cache
+                - name: workingdir
+                  value: $(workspaces.source.path)/repo
+                - name: awsCredentialFile
+                  value: $(workspaces.cred.path)/credentials
+                - name: awsConfigFile
+                  value: $(workspaces.cred.path)/config
+                - name: force-cache-upload
+                  value: $(params.force-cache-upload)
+                - name: googleCredentialsPath
+                  value: $(workspaces.cred.path)/creds.json
+            - name: cache-fetch-test
+              ref:
+                name: cache-fetch
+              params:
+                - name: patterns
+                  value: $(params.cachePatterns)
+                - name: source
+                  value: $(params.registry)/cache-go:{{hash}}
+                - name: cachePath
+                  value: $(workspaces.test.path)/cache
+                - name: workingdir
+                  value: $(workspaces.source.path)/repo
+                - name: awsCredentialFile
+                  value: $(workspaces.cred.path)/credentials
+                - name: awsConfigFile
+                  value: $(workspaces.cred.path)/config
+                - name: googleCredentialsPath
+                  value: $(workspaces.cred.path)/creds.json
+            - name: validate-cache
+              workingDir: $(workspaces.test.path)/cache
+              image: $(params.image)
+              env:
+                - name: GOCACHE
+                  value: $(workspaces.source.path)/cache/gocache
+                - name: GOMODCACHE
+                  value: $(workspaces.source.path)/cache/gomodcache
+              script: |
+                set -x
+                echo "Cache size is $(du -sh $(workspaces.test.path)/cache)"
+                if [ $(du -shb $(workspaces.test.path)/cache | cut -f1) -lt 1000 ] ;then
+                      echo "cache fetch failed"
+                      exit 1
+                fi
+                exit 0
+  params:
+    - name: repo_url
+      value: https://github.com/chmouel/go-helloworld
+    - name: revision
+      value: main
+    - name: registry
+      value: gs://tekton-cache
+    - name: buildCommand
+      value: go build -v ./
+    - name: image
+      value: golang:1.21
+  workspaces:
+    - name: cred
+      secret:
+        secretName: creds
+    - name: source
+      emptyDir: {}
+    - name: test
+      emptyDir: {}