Merge branch 'main' into dependabot/github_actions/helm/kind-action-1…

….10.0

.github/workflows/latest.yaml

@@ -75,10 +75,7 @@ jobs:
     needs: [build]
 
     # https://docs.github.com/en/actions/reference/authentication-in-a-workflow
-    permissions:
-      id-token: write
-      packages: write
-      contents: read
+    permissions: write-all
 
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

README.md

@@ -1,15 +1,80 @@
 # tekton-caches [![build-test-publish](https://github.com/openshift-pipelines/tekton-caches/actions/workflows/latest.yaml/badge.svg)](https://github.com/openshift-pipelines/tekton-caches/actions/workflows/latest.yaml)
 
-Tools (and Task/StepAction) to managing within Tekton
-
-```bash
-# With OCI
-$ cache fetch --hasfiles '**/go.sum' --target oci://quay.io/vdemeest/cache/go-cache:{{hash}} --folder /workspaces/go-cache
-$ cache upload --hashfiles '**/go.sum' --target oci://quay.io/vdemeest/cache/go-cache:{{hash}} --folder /workspaces/go-cache
-# With s3
-$ cache fetch --hashfiles '**/go.sum' --target s3://my-bucket/path/to/my/cache --folder /workspaces/go-cache
-$ cache fetch --hashfiles '**/go.sum' --target s3://my-bucket/path/to/my/cache --folder /workspaces/go-cache
-# With gcs
-$ cache fetch --hashfiles '**/go.sum' --target gcs://my-bucket/path/to/my/cache --folder /workspaces/go-cache
-$ cache fetch --hashfiles '**/go.sum' --target gcs://my-bucket/path/to/my/cache --folder /workspaces/go-cache
+This is a tool to cache resources like go cache/maven or others on TektonCD
+pipelines.
+
+This tool supports uploading the cache to an OCI registry and plans to support
+S3, GCS and other storage backends.
+
+It uses the new [StepActions](https://tekton.dev/docs/pipelines/stepactions/)
+feature of TektonCD Pipelines but can be as well used without it.
+
+See the StepActions in the [tekton/](./tekton) directory.
+
+## Example
+
+This is an example of a build pipeline for a go application caching and reusing
+the go cache. If the `go.mod` and `go.sum` are changed the cache is invalidated and
+rebuilt.
+
+### Pre-requisites
+
+- You need a recent TektonCD pipelines installed with the StepActions feature-flags enabled.
+
+```shell
+kubectl patch configmap -n tekton-pipelines --type merge -p '{"data":{"enable-step-actions": "true"}}' feature-flags
+```
+
+- A registry to push the images to. Example: docker.io/loginname. Make sure you
+  have setup tekton to be able to push/fetch from that registry, see the
+  [TektonCD pipelines documentation](https://tekton.dev/docs/pipelines/auth/#configuring-authentication-for-docker)
+
+### Usage
+
+Create the go pipeline example from the examples directory:
+
+```shell
+kubectl create -f pipeline-go.yaml
+```
+
+Start it with the tkn cli (change the value as needed):
+
+```shell
+tkn pipeline start pipeline-go --param repo_url=https://github.com/vdemeester/go-helloworld-app --param revision=main --param registry=docker.io/username -w name=source,emptyDir=
+```
+
+or with a PipelineRun yaml object:
+
+```shell
+```yaml
+kind: PipelineRun
+metadata:
+  name: build-go-application-with-caching-run
+spec:
+  pipelineRef:
+    name: pipeline-go
+  params:
+    - name: repo_url
+      value: https://github.com/vdemeester/go-helloworld-app
+    - name: revision
+      value: main
+    - name: registry
+      value: docker.io/username
+  workspaces:
+    - name: source
+      emptyDir: {}
+```
+
+- you can as well redefine the `buildCommand` which by default do a `go build
+  -v ./` with the `buildCommand` parameter, for example if you want instead to
+  run the tests on a repo with caching:
+
+```shell
+tkn pipeline start pipeline-go --param repo_url=https://github.com/chmouel/gosmee \ 
+  --param revision=main --param registry=docker.io/username \
+  --param=buildCommand="make test" -w name=source,emptyDir=
 ```
+
+## License
+
+[Apache License 2.0](./LICENSE)

cmd/cache/fetch.go

@@ -18,6 +18,7 @@ const (
 	patternsFlag   = "pattern"
 	sourceFlag     = "source"
 	folderFlag     = "folder"
+	insecureFlag   = "insecure"
 )
 
 func fetchCmd() *cobra.Command {
@@ -59,17 +60,22 @@ func fetchCmd() *cobra.Command {
 				return err
 			}
 
+			insecure, err := cmd.Flags().GetBool(insecureFlag)
+			if err != nil {
+				return err
+			}
 			// FIXME: Wrap the error.
 			// If not, warn and do not fail
 			// fmt.Fprintf(os.Stderr, "Repository %s doesn't exists or isn't reachable, fetching no cache.\n", cacheImageRef)
-			return fetch.Fetch(cmd.Context(), hashStr, target, folder)
+			return fetch.Fetch(cmd.Context(), hashStr, target, folder, insecure)
 		},
 	}
 
 	cmd.Flags().StringArray(patternsFlag, []string{}, "Files pattern to compute the hash from")
 	cmd.Flags().String(sourceFlag, "", "Cache source reference")
 	cmd.Flags().String(folderFlag, "", "Folder where to extract the content of the cache if it exists")
 	cmd.Flags().String(workingdirFlag, ".", "Working dir from where the files patterns needs to be taken")
+	cmd.Flags().Bool(insecureFlag, false, "Wether to use insecure transport or not to upload to insecure registry")
 
 	return cmd
 }

cmd/cache/upload.go

@@ -34,6 +34,11 @@ func uploadCmd() *cobra.Command {
 			if err != nil {
 				return err
 			}
+
+			insecure, err := cmd.Flags().GetBool(insecureFlag)
+			if err != nil {
+				return err
+			}
 			matches := glob(workingdir, func(s string) bool {
 				m, err := patternmatcher.Matches(s, patterns)
 				if err != nil {
@@ -53,13 +58,15 @@ func uploadCmd() *cobra.Command {
 			if err != nil {
 				return err
 			}
-			return upload.Upload(cmd.Context(), hashStr, target, folder)
+			// TODO: use a struct to pas arguments
+			return upload.Upload(cmd.Context(), hashStr, target, folder, insecure)
 		},
 	}
 	cmd.Flags().StringArray(patternsFlag, []string{}, "Files pattern to compute the hash from")
 	cmd.Flags().String(targetFlag, "", "Cache target reference")
 	cmd.Flags().String(folderFlag, "", "Folder where to extract the content of the cache if it exists")
 	cmd.Flags().String(workingdirFlag, ".", "Working dir from where the files patterns needs to be taken")
+	cmd.Flags().Bool(insecureFlag, false, "Wether to use insecure transport or not to upload to insecure registry")
 
 	return cmd
 }

examples/pipeline-go.yaml

@@ -0,0 +1,97 @@
+---
+apiVersion: tekton.dev/v1
+kind: Pipeline
+metadata:
+  name: pipeline-go
+spec:
+  params:
+    - name: repo_url
+      type: string
+    - name: revision
+      type: string
+    - name: registry
+      type: string
+    - name: buildCommand
+      type: string
+      default: go build -v .
+  workspaces:
+    - name: source
+  tasks:
+    - displayName: Build go application
+      name: build-task
+      workspaces:
+        - name: source
+          workspace: source
+      taskSpec:
+        workspaces:
+          - name: source
+        params:
+          - name: buildCommand
+            default: $(params.buildCommand)
+        steps:
+          - name: create-repo
+            image: cgr.dev/chainguard/go
+            script: |
+              mkdir -p $(workspaces.source.path)/repo
+              chmod 777 $(workspaces.source.path)/repo
+          - name: fetch-repo
+            ref:
+              resolver: http
+              params:
+                - name: url
+                  value: https://raw.githubusercontent.com/tektoncd/catalog/main/stepaction/git-clone/0.1/git-clone.yaml
+            params:
+              - name: output-path
+                value: $(workspaces.source.path)/repo
+              - name: url
+                value: $(params.repo_url)
+              - name: revision
+                value: $(params.revision)
+          - name: fetch-cache
+            ref:
+              resolver: http
+              params:
+                - name: url
+                  value: https://raw.githubusercontent.com/openshift-pipelines/tekton-caches/main/tekton/cache-fetch.yaml
+            params:
+              - name: patterns
+                value:
+                  - "**.go"
+                  - "**go.sum"
+              - name: source
+                value: oci://$(params.registry)/cache-go:{{hash}}
+              - name: cachePath
+                value: $(workspaces.source.path)/cache
+              - name: workingdir
+                value: $(workspaces.source.path)/repo
+          - image: cgr.dev/chainguard/go
+            workingDir: $(workspaces.source.path)/repo
+            name: noop-task
+            env:
+              - name: GOCACHE
+                value: $(workspaces.source.path)/cache/gocache
+              - name: GOMODCACHE
+                value: $(workspaces.source.path)/cache/gomodcache
+            script: |
+              set -x
+              git config --global --add safe.directory $(workspaces.source.path)/repo
+              $(params.buildCommand)
+              du -shk $GOPATH
+              du -shk $GOMODCACHE
+          - name: cache-upload
+            ref:
+              resolver: http
+              params:
+                - name: url
+                  value: https://raw.githubusercontent.com/openshift-pipelines/tekton-caches/main/tekton/cache-upload.yaml
+            params:
+              - name: patterns
+                value:
+                  - "**.go"
+                  - "**go.sum"
+              - name: target
+                value: oci://$(params.registry)/cache-go:{{hash}}
+              - name: cachePath
+                value: $(workspaces.source.path)/cache
+              - name: workingdir
+                value: $(workspaces.source.path)/repo

examples/pipelinerun.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: tekton.dev/v1
+kind: PipelineRun
+metadata:
+  name: build-go-application-with-caching-run
+spec:
+  pipelineRef:
+    name: build-go-application-with-caching
+  params:
+    - name: repo_url
+      value: https://github.com/vdemeester/go-helloworld-app
+    - name: revision
+      value: main
+    - name: registry
+      value: registry.civuole.local/cache
+  workspaces:
+    - name: source
+      emptyDir: {}

internal/fetch/fetch.go

@@ -4,20 +4,27 @@ import (
 	"context"
 	"fmt"
 	"net/url"
+	"os"
 	"strings"
 
 	"github.com/openshift-pipelines/tekton-caches/internal/provider/oci"
 )
 
-func Fetch(ctx context.Context, hash, target, folder string) error {
+func Fetch(ctx context.Context, hash, target, folder string, insecure bool) error {
+	// check that folder exists or automatically create it
+	if _, err := os.Stat(folder); os.IsNotExist(err) {
+		if err := os.MkdirAll(folder, 0755); err != nil {
+			return fmt.Errorf("failed to create folder: %w", err)
+		}
+	}
 	u, err := url.Parse(target)
 	if err != nil {
 		return err
 	}
 	newTarget := strings.TrimPrefix(target, u.Scheme+"://")
 	switch u.Scheme {
 	case "oci":
-		return oci.Fetch(ctx, hash, newTarget, folder)
+		return oci.Fetch(ctx, hash, newTarget, folder, insecure)
 	case "s3":
 		return fmt.Errorf("s3 schema not (yet) supported: %s", target)
 	case "gcs":

internal/provider/oci/fetch.go

@@ -11,12 +11,16 @@ import (
 	"github.com/google/go-containerregistry/pkg/crane"
 )
 
-func Fetch(ctx context.Context, hash, target, folder string) error {
+func Fetch(ctx context.Context, hash, target, folder string, insecure bool) error {
 	cacheImageRef := strings.ReplaceAll(target, "{{hash}}", hash)
 	fmt.Fprintf(os.Stderr, "Trying to fetch oci image %s in %s\n", cacheImageRef, folder)
 
+	options := []crane.Option{}
+	if insecure {
+		options = append(options, crane.Insecure)
+	}
 	// Try to fetch it (if it exists)
-	image, err := crane.Pull(cacheImageRef)
+	image, err := crane.Pull(cacheImageRef, options...)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Warning: %s\n", err)
 		return err

internal/provider/oci/upload.go

@@ -15,7 +15,7 @@ import (
 	"github.com/google/go-containerregistry/pkg/v1/types"
 )
 
-func Upload(ctx context.Context, hash, target, folder string) error {
+func Upload(ctx context.Context, hash, target, folder string, insecure bool) error {
 	cacheImageRef := strings.ReplaceAll(target, "{{hash}}", hash)
 	fmt.Fprintf(os.Stderr, "Upload %s content to oci image %s\n", folder, cacheImageRef)
 
@@ -42,7 +42,11 @@ func Upload(ctx context.Context, hash, target, folder string) error {
 		return nil
 	}
 
-	if err := crane.Push(image, cacheImageRef); err != nil {
+	options := []crane.Option{}
+	if insecure {
+		options = append(options, crane.Insecure)
+	}
+	if err := crane.Push(image, cacheImageRef, options...); err != nil {
 		return err
 	}
 

internal/upload/upload.go

@@ -9,15 +9,15 @@ import (
 	"github.com/openshift-pipelines/tekton-caches/internal/provider/oci"
 )
 
-func Upload(ctx context.Context, hash, target, folder string) error {
+func Upload(ctx context.Context, hash, target, folder string, insecure bool) error {
 	u, err := url.Parse(target)
 	if err != nil {
 		return err
 	}
 	newTarget := strings.TrimPrefix(target, u.Scheme+"://")
 	switch u.Scheme {
 	case "oci":
-		return oci.Upload(ctx, hash, newTarget, folder)
+		return oci.Upload(ctx, hash, newTarget, folder, insecure)
 	case "s3":
 		return fmt.Errorf("s3 schema not (yet) supported: %s", target)
 	case "gcs":