Create some StepAction to fetch and upload caches…

And experiment with a small go project, GOCACHE and GOMODCACHE.

Signed-off-by: Vincent Demeester <[email protected]>

.github/workflows/latest.yaml

@@ -20,32 +20,45 @@ jobs:
     - run: go env
     - name: go build
       run: go build -v ./...
-    - name: go test
+    - name: go unit test
       run: go test -v ./...
 
-  # e2e:
-  #   name: e2e tests
-  #   runs-on: ubuntu-latest
-  #   needs: [ build ]
-  # 
-  #   steps:
-  #   - uses: ko-build/[email protected]
-  #   - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-  #   - uses: chainguard-dev/actions/setup-kind@main
-  #     with:
-  #       k8s-version: v1.23.x
-  #   - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
-  #     with:
-  #       go-version: "1.21"
-  #   - uses: vdemeester/setup-tektoncd@main
-  #     with:
-  #       pipeline: v0.40.x
-  #       pipeline-feature-flags: '{"enable-api-fields": "alpha"}'
-  #   - name: install manual-approval-gate custom task
-  #     run: |
-  #       # FIXME: remove this once >= 0.41.x
-  #       kubectl apply -f https://storage.googleapis.com/tekton-releases/pipeline/previous/v0.40.2/resolvers.yaml
-  #       ko apply --local -f config/
+  e2e:
+    name: e2e tests
+    runs-on: ubuntu-latest
+    needs: [ build ]
+
+    steps:
+    - uses: ko-build/[email protected]
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 
+    - uses: azure/setup-helm@v3
+      # instantiate a KinD (Kubernetes in Docker) cluster, installs `kubectl` and configures the
+      # `kubeconfig` to reach the local cluster
+    - uses: helm/[email protected]
+      with:
+        cluster_name: kind
+        wait: 120s
+    - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+      with:
+        go-version: "1.21.x"
+    - uses: openshift-pipelines/setup-tektoncd@v1
+      with:
+        pipeline_version: v0.56.1
+        feature_flags: '{"enable-step-actions": "true"}'
+    - name: tests
+      run: |
+        kubectl -n registry get deployment
+        kubectl -n registry port-forward deployment/registry 5000:5000 &
+        REGISTRY=registry.registry.svc.cluster.local:32222
+        KO_DOCKER_REPO=127.0.0.1:5000/cache ko publish --base-import-paths --tags=latest ./cmd/cache
+        sed "s/image:.*/image: ${REGISTRY}\/cache:latest/g" tekton/cache-fetch.yaml | tee | kubectl apply -f -
+        sed "s/image:.*/image: ${REGISTRY}\/cache:latest/g" tekton/cache-upload.yaml | tee | kubectl apply -f -
+        kubectl apply -f tests/
+        tkn task start cache-fetch-go -p gitURL=https://github.com/vdemeester/go-helloworld-app -p gitRevision=main -p cachePatterns="**.go,**go.sum" -p cacheURIBase=oci://${REGISTRY}/cache/go -w name=source,emptyDir= -w name=gocache,emptyDir= -w name=gomodcache,emptyDir= --showlog
+        tkn task start cache-upload-go -p gitURL=https://github.com/vdemeester/go-helloworld-app -p gitRevision=main -p cachePatterns="**.go,**go.sum" -p cacheURIBase=oci://${REGISTRY}/cache/go -w name=source,emptyDir= -w name=gocache,emptyDir= -w name=gomodcache,emptyDir= --showlog
+        tkn task start cache-fetch-go -p gitURL=https://github.com/vdemeester/go-helloworld-app -p gitRevision=main -p cachePatterns="**.go,**go.sum" -p cacheURIBase=oci://${REGISTRY}/cache/go -w name=source,emptyDir= -w name=gocache,emptyDir= -w name=gomodcache,emptyDir= --showlog
+        tkn taskrun list
+        # FIXME: fail if something failed
         
   publish:
     name: publish latest

.ko.yaml

@@ -0,0 +1 @@
+defaultBaseImage: registry.access.redhat.com/ubi8/ubi-minimal

cmd/cache/fetch.go

@@ -1,53 +1,86 @@
 package main
 
 import (
+	"fmt"
+	"io/fs"
+	"os"
 	"path/filepath"
 
+	"github.com/moby/patternmatcher"
 	"github.com/openshift-pipelines/tekton-caches/internal/fetch"
 	"github.com/openshift-pipelines/tekton-caches/internal/hash"
 	"github.com/spf13/cobra"
 )
 
 const (
-	filesFlag  = "hashfiles"
-	targetFlag = "target"
-	folderFlag = "folder"
+	workingdirFlag = "workingdir"
+	filesFlag      = "hashfiles"
+	patternsFlag   = "pattern"
+	sourceFlag     = "source"
+	folderFlag     = "folder"
 )
 
 func fetchCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use: "fetch",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			files, err := cmd.Flags().GetString(filesFlag)
+			target, err := cmd.Flags().GetString(sourceFlag)
 			if err != nil {
 				return err
 			}
-			target, err := cmd.Flags().GetString(targetFlag)
+			folder, err := cmd.Flags().GetString(folderFlag)
 			if err != nil {
 				return err
 			}
-			folder, err := cmd.Flags().GetString(folderFlag)
+			workingdir, err := cmd.Flags().GetString(workingdirFlag)
 			if err != nil {
 				return err
 			}
-			// FIXME error out if empty
-
-			matches, err := filepath.Glob(files)
+			patterns, err := cmd.Flags().GetStringArray(patternsFlag)
 			if err != nil {
 				return err
 			}
+			matches := glob(workingdir, func(s string) bool {
+				m, err := patternmatcher.Matches(s, patterns)
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "error trying to match files with '%v': %s", patterns, err)
+					return false
+				}
+				return m
+			})
+			if len(matches) == 0 {
+				return fmt.Errorf("Didn't match any files with %v", patterns)
+			} else {
+				fmt.Fprintf(os.Stderr, "Matched the following files: %v\n", matches)
+			}
 			// TODO: Hash files based of matches
 			hashStr, err := hash.Compute(matches)
 			if err != nil {
 				return err
 			}
-			return fetch.Try(cmd.Context(), hashStr, target, folder)
+
+			// FIXME: Wrap the error.
+			// If not, warn and do not fail
+			// fmt.Fprintf(os.Stderr, "Repository %s doesn't exists or isn't reachable, fetching no cache.\n", cacheImageRef)
+			return fetch.Fetch(cmd.Context(), hashStr, target, folder)
 		},
 	}
 
-	cmd.Flags().String(filesFlag, "", "Files pattern to compute the hash from")
-	cmd.Flags().String(targetFlag, "", "Cache oci image target reference")
+	cmd.Flags().StringArray(patternsFlag, []string{}, "Files pattern to compute the hash from")
+	cmd.Flags().String(sourceFlag, "", "Cache source reference")
 	cmd.Flags().String(folderFlag, "", "Folder where to extract the content of the cache if it exists")
+	cmd.Flags().String(workingdirFlag, ".", "Working dir from where the files patterns needs to be taken")
 
 	return cmd
 }
+
+func glob(root string, fn func(string) bool) []string {
+	var files []string
+	filepath.WalkDir(root, func(s string, d fs.DirEntry, e error) error {
+		if fn(s) {
+			files = append(files, s)
+		}
+		return nil
+	})
+	return files
+}

cmd/cache/upload.go

@@ -1,35 +1,53 @@
 package main
 
 import (
-	"path/filepath"
+	"fmt"
+	"os"
 
+	"github.com/moby/patternmatcher"
 	"github.com/openshift-pipelines/tekton-caches/internal/hash"
 	"github.com/openshift-pipelines/tekton-caches/internal/upload"
 	"github.com/spf13/cobra"
 )
 
+const (
+	targetFlag = "target"
+)
+
 func uploadCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use: "upload",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			files, err := cmd.Flags().GetString(filesFlag)
+			target, err := cmd.Flags().GetString(targetFlag)
 			if err != nil {
 				return err
 			}
-			target, err := cmd.Flags().GetString(targetFlag)
+			folder, err := cmd.Flags().GetString(folderFlag)
 			if err != nil {
 				return err
 			}
-			folder, err := cmd.Flags().GetString(folderFlag)
+			workingdir, err := cmd.Flags().GetString(workingdirFlag)
 			if err != nil {
 				return err
 			}
-			// FIXME error out if empty
-
-			matches, err := filepath.Glob(files)
+			patterns, err := cmd.Flags().GetStringArray(patternsFlag)
 			if err != nil {
 				return err
 			}
+			matches := glob(workingdir, func(s string) bool {
+				m, err := patternmatcher.Matches(s, patterns)
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "error trying to match files with '%v': %s", patterns, err)
+					return false
+				}
+				return m
+			})
+			if len(matches) == 0 {
+				return fmt.Errorf("Didn't match any files with %v", patterns)
+			} else {
+				fmt.Fprintf(os.Stderr, "Matched the following files: %v\n", matches)
+			}
+
 			// TODO: Hash files based of matches
 			hashStr, err := hash.Compute(matches)
 			if err != nil {
@@ -38,9 +56,10 @@ func uploadCmd() *cobra.Command {
 			return upload.Upload(cmd.Context(), hashStr, target, folder)
 		},
 	}
-	cmd.Flags().String(filesFlag, "", "Files pattern to compute the hash from")
-	cmd.Flags().String(targetFlag, "", "Cache oci image target reference")
+	cmd.Flags().StringArray(patternsFlag, []string{}, "Files pattern to compute the hash from")
+	cmd.Flags().String(targetFlag, "", "Cache target reference")
 	cmd.Flags().String(folderFlag, "", "Folder where to extract the content of the cache if it exists")
+	cmd.Flags().String(workingdirFlag, ".", "Working dir from where the files patterns needs to be taken")
 
 	return cmd
 }

go.mod

@@ -5,6 +5,7 @@ go 1.21
 require (
 	github.com/codeclysm/extract/v3 v3.1.1
 	github.com/google/go-containerregistry v0.19.0
+	github.com/moby/patternmatcher v0.6.0
 	github.com/spf13/cobra v1.8.0
 )
 

go.sum

@@ -45,6 +45,8 @@ github.com/mattn/go-colorable v0.0.6/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO
 github.com/mattn/go-isatty v0.0.0-20160806122752-66b8e73f3f5c/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk=
+github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8=

internal/fetch/fetch.go

@@ -9,15 +9,15 @@ import (
 	"github.com/openshift-pipelines/tekton-caches/internal/provider/oci"
 )
 
-func Try(ctx context.Context, hash, target, folder string) error {
+func Fetch(ctx context.Context, hash, target, folder string) error {
 	u, err := url.Parse(target)
 	if err != nil {
 		return err
 	}
 	newTarget := strings.TrimPrefix(target, u.Scheme+"://")
 	switch u.Scheme {
 	case "oci":
-		return oci.Try(ctx, hash, newTarget, folder)
+		return oci.Fetch(ctx, hash, newTarget, folder)
 	case "s3":
 		return fmt.Errorf("s3 schema not (yet) supported: %s", target)
 	case "gcs":

internal/provider/oci/fetch.go

@@ -11,17 +11,15 @@ import (
 	"github.com/google/go-containerregistry/pkg/crane"
 )
 
-func Try(ctx context.Context, hash, target, folder string) error {
+func Fetch(ctx context.Context, hash, target, folder string) error {
 	cacheImageRef := strings.ReplaceAll(target, "{{hash}}", hash)
 	fmt.Fprintf(os.Stderr, "Trying to fetch oci image %s in %s\n", cacheImageRef, folder)
 
 	// Try to fetch it (if it exists)
 	image, err := crane.Pull(cacheImageRef)
 	if err != nil {
-		// If not, warn and do not fail
 		fmt.Fprintf(os.Stderr, "Warning: %s\n", err)
-		fmt.Fprintf(os.Stderr, "Repository %s doesn't exists or isn't reachable, fetching no cache.\n", cacheImageRef)
-		return nil
+		return err
 	}
 
 	f, err := os.Create(filepath.Join(folder, "cache.tar"))

tekton/cache-fetch.yaml

@@ -0,0 +1,65 @@
+apiVersion: tekton.dev/v1alpha1
+kind: StepAction
+metadata:
+  name: cache-fetch
+  annotations:
+    tekton.dev/pipelines.minVersion: "0.56.0"
+    tekton.dev/tags: "cache"
+spec:
+  params:
+    - name: patterns
+      description: |
+        Regular expression to select files to include to compute the hash.
+        For example, in the case of a Go project, you can use `go.mod` for this, so the value would be "**/go.sum" (to work with possible sub go modules as well).
+      type: array
+    - name: source
+      description: |
+        The source from where the cache should be fetched. It's a URI with the scheme defining the "provider". In addition, one can add a {{hash}} variable to use the computed hash in the reference (oci image tags, path in s3, …)
+        Currently supported:
+        - oci:// (e.g. oci://quay.io/vdemeester/go-cache:{{hash}}
+        - s3:// (e.g. s3://
+      type: string
+    - name: cachePath
+      description: |
+        Path where to extract the cache content.
+        It can refer any folder, backed by a workspace or a volume, or nothing.
+      type: string
+    - name: workingdir
+      description: |
+        The working dir from where the files patterns needs to be taken
+      type: string
+  results: # Any result to "publish" ?
+    - name: fetched
+      description: |
+        Whether a cache was fetched or not (true/false). This step won't fail if it didn't manage to fetch cache. This results allows the next step to act whether something was fetched or not.
+  env:
+    - name: PARAM_SOURCE
+      value: $(params.source)
+    - name: PARAM_CACHE_PATH
+      value: $(params.cachePath)
+    - name: PARAM_WORKINGDIR
+      value: $(params.workingdir)
+  # FIXME: use a released version once something is released :)
+  image: ghcr.io/openshift-pipelines/tekton-caches/cache:latest
+  args: ["$(params.patterns[*])"]
+  script: |
+    #!/bin/sh
+
+    PATTERN_FLAGS=""
+    echo "Patterns: $*"
+    for p in $*; do
+      PATTERN_FLAGS="${PATTERN_FLAGS} --pattern ${p}"
+    done
+
+    set -x
+    /ko-app/cache fetch ${PATTERN_FLAGS} \
+                        --source ${PARAM_SOURCE} \
+                        --folder ${PARAM_CACHE_PATH} \
+                        --workingdir ${PARAM_WORKINGDIR}
+    if [ $? -eq 0 ]; then
+      echo -n true > $(step.results.fetched.path)
+    else
+      echo -n false > $(step.results.fetched.path)
+    fi
+
+    exit 0