From 2a6d49fdab3d36392628406576dc877ff4421cb8 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Wed, 24 Apr 2024 18:19:06 +0200 Subject: [PATCH 1/7] add insecure flag --- cmd/cache/fetch.go | 8 +++++++- cmd/cache/upload.go | 9 ++++++++- internal/fetch/fetch.go | 4 ++-- internal/provider/oci/fetch.go | 8 ++++++-- internal/provider/oci/upload.go | 8 ++++++-- internal/upload/upload.go | 4 ++-- 6 files changed, 31 insertions(+), 10 deletions(-) diff --git a/cmd/cache/fetch.go b/cmd/cache/fetch.go index af3a1b6de..ce432d81c 100644 --- a/cmd/cache/fetch.go +++ b/cmd/cache/fetch.go @@ -18,6 +18,7 @@ const ( patternsFlag = "pattern" sourceFlag = "source" folderFlag = "folder" + insecureFlag = "insecure" ) func fetchCmd() *cobra.Command { @@ -59,10 +60,14 @@ func fetchCmd() *cobra.Command { return err } + insecure, err := cmd.Flags().GetBool(insecureFlag) + if err != nil { + return err + } // FIXME: Wrap the error. // If not, warn and do not fail // fmt.Fprintf(os.Stderr, "Repository %s doesn't exists or isn't reachable, fetching no cache.\n", cacheImageRef) - return fetch.Fetch(cmd.Context(), hashStr, target, folder) + return fetch.Fetch(cmd.Context(), hashStr, target, folder, insecure) }, } @@ -70,6 +75,7 @@ func fetchCmd() *cobra.Command { cmd.Flags().String(sourceFlag, "", "Cache source reference") cmd.Flags().String(folderFlag, "", "Folder where to extract the content of the cache if it exists") cmd.Flags().String(workingdirFlag, ".", "Working dir from where the files patterns needs to be taken") + cmd.Flags().Bool(insecureFlag, false, "Wether to use insecure transport or not to upload to insecure registry") return cmd } diff --git a/cmd/cache/upload.go b/cmd/cache/upload.go index 5b7911943..8152f9926 100644 --- a/cmd/cache/upload.go +++ b/cmd/cache/upload.go @@ -34,6 +34,11 @@ func uploadCmd() *cobra.Command { if err != nil { return err } + + insecure, err := cmd.Flags().GetBool(insecureFlag) + if err != nil { + return err + } matches := glob(workingdir, func(s string) bool { m, err := patternmatcher.Matches(s, patterns) if err != nil { @@ -53,13 +58,15 @@ func uploadCmd() *cobra.Command { if err != nil { return err } - return upload.Upload(cmd.Context(), hashStr, target, folder) + // TODO: use a struct to pas arguments + return upload.Upload(cmd.Context(), hashStr, target, folder, insecure) }, } cmd.Flags().StringArray(patternsFlag, []string{}, "Files pattern to compute the hash from") cmd.Flags().String(targetFlag, "", "Cache target reference") cmd.Flags().String(folderFlag, "", "Folder where to extract the content of the cache if it exists") cmd.Flags().String(workingdirFlag, ".", "Working dir from where the files patterns needs to be taken") + cmd.Flags().Bool(insecureFlag, false, "Wether to use insecure transport or not to upload to insecure registry") return cmd } diff --git a/internal/fetch/fetch.go b/internal/fetch/fetch.go index 371a55375..e5e562461 100644 --- a/internal/fetch/fetch.go +++ b/internal/fetch/fetch.go @@ -9,7 +9,7 @@ import ( "github.com/openshift-pipelines/tekton-caches/internal/provider/oci" ) -func Fetch(ctx context.Context, hash, target, folder string) error { +func Fetch(ctx context.Context, hash, target, folder string, insecure bool) error { u, err := url.Parse(target) if err != nil { return err @@ -17,7 +17,7 @@ func Fetch(ctx context.Context, hash, target, folder string) error { newTarget := strings.TrimPrefix(target, u.Scheme+"://") switch u.Scheme { case "oci": - return oci.Fetch(ctx, hash, newTarget, folder) + return oci.Fetch(ctx, hash, newTarget, folder, insecure) case "s3": return fmt.Errorf("s3 schema not (yet) supported: %s", target) case "gcs": diff --git a/internal/provider/oci/fetch.go b/internal/provider/oci/fetch.go index 419f7a2f5..55bf21614 100644 --- a/internal/provider/oci/fetch.go +++ b/internal/provider/oci/fetch.go @@ -11,12 +11,16 @@ import ( "github.com/google/go-containerregistry/pkg/crane" ) -func Fetch(ctx context.Context, hash, target, folder string) error { +func Fetch(ctx context.Context, hash, target, folder string, insecure bool) error { cacheImageRef := strings.ReplaceAll(target, "{{hash}}", hash) fmt.Fprintf(os.Stderr, "Trying to fetch oci image %s in %s\n", cacheImageRef, folder) + options := []crane.Option{} + if insecure { + options = append(options, crane.Insecure) + } // Try to fetch it (if it exists) - image, err := crane.Pull(cacheImageRef) + image, err := crane.Pull(cacheImageRef, options...) if err != nil { fmt.Fprintf(os.Stderr, "Warning: %s\n", err) return err diff --git a/internal/provider/oci/upload.go b/internal/provider/oci/upload.go index 38eb85c21..80bd2d104 100644 --- a/internal/provider/oci/upload.go +++ b/internal/provider/oci/upload.go @@ -15,7 +15,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/types" ) -func Upload(ctx context.Context, hash, target, folder string) error { +func Upload(ctx context.Context, hash, target, folder string, insecure bool) error { cacheImageRef := strings.ReplaceAll(target, "{{hash}}", hash) fmt.Fprintf(os.Stderr, "Upload %s content to oci image %s\n", folder, cacheImageRef) @@ -42,7 +42,11 @@ func Upload(ctx context.Context, hash, target, folder string) error { return nil } - if err := crane.Push(image, cacheImageRef); err != nil { + options := []crane.Option{} + if insecure { + options = append(options, crane.Insecure) + } + if err := crane.Push(image, cacheImageRef, options...); err != nil { return err } diff --git a/internal/upload/upload.go b/internal/upload/upload.go index 02ee84476..9a2522729 100644 --- a/internal/upload/upload.go +++ b/internal/upload/upload.go @@ -9,7 +9,7 @@ import ( "github.com/openshift-pipelines/tekton-caches/internal/provider/oci" ) -func Upload(ctx context.Context, hash, target, folder string) error { +func Upload(ctx context.Context, hash, target, folder string, insecure bool) error { u, err := url.Parse(target) if err != nil { return err @@ -17,7 +17,7 @@ func Upload(ctx context.Context, hash, target, folder string) error { newTarget := strings.TrimPrefix(target, u.Scheme+"://") switch u.Scheme { case "oci": - return oci.Upload(ctx, hash, newTarget, folder) + return oci.Upload(ctx, hash, newTarget, folder, insecure) case "s3": return fmt.Errorf("s3 schema not (yet) supported: %s", target) case "gcs": From 1c3b2261119433b7f7848d2bc1a57113d280fb9f Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Wed, 24 Apr 2024 18:42:20 +0200 Subject: [PATCH 2/7] add insecure to the task as well --- tekton/cache-fetch.yaml | 8 ++++++++ tekton/cache-upload.yaml | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/tekton/cache-fetch.yaml b/tekton/cache-fetch.yaml index dbe92975d..021e812ce 100644 --- a/tekton/cache-fetch.yaml +++ b/tekton/cache-fetch.yaml @@ -28,6 +28,11 @@ spec: description: | The working dir from where the files patterns needs to be taken type: string + - name: insecure + description: | + Whether to use insecure mode for fetching the cache + type: boolean + default: false results: # Any result to "publish" ? - name: fetched description: | @@ -39,6 +44,8 @@ spec: value: $(params.cachePath) - name: PARAM_WORKINGDIR value: $(params.workingdir) + - name: PARAM_INSECURE + value: $(params.insecure) # FIXME: use a released version once something is released :) image: ghcr.io/openshift-pipelines/tekton-caches/cache:latest args: ["$(params.patterns[*])"] @@ -55,6 +62,7 @@ spec: /ko-app/cache fetch ${PATTERN_FLAGS} \ --source ${PARAM_SOURCE} \ --folder ${PARAM_CACHE_PATH} \ + --insecure ${PARAM_INSECURE} \ --workingdir ${PARAM_WORKINGDIR} if [ $? -eq 0 ]; then echo -n true > $(step.results.fetched.path) diff --git a/tekton/cache-upload.yaml b/tekton/cache-upload.yaml index 39aac5cd6..cacad33cd 100644 --- a/tekton/cache-upload.yaml +++ b/tekton/cache-upload.yaml @@ -28,6 +28,11 @@ spec: description: | The working dir from where the files patterns needs to be taken type: string + - name: insecure + description: | + Whether to use insecure mode for fetching the cache + type: boolean + default: false results: # Any result to "publish" ? - name: fetched description: | @@ -39,6 +44,8 @@ spec: value: $(params.cachePath) - name: PARAM_WORKINGDIR value: $(params.workingdir) + - name: PARAM_INSECURE + value: $(params.insecure) # FIXME: use a released version once something is released :) image: ghcr.io/openshift-pipelines/tekton-caches/cache:latest args: ["$(params.patterns[*])"] @@ -55,4 +62,5 @@ spec: /ko-app/cache upload ${PATTERN_FLAGS} \ --target ${PARAM_TARGET} \ --folder ${PARAM_CACHE_PATH} \ + --insecure ${PARAM_INSECURE} \ --workingdir ${PARAM_WORKINGDIR} From 0cbfeaa1d1cfcba662e86f3d8692f28168585c16 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Wed, 24 Apr 2024 18:51:42 +0200 Subject: [PATCH 3/7] there is no boolean in tekton let's use string instead --- tekton/cache-fetch.yaml | 4 ++-- tekton/cache-upload.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tekton/cache-fetch.yaml b/tekton/cache-fetch.yaml index 021e812ce..e6f47c1c4 100644 --- a/tekton/cache-fetch.yaml +++ b/tekton/cache-fetch.yaml @@ -31,8 +31,8 @@ spec: - name: insecure description: | Whether to use insecure mode for fetching the cache - type: boolean - default: false + type: string + default: "false" results: # Any result to "publish" ? - name: fetched description: | diff --git a/tekton/cache-upload.yaml b/tekton/cache-upload.yaml index cacad33cd..fca127d24 100644 --- a/tekton/cache-upload.yaml +++ b/tekton/cache-upload.yaml @@ -31,8 +31,8 @@ spec: - name: insecure description: | Whether to use insecure mode for fetching the cache - type: boolean - default: false + type: string + default: "false" results: # Any result to "publish" ? - name: fetched description: | From ea1af5f28a4d9533c55658421b4aaa67aade6d41 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Fri, 26 Apr 2024 10:02:38 +0200 Subject: [PATCH 4/7] Create folder target on fetch if not exist When fetching the folder target may not exist yet so let create it automatically if not exist. --- internal/fetch/fetch.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/fetch/fetch.go b/internal/fetch/fetch.go index e5e562461..591a02192 100644 --- a/internal/fetch/fetch.go +++ b/internal/fetch/fetch.go @@ -4,12 +4,19 @@ import ( "context" "fmt" "net/url" + "os" "strings" "github.com/openshift-pipelines/tekton-caches/internal/provider/oci" ) func Fetch(ctx context.Context, hash, target, folder string, insecure bool) error { + // check that folder exists or automatically create it + if _, err := os.Stat(folder); os.IsNotExist(err) { + if err := os.MkdirAll(folder, 0755); err != nil { + return fmt.Errorf("failed to create folder: %w", err) + } + } u, err := url.Parse(target) if err != nil { return err From b7abed0f064f72fa66e34a920f7b4a16ebf91e6c Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Fri, 26 Apr 2024 11:32:58 +0200 Subject: [PATCH 5/7] Add pipeline example and update readme Signed-off-by: Chmouel Boudjnah --- README.md | 79 +++++++++++++++++++++++++++++------ examples/pipeline-go.yaml | 88 +++++++++++++++++++++++++++++++++++++++ examples/pipelinerun.yaml | 18 ++++++++ 3 files changed, 173 insertions(+), 12 deletions(-) create mode 100644 examples/pipeline-go.yaml create mode 100644 examples/pipelinerun.yaml diff --git a/README.md b/README.md index 9247c6a93..40a88498a 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,70 @@ # tekton-caches [![build-test-publish](https://github.com/openshift-pipelines/tekton-caches/actions/workflows/latest.yaml/badge.svg)](https://github.com/openshift-pipelines/tekton-caches/actions/workflows/latest.yaml) -Tools (and Task/StepAction) to managing within Tekton - -```bash -# With OCI -$ cache fetch --hasfiles '**/go.sum' --target oci://quay.io/vdemeest/cache/go-cache:{{hash}} --folder /workspaces/go-cache -$ cache upload --hashfiles '**/go.sum' --target oci://quay.io/vdemeest/cache/go-cache:{{hash}} --folder /workspaces/go-cache -# With s3 -$ cache fetch --hashfiles '**/go.sum' --target s3://my-bucket/path/to/my/cache --folder /workspaces/go-cache -$ cache fetch --hashfiles '**/go.sum' --target s3://my-bucket/path/to/my/cache --folder /workspaces/go-cache -# With gcs -$ cache fetch --hashfiles '**/go.sum' --target gcs://my-bucket/path/to/my/cache --folder /workspaces/go-cache -$ cache fetch --hashfiles '**/go.sum' --target gcs://my-bucket/path/to/my/cache --folder /workspaces/go-cache +This is a tool to cache resources like go cache/maven or others on TektonCD +pipelines. + +This tool supports uploading the cache to an OCI registry and plans to support +S3, GCS and other storage backends. + +It uses the new [StepActions](https://tekton.dev/docs/pipelines/stepactions/) +feature of TektonCD Pipelines but can be as well used without it. + +See the StepActions in the [tekton/](./tekton) directory. + +## Example + +This is an example of a build pipeline for a go application caching and reusing +the go cache. If the `go.mod` and `go.sum` are changed the cache is invalidated and +rebuilt. + +### Pre-requisites + +- You need a recent TektonCD pipelines installed with the StepActions feature-flags enabled. + +```shell +kubectl patch configmap -n tekton-pipelines --type merge -p '{"data":{"enable-step-actions": "true"}}' feature-flags ``` + +- A registry to push the images to. Example: docker.io/loginname. Make sure you + have setup tekton to be able to push/fetch from that registry, see the + [TektonCD pipelines documentation](https://tekton.dev/docs/pipelines/auth/#configuring-authentication-for-docker) + +### Usage + +Create the go pipeline example from the examples directory: + +```shell +kubectl create -f pipeline-go.yaml +``` + +Start it with the tkn cli (change the value as needed): + +```shell +tkn pipeline start pipeline-go --param repo_url=https://github.com/vdemeester/go-helloworld-app --param revision=main --param registry=docker.io/username -w name=source,emptyDir= +``` + +or with a PipelineRun yaml object: + +```shell +```yaml +kind: PipelineRun +metadata: + name: build-go-application-with-caching-run +spec: + pipelineRef: + name: pipeline-go + params: + - name: repo_url + value: https://github.com/vdemeester/go-helloworld-app + - name: revision + value: main + - name: registry + value: docker.io/username + workspaces: + - name: source + emptyDir: {} +``` + +## License + +[Apache License 2.0](./LICENSE) diff --git a/examples/pipeline-go.yaml b/examples/pipeline-go.yaml new file mode 100644 index 000000000..308c196e5 --- /dev/null +++ b/examples/pipeline-go.yaml @@ -0,0 +1,88 @@ +--- +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: pipeline-go +spec: + params: + - name: repo_url + type: string + - name: revision + type: string + - name: registry + type: string + workspaces: + - name: source + tasks: + - displayName: Build go application + name: noop-task + workspaces: + - name: source + workspace: source + taskSpec: + workspaces: + - name: source + steps: + - name: create-repo + image: cgr.dev/chainguard/go + script: | + mkdir -p $(workspaces.source.path)/repo + chmod 777 $(workspaces.source.path)/repo + - name: fetch-repo + ref: + resolver: http + params: + - name: url + value: https://raw.githubusercontent.com/tektoncd/catalog/main/stepaction/git-clone/0.1/git-clone.yaml + params: + - name: output-path + value: $(workspaces.source.path)/repo + - name: url + value: $(params.repo_url) + - name: revision + value: $(params.revision) + - name: cache-fetch + ref: + resolver: http + params: + - name: url + value: https://raw.githubusercontent.com/openshift-pipelines/tekton-caches/main/tekton/cache-fetch.yaml + params: + - name: patterns + value: + - "**.go" + - "**go.sum" + - name: source + value: oci://$(params.registry)/cache-go:{{hash}} + - name: cachePath + value: $(workspaces.source.path)/cache + - name: workingdir + value: $(workspaces.source.path)/repo + - image: cgr.dev/chainguard/go + workingDir: $(workspaces.source.path)/repo + name: noop-task + script: | + set -x + git config --global --add safe.directory $(workspaces.source.path)/repo + export GOCACHE=$(workspaces.source.path)/cache/gocache + export GOMODCACHE=$(workspaces.source.path)/cache/gomodcache + go build -v . + du -shk $GOPATH + du -shk $GOMODCACHE + - name: cache-upload + ref: + resolver: http + params: + - name: url + value: https://raw.githubusercontent.com/openshift-pipelines/tekton-caches/main/tekton/cache-upload.yaml + params: + - name: patterns + value: + - "**.go" + - "**go.sum" + - name: target + value: oci://$(params.registry)/cache-go:{{hash}} + - name: cachePath + value: $(workspaces.source.path)/cache + - name: workingdir + value: $(workspaces.source.path)/repo diff --git a/examples/pipelinerun.yaml b/examples/pipelinerun.yaml new file mode 100644 index 000000000..8b0fe4c4a --- /dev/null +++ b/examples/pipelinerun.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + name: build-go-application-with-caching-run +spec: + pipelineRef: + name: build-go-application-with-caching + params: + - name: repo_url + value: https://github.com/vdemeester/go-helloworld-app + - name: revision + value: main + - name: registry + value: registry.civuole.local/cache + workspaces: + - name: source + emptyDir: {} From 27a3e1baa52790947444f5f1d4174ad4e3f9373b Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Fri, 26 Apr 2024 12:39:17 +0200 Subject: [PATCH 6/7] Add buildCommand And show how to run tests Signed-off-by: Chmouel Boudjnah --- README.md | 10 ++++++++++ examples/pipeline-go.yaml | 19 ++++++++++++++----- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 40a88498a..5bbf6e72a 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,16 @@ spec: emptyDir: {} ``` +- you can as well redefine the `buildCommand` which by default do a `go build + -v ./` with the `buildCommand` parameter, for example if you want instead to + run the tests on a repo with caching: + +```shell +tkn pipeline start pipeline-go --param repo_url=https://github.com/chmouel/gosmee \ + --param revision=main --param registry=docker.io/username \ + --param=buildCommand="make test" -w name=source,emptyDir= +``` + ## License [Apache License 2.0](./LICENSE) diff --git a/examples/pipeline-go.yaml b/examples/pipeline-go.yaml index 308c196e5..b3ef3e835 100644 --- a/examples/pipeline-go.yaml +++ b/examples/pipeline-go.yaml @@ -11,17 +11,23 @@ spec: type: string - name: registry type: string + - name: buildCommand + type: string + default: go build -v . workspaces: - name: source tasks: - displayName: Build go application - name: noop-task + name: build-task workspaces: - name: source workspace: source taskSpec: workspaces: - name: source + params: + - name: buildCommand + default: $(params.buildCommand) steps: - name: create-repo image: cgr.dev/chainguard/go @@ -41,7 +47,7 @@ spec: value: $(params.repo_url) - name: revision value: $(params.revision) - - name: cache-fetch + - name: fetch-cache ref: resolver: http params: @@ -61,12 +67,15 @@ spec: - image: cgr.dev/chainguard/go workingDir: $(workspaces.source.path)/repo name: noop-task + env: + - name: GOCACHE + value: $(workspaces.source.path)/cache/gocache + - name: GOMODCACHE + value: $(workspaces.source.path)/cache/gomodcache script: | set -x git config --global --add safe.directory $(workspaces.source.path)/repo - export GOCACHE=$(workspaces.source.path)/cache/gocache - export GOMODCACHE=$(workspaces.source.path)/cache/gomodcache - go build -v . + $(params.buildCommand) du -shk $GOPATH du -shk $GOMODCACHE - name: cache-upload From a6b6670bd5cf4d043b0a424394b2d17eeeef23fc Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Fri, 26 Apr 2024 13:14:01 +0200 Subject: [PATCH 7/7] add permissions write-all to workflow we are getting this error: Error: failed to publish images: error publishing ko://github.com/openshift-pipelines/tekton-caches/cmd/cache: writing sbom: POST https://ghcr.io/v2/openshift-pipelines/tekton-caches/cache/blobs/uploads/: DENIED: installation not allowed to Write organization package --- .github/workflows/latest.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/latest.yaml b/.github/workflows/latest.yaml index 5f41cf60c..7feda75c0 100644 --- a/.github/workflows/latest.yaml +++ b/.github/workflows/latest.yaml @@ -75,10 +75,7 @@ jobs: needs: [build] # https://docs.github.com/en/actions/reference/authentication-in-a-workflow - permissions: - id-token: write - packages: write - contents: read + permissions: write-all steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1