diff --git a/.github/workflows/latest.yaml b/.github/workflows/latest.yaml index 0720c30c1..6c2a924bb 100644 --- a/.github/workflows/latest.yaml +++ b/.github/workflows/latest.yaml @@ -75,10 +75,7 @@ jobs: needs: [build] # https://docs.github.com/en/actions/reference/authentication-in-a-workflow - permissions: - id-token: write - packages: write - contents: read + permissions: write-all steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 diff --git a/README.md b/README.md index 9247c6a93..5bbf6e72a 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,80 @@ # tekton-caches [![build-test-publish](https://github.com/openshift-pipelines/tekton-caches/actions/workflows/latest.yaml/badge.svg)](https://github.com/openshift-pipelines/tekton-caches/actions/workflows/latest.yaml) -Tools (and Task/StepAction) to managing within Tekton - -```bash -# With OCI -$ cache fetch --hasfiles '**/go.sum' --target oci://quay.io/vdemeest/cache/go-cache:{{hash}} --folder /workspaces/go-cache -$ cache upload --hashfiles '**/go.sum' --target oci://quay.io/vdemeest/cache/go-cache:{{hash}} --folder /workspaces/go-cache -# With s3 -$ cache fetch --hashfiles '**/go.sum' --target s3://my-bucket/path/to/my/cache --folder /workspaces/go-cache -$ cache fetch --hashfiles '**/go.sum' --target s3://my-bucket/path/to/my/cache --folder /workspaces/go-cache -# With gcs -$ cache fetch --hashfiles '**/go.sum' --target gcs://my-bucket/path/to/my/cache --folder /workspaces/go-cache -$ cache fetch --hashfiles '**/go.sum' --target gcs://my-bucket/path/to/my/cache --folder /workspaces/go-cache +This is a tool to cache resources like go cache/maven or others on TektonCD +pipelines. + +This tool supports uploading the cache to an OCI registry and plans to support +S3, GCS and other storage backends. + +It uses the new [StepActions](https://tekton.dev/docs/pipelines/stepactions/) +feature of TektonCD Pipelines but can be as well used without it. + +See the StepActions in the [tekton/](./tekton) directory. + +## Example + +This is an example of a build pipeline for a go application caching and reusing +the go cache. If the `go.mod` and `go.sum` are changed the cache is invalidated and +rebuilt. + +### Pre-requisites + +- You need a recent TektonCD pipelines installed with the StepActions feature-flags enabled. + +```shell +kubectl patch configmap -n tekton-pipelines --type merge -p '{"data":{"enable-step-actions": "true"}}' feature-flags +``` + +- A registry to push the images to. Example: docker.io/loginname. Make sure you + have setup tekton to be able to push/fetch from that registry, see the + [TektonCD pipelines documentation](https://tekton.dev/docs/pipelines/auth/#configuring-authentication-for-docker) + +### Usage + +Create the go pipeline example from the examples directory: + +```shell +kubectl create -f pipeline-go.yaml +``` + +Start it with the tkn cli (change the value as needed): + +```shell +tkn pipeline start pipeline-go --param repo_url=https://github.com/vdemeester/go-helloworld-app --param revision=main --param registry=docker.io/username -w name=source,emptyDir= +``` + +or with a PipelineRun yaml object: + +```shell +```yaml +kind: PipelineRun +metadata: + name: build-go-application-with-caching-run +spec: + pipelineRef: + name: pipeline-go + params: + - name: repo_url + value: https://github.com/vdemeester/go-helloworld-app + - name: revision + value: main + - name: registry + value: docker.io/username + workspaces: + - name: source + emptyDir: {} +``` + +- you can as well redefine the `buildCommand` which by default do a `go build + -v ./` with the `buildCommand` parameter, for example if you want instead to + run the tests on a repo with caching: + +```shell +tkn pipeline start pipeline-go --param repo_url=https://github.com/chmouel/gosmee \ + --param revision=main --param registry=docker.io/username \ + --param=buildCommand="make test" -w name=source,emptyDir= ``` + +## License + +[Apache License 2.0](./LICENSE) diff --git a/cmd/cache/fetch.go b/cmd/cache/fetch.go index af3a1b6de..ce432d81c 100644 --- a/cmd/cache/fetch.go +++ b/cmd/cache/fetch.go @@ -18,6 +18,7 @@ const ( patternsFlag = "pattern" sourceFlag = "source" folderFlag = "folder" + insecureFlag = "insecure" ) func fetchCmd() *cobra.Command { @@ -59,10 +60,14 @@ func fetchCmd() *cobra.Command { return err } + insecure, err := cmd.Flags().GetBool(insecureFlag) + if err != nil { + return err + } // FIXME: Wrap the error. // If not, warn and do not fail // fmt.Fprintf(os.Stderr, "Repository %s doesn't exists or isn't reachable, fetching no cache.\n", cacheImageRef) - return fetch.Fetch(cmd.Context(), hashStr, target, folder) + return fetch.Fetch(cmd.Context(), hashStr, target, folder, insecure) }, } @@ -70,6 +75,7 @@ func fetchCmd() *cobra.Command { cmd.Flags().String(sourceFlag, "", "Cache source reference") cmd.Flags().String(folderFlag, "", "Folder where to extract the content of the cache if it exists") cmd.Flags().String(workingdirFlag, ".", "Working dir from where the files patterns needs to be taken") + cmd.Flags().Bool(insecureFlag, false, "Wether to use insecure transport or not to upload to insecure registry") return cmd } diff --git a/cmd/cache/upload.go b/cmd/cache/upload.go index 5b7911943..8152f9926 100644 --- a/cmd/cache/upload.go +++ b/cmd/cache/upload.go @@ -34,6 +34,11 @@ func uploadCmd() *cobra.Command { if err != nil { return err } + + insecure, err := cmd.Flags().GetBool(insecureFlag) + if err != nil { + return err + } matches := glob(workingdir, func(s string) bool { m, err := patternmatcher.Matches(s, patterns) if err != nil { @@ -53,13 +58,15 @@ func uploadCmd() *cobra.Command { if err != nil { return err } - return upload.Upload(cmd.Context(), hashStr, target, folder) + // TODO: use a struct to pas arguments + return upload.Upload(cmd.Context(), hashStr, target, folder, insecure) }, } cmd.Flags().StringArray(patternsFlag, []string{}, "Files pattern to compute the hash from") cmd.Flags().String(targetFlag, "", "Cache target reference") cmd.Flags().String(folderFlag, "", "Folder where to extract the content of the cache if it exists") cmd.Flags().String(workingdirFlag, ".", "Working dir from where the files patterns needs to be taken") + cmd.Flags().Bool(insecureFlag, false, "Wether to use insecure transport or not to upload to insecure registry") return cmd } diff --git a/examples/pipeline-go.yaml b/examples/pipeline-go.yaml new file mode 100644 index 000000000..b3ef3e835 --- /dev/null +++ b/examples/pipeline-go.yaml @@ -0,0 +1,97 @@ +--- +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: pipeline-go +spec: + params: + - name: repo_url + type: string + - name: revision + type: string + - name: registry + type: string + - name: buildCommand + type: string + default: go build -v . + workspaces: + - name: source + tasks: + - displayName: Build go application + name: build-task + workspaces: + - name: source + workspace: source + taskSpec: + workspaces: + - name: source + params: + - name: buildCommand + default: $(params.buildCommand) + steps: + - name: create-repo + image: cgr.dev/chainguard/go + script: | + mkdir -p $(workspaces.source.path)/repo + chmod 777 $(workspaces.source.path)/repo + - name: fetch-repo + ref: + resolver: http + params: + - name: url + value: https://raw.githubusercontent.com/tektoncd/catalog/main/stepaction/git-clone/0.1/git-clone.yaml + params: + - name: output-path + value: $(workspaces.source.path)/repo + - name: url + value: $(params.repo_url) + - name: revision + value: $(params.revision) + - name: fetch-cache + ref: + resolver: http + params: + - name: url + value: https://raw.githubusercontent.com/openshift-pipelines/tekton-caches/main/tekton/cache-fetch.yaml + params: + - name: patterns + value: + - "**.go" + - "**go.sum" + - name: source + value: oci://$(params.registry)/cache-go:{{hash}} + - name: cachePath + value: $(workspaces.source.path)/cache + - name: workingdir + value: $(workspaces.source.path)/repo + - image: cgr.dev/chainguard/go + workingDir: $(workspaces.source.path)/repo + name: noop-task + env: + - name: GOCACHE + value: $(workspaces.source.path)/cache/gocache + - name: GOMODCACHE + value: $(workspaces.source.path)/cache/gomodcache + script: | + set -x + git config --global --add safe.directory $(workspaces.source.path)/repo + $(params.buildCommand) + du -shk $GOPATH + du -shk $GOMODCACHE + - name: cache-upload + ref: + resolver: http + params: + - name: url + value: https://raw.githubusercontent.com/openshift-pipelines/tekton-caches/main/tekton/cache-upload.yaml + params: + - name: patterns + value: + - "**.go" + - "**go.sum" + - name: target + value: oci://$(params.registry)/cache-go:{{hash}} + - name: cachePath + value: $(workspaces.source.path)/cache + - name: workingdir + value: $(workspaces.source.path)/repo diff --git a/examples/pipelinerun.yaml b/examples/pipelinerun.yaml new file mode 100644 index 000000000..8b0fe4c4a --- /dev/null +++ b/examples/pipelinerun.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + name: build-go-application-with-caching-run +spec: + pipelineRef: + name: build-go-application-with-caching + params: + - name: repo_url + value: https://github.com/vdemeester/go-helloworld-app + - name: revision + value: main + - name: registry + value: registry.civuole.local/cache + workspaces: + - name: source + emptyDir: {} diff --git a/internal/fetch/fetch.go b/internal/fetch/fetch.go index 371a55375..591a02192 100644 --- a/internal/fetch/fetch.go +++ b/internal/fetch/fetch.go @@ -4,12 +4,19 @@ import ( "context" "fmt" "net/url" + "os" "strings" "github.com/openshift-pipelines/tekton-caches/internal/provider/oci" ) -func Fetch(ctx context.Context, hash, target, folder string) error { +func Fetch(ctx context.Context, hash, target, folder string, insecure bool) error { + // check that folder exists or automatically create it + if _, err := os.Stat(folder); os.IsNotExist(err) { + if err := os.MkdirAll(folder, 0755); err != nil { + return fmt.Errorf("failed to create folder: %w", err) + } + } u, err := url.Parse(target) if err != nil { return err @@ -17,7 +24,7 @@ func Fetch(ctx context.Context, hash, target, folder string) error { newTarget := strings.TrimPrefix(target, u.Scheme+"://") switch u.Scheme { case "oci": - return oci.Fetch(ctx, hash, newTarget, folder) + return oci.Fetch(ctx, hash, newTarget, folder, insecure) case "s3": return fmt.Errorf("s3 schema not (yet) supported: %s", target) case "gcs": diff --git a/internal/provider/oci/fetch.go b/internal/provider/oci/fetch.go index 419f7a2f5..55bf21614 100644 --- a/internal/provider/oci/fetch.go +++ b/internal/provider/oci/fetch.go @@ -11,12 +11,16 @@ import ( "github.com/google/go-containerregistry/pkg/crane" ) -func Fetch(ctx context.Context, hash, target, folder string) error { +func Fetch(ctx context.Context, hash, target, folder string, insecure bool) error { cacheImageRef := strings.ReplaceAll(target, "{{hash}}", hash) fmt.Fprintf(os.Stderr, "Trying to fetch oci image %s in %s\n", cacheImageRef, folder) + options := []crane.Option{} + if insecure { + options = append(options, crane.Insecure) + } // Try to fetch it (if it exists) - image, err := crane.Pull(cacheImageRef) + image, err := crane.Pull(cacheImageRef, options...) if err != nil { fmt.Fprintf(os.Stderr, "Warning: %s\n", err) return err diff --git a/internal/provider/oci/upload.go b/internal/provider/oci/upload.go index 38eb85c21..80bd2d104 100644 --- a/internal/provider/oci/upload.go +++ b/internal/provider/oci/upload.go @@ -15,7 +15,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/types" ) -func Upload(ctx context.Context, hash, target, folder string) error { +func Upload(ctx context.Context, hash, target, folder string, insecure bool) error { cacheImageRef := strings.ReplaceAll(target, "{{hash}}", hash) fmt.Fprintf(os.Stderr, "Upload %s content to oci image %s\n", folder, cacheImageRef) @@ -42,7 +42,11 @@ func Upload(ctx context.Context, hash, target, folder string) error { return nil } - if err := crane.Push(image, cacheImageRef); err != nil { + options := []crane.Option{} + if insecure { + options = append(options, crane.Insecure) + } + if err := crane.Push(image, cacheImageRef, options...); err != nil { return err } diff --git a/internal/upload/upload.go b/internal/upload/upload.go index 02ee84476..9a2522729 100644 --- a/internal/upload/upload.go +++ b/internal/upload/upload.go @@ -9,7 +9,7 @@ import ( "github.com/openshift-pipelines/tekton-caches/internal/provider/oci" ) -func Upload(ctx context.Context, hash, target, folder string) error { +func Upload(ctx context.Context, hash, target, folder string, insecure bool) error { u, err := url.Parse(target) if err != nil { return err @@ -17,7 +17,7 @@ func Upload(ctx context.Context, hash, target, folder string) error { newTarget := strings.TrimPrefix(target, u.Scheme+"://") switch u.Scheme { case "oci": - return oci.Upload(ctx, hash, newTarget, folder) + return oci.Upload(ctx, hash, newTarget, folder, insecure) case "s3": return fmt.Errorf("s3 schema not (yet) supported: %s", target) case "gcs": diff --git a/tekton/cache-fetch.yaml b/tekton/cache-fetch.yaml index dbe92975d..e6f47c1c4 100644 --- a/tekton/cache-fetch.yaml +++ b/tekton/cache-fetch.yaml @@ -28,6 +28,11 @@ spec: description: | The working dir from where the files patterns needs to be taken type: string + - name: insecure + description: | + Whether to use insecure mode for fetching the cache + type: string + default: "false" results: # Any result to "publish" ? - name: fetched description: | @@ -39,6 +44,8 @@ spec: value: $(params.cachePath) - name: PARAM_WORKINGDIR value: $(params.workingdir) + - name: PARAM_INSECURE + value: $(params.insecure) # FIXME: use a released version once something is released :) image: ghcr.io/openshift-pipelines/tekton-caches/cache:latest args: ["$(params.patterns[*])"] @@ -55,6 +62,7 @@ spec: /ko-app/cache fetch ${PATTERN_FLAGS} \ --source ${PARAM_SOURCE} \ --folder ${PARAM_CACHE_PATH} \ + --insecure ${PARAM_INSECURE} \ --workingdir ${PARAM_WORKINGDIR} if [ $? -eq 0 ]; then echo -n true > $(step.results.fetched.path) diff --git a/tekton/cache-upload.yaml b/tekton/cache-upload.yaml index 39aac5cd6..fca127d24 100644 --- a/tekton/cache-upload.yaml +++ b/tekton/cache-upload.yaml @@ -28,6 +28,11 @@ spec: description: | The working dir from where the files patterns needs to be taken type: string + - name: insecure + description: | + Whether to use insecure mode for fetching the cache + type: string + default: "false" results: # Any result to "publish" ? - name: fetched description: | @@ -39,6 +44,8 @@ spec: value: $(params.cachePath) - name: PARAM_WORKINGDIR value: $(params.workingdir) + - name: PARAM_INSECURE + value: $(params.insecure) # FIXME: use a released version once something is released :) image: ghcr.io/openshift-pipelines/tekton-caches/cache:latest args: ["$(params.patterns[*])"] @@ -55,4 +62,5 @@ spec: /ko-app/cache upload ${PATTERN_FLAGS} \ --target ${PARAM_TARGET} \ --folder ${PARAM_CACHE_PATH} \ + --insecure ${PARAM_INSECURE} \ --workingdir ${PARAM_WORKINGDIR}