-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
SRVKP-6439 : Implement s3 provide for cache step action
SRVKP-6439 : Add Vendor Directory SRVKP-6439 : Encapsualte S3 Provider FixLint SRVKP-6439 : FixLint SRVKP-6439 : FixLint SRVKP-6439 : FixLint SRVKP-6439 : FixLint Fix Lint
- Loading branch information
1 parent
25e44c5
commit d57ce76
Showing
584 changed files
with
166,319 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
|
|
||
| kubectl patch configmap -n tekton-pipelines --type merge -p '{"data":{"enable-step-actions": "true"}}' feature-flags | ||
|
|
||
| kubectl create secret generic regcred \ | ||
| --from-file=config.json=${HOME}/.docker/config.json | ||
|
|
||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,108 @@ | ||
| --- | ||
| apiVersion: tekton.dev/v1 | ||
| kind: Pipeline | ||
| metadata: | ||
| name: pipeline | ||
| spec: | ||
| params: | ||
| - name: repo_url | ||
| type: string | ||
| - name: revision | ||
| type: string | ||
| - name: registry | ||
| type: string | ||
| - name: buildCommand | ||
| type: string | ||
| default: go build -v . | ||
| - name: cachePatterns | ||
| type: array | ||
| default: [ "**go.mod", "**go.sum" ] | ||
| - name: image | ||
| type: string | ||
| default: golang:latest | ||
| - name: force-cache-upload | ||
| type: string | ||
| default: "false" | ||
| workspaces: | ||
| - name: source | ||
| - name: cred | ||
| tasks: | ||
| - displayName: Build go application | ||
| name: build-task | ||
| workspaces: | ||
| - name: source | ||
| workspace: source | ||
| taskSpec: | ||
| workspaces: | ||
| - name: source | ||
| - name: cred | ||
| params: | ||
| - name: buildCommand | ||
| default: $(params.buildCommand) | ||
| - name: cachePatterns | ||
| default: $(params.cachePatterns) | ||
| - name: image | ||
| default: $(params.image) | ||
| steps: | ||
| - name: create-repo | ||
| image: $(params.image) | ||
| script: | | ||
| mkdir -p $(workspaces.source.path)/repo | ||
| chmod 777 $(workspaces.source.path)/repo | ||
| - name: fetch-repo | ||
| ref: | ||
| resolver: http | ||
| params: | ||
| - name: url | ||
| value: https://raw.githubusercontent.com/tektoncd/catalog/main/stepaction/git-clone/0.1/git-clone.yaml | ||
| params: | ||
| - name: output-path | ||
| value: $(workspaces.source.path)/repo | ||
| - name: url | ||
| value: $(params.repo_url) | ||
| - name: revision | ||
| value: $(params.revision) | ||
| - name: cache-fetch | ||
| ref: | ||
| name: cache-fetch | ||
| params: | ||
| - name: patterns | ||
| value: $(params.cachePatterns) | ||
| - name: source | ||
| value: $(params.registry)/cache-go:{{hash}} | ||
| - name: cachePath | ||
| value: $(workspaces.source.path)/cache | ||
| - name: workingdir | ||
| value: $(workspaces.source.path)/repo | ||
| - name: cred-store | ||
| value: $(workspaces.cred.path) | ||
|
|
||
| - name: run-go-build | ||
| workingDir: $(workspaces.source.path)/repo | ||
| image: $(params.image) | ||
| env: | ||
| - name: GOCACHE | ||
| value: $(workspaces.source.path)/cache/gocache | ||
| - name: GOMODCACHE | ||
| value: $(workspaces.source.path)/cache/gomodcache | ||
| script: | | ||
| set -x | ||
| git config --global --add safe.directory $(workspaces.source.path)/repo | ||
| $(params.buildCommand) | ||
| echo "Cache size is $(du -sh $(workspaces.source.path)/cache)" | ||
| - name: cache-upload | ||
| ref: | ||
| name: cache-upload | ||
| params: | ||
| - name: patterns | ||
| value: $(params.cachePatterns) | ||
| - name: target | ||
| value: $(params.registry)/cache-go:{{hash}} | ||
| - name: cachePath | ||
| value: $(workspaces.source.path)/cache | ||
| - name: workingdir | ||
| value: $(workspaces.source.path)/repo | ||
| - name: cred-store | ||
| value: $(workspaces.cred.path) | ||
| - name: force-cache-upload | ||
| value: $(params.force-cache-upload) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| --- | ||
| apiVersion: tekton.dev/v1 | ||
| kind: PipelineRun | ||
| metadata: | ||
| generateName: pipelinerun-oci- | ||
| spec: | ||
| pipelineRef: | ||
| name: pipeline | ||
| params: | ||
| - name: repo_url | ||
| value: https://github.com/chmouel/go-helloworld | ||
| - name: revision | ||
| value: main | ||
| # This uses S3 bucket to upload Caches | ||
| - name: registry | ||
| value: oci://docker.io/<username> | ||
| - name: buildCommand | ||
| value: go build -v ./ | ||
| - name: image | ||
| value: golang:1.21 | ||
| workspaces: | ||
| - name: cred | ||
| secret: | ||
| secretName: regcred | ||
| - name: source | ||
| emptyDir: {} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| --- | ||
| apiVersion: tekton.dev/v1 | ||
| kind: PipelineRun | ||
| metadata: | ||
| generateName: pipelinerun-s3- | ||
| spec: | ||
| pipelineRef: | ||
| name: pipeline | ||
| params: | ||
| - name: repo_url | ||
| value: https://github.com/chmouel/go-helloworld | ||
| - name: revision | ||
| value: main | ||
| # This uses S3 bucket to upload Caches | ||
| - name: registry | ||
| value: s3://tekton-caches | ||
| - name: buildCommand | ||
| value: go build -v ./ | ||
| - name: image | ||
| value: golang:1.21 | ||
| workspaces: | ||
| - name: cred | ||
| secret: | ||
| secretName: aws-cred | ||
| - name: source | ||
| emptyDir: {} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| apiVersion: tekton.dev/v1alpha1 | ||
| kind: StepAction | ||
| metadata: | ||
| name: cache-fetch | ||
| annotations: | ||
| tekton.dev/pipelines.minVersion: "0.56.0" | ||
| tekton.dev/tags: "cache" | ||
| spec: | ||
| params: | ||
| - name: patterns | ||
| description: | | ||
| Regular expression to select files to include to compute the hash. | ||
| For example, in the case of a Go project, you can use `go.mod` for this, so the value would be "**/go.sum" (to work with possible sub go modules as well). | ||
| type: array | ||
| - name: source | ||
| description: | | ||
| The source from where the cache should be fetched. It's a URI with the scheme defining the "provider". In addition, one can add a {{hash}} variable to use the computed hash in the reference (oci image tags, path in s3, …) | ||
| Currently supported: | ||
| - oci:// (e.g. oci://quay.io/vdemeester/go-cache:{{hash}} | ||
| - s3:// (e.g. s3:// | ||
| type: string | ||
| - name: cachePath | ||
| description: | | ||
| Path where to extract the cache content. | ||
| It can refer any folder, backed by a workspace or a volume, or nothing. | ||
| type: string | ||
| - name: workingdir | ||
| description: | | ||
| The working dir from where the files patterns needs to be taken | ||
| type: string | ||
| - name: insecure | ||
| description: | | ||
| Whether to use insecure mode for fetching the cache | ||
| type: string | ||
| default: "false" | ||
| - name: googleCredentialsPath | ||
| description: | | ||
| The path where to find the google credentials. If left empty, it is ignored. | ||
| type: string | ||
| default: "" | ||
| - name: cred-store | ||
| description: | | ||
| The path where to find the creds to download cache files . If left empty, it is ignored. | ||
| type: string | ||
| default: "" | ||
| results: # Any result to "publish" ? | ||
| - name: fetched | ||
| description: | | ||
| Whether a cache was fetched or not (true/false). This step won't fail if it didn't manage to fetch cache. This results allows the next step to act whether something was fetched or not. | ||
| env: | ||
| - name: PARAM_SOURCE | ||
| value: $(params.source) | ||
| - name: PARAM_CACHE_PATH | ||
| value: $(params.cachePath) | ||
| - name: PARAM_WORKINGDIR | ||
| value: $(params.workingdir) | ||
| - name: PARAM_INSECURE | ||
| value: $(params.insecure) | ||
| - name: GOOGLE_APPLICATION_CREDENTIALS | ||
| value: $(params.googleCredentialsPath) | ||
| - name: CRED_STORE | ||
| value: $(params.cred-store) | ||
|
|
||
| # FIXME: use a released version once something is released :) | ||
| image: ko://github.com/openshift-pipelines/tekton-caches/cmd/cache | ||
| args: ["$(params.patterns[*])"] | ||
| script: | | ||
| #!/bin/sh | ||
| PATTERN_FLAGS="" | ||
| echo "Patterns: $*" | ||
| for p in $*; do | ||
| PATTERN_FLAGS="${PATTERN_FLAGS} --pattern ${p}" | ||
| done | ||
| set -x | ||
| /ko-app/cache fetch ${PATTERN_FLAGS} \ | ||
| --source ${PARAM_SOURCE} \ | ||
| --folder ${PARAM_CACHE_PATH} \ | ||
| --insecure ${PARAM_INSECURE} \ | ||
| --workingdir ${PARAM_WORKINGDIR} | ||
| if [ $? -eq 0 ]; then | ||
| echo -n true > $(step.results.fetched.path) | ||
| else | ||
| echo -n false > $(step.results.fetched.path) | ||
| fi | ||
| exit 0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| apiVersion: tekton.dev/v1alpha1 | ||
| kind: StepAction | ||
| metadata: | ||
| name: cache-upload | ||
| annotations: | ||
| tekton.dev/pipelines.minVersion: "0.56.0" | ||
| tekton.dev/tags: "cache" | ||
| spec: | ||
| params: | ||
| - name: patterns | ||
| description: | | ||
| Regular expression to select files to include to compute the hash. | ||
| For example, in the case of a Go project, you can use `go.mod` for this, so the value would be "**/go.sum" (to work with possible sub go modules as well). | ||
| type: array | ||
| - name: target | ||
| description: | | ||
| The target from where the cache should be uploaded. It's a URI with the scheme defining the "provider". In addition, one can add a {{hash}} variable to use the computed hash in the reference (oci image tags, path in s3, …) | ||
| Currently supported: | ||
| - oci:// (e.g. oci://quay.io/vdemeester/go-cache:{{hash}} | ||
| - s3:// (e.g. s3:// | ||
| type: string | ||
| - name: cachePath | ||
| description: | | ||
| Path where to extract the cache content. | ||
| It can refer any folder, backed by a workspace or a volume, or nothing. | ||
| type: string | ||
| - name: workingdir | ||
| description: | | ||
| The working dir from where the files patterns needs to be taken | ||
| type: string | ||
| - name: insecure | ||
| description: | | ||
| Whether to use insecure mode for fetching the cache | ||
| type: string | ||
| default: "false" | ||
| - name: fetched | ||
| description: | | ||
| Wether cache was fetched or not previously | ||
| type: string | ||
| default: "false" | ||
| - name: force-cache-upload | ||
| description: | | ||
| Whether to force the cache upload even if it was fetched previously | ||
| type: string | ||
| default: "false" | ||
| - name: googleCredentialsPath | ||
| description: | | ||
| The path where to find the google credentials. If left empty, it is ignored. | ||
| type: string | ||
| default: "" | ||
| - name: cred-store | ||
| description: | | ||
| The path where to find the creds to upload cache files . If left empty, it is ignored. | ||
| type: string | ||
| default: "" | ||
| env: | ||
| - name: PARAM_TARGET | ||
| value: $(params.target) | ||
| - name: PARAM_CACHE_PATH | ||
| value: $(params.cachePath) | ||
| - name: PARAM_WORKINGDIR | ||
| value: $(params.workingdir) | ||
| - name: PARAM_INSECURE | ||
| value: $(params.insecure) | ||
| - name: RESULT_CACHE_FETCHED | ||
| value: $(params.fetched) | ||
| - name: PARAM_FORCE_CACHE_UPLOAD | ||
| value: $(params.force-cache-upload) | ||
| - name: GOOGLE_APPLICATION_CREDENTIALS | ||
| value: $(params.googleCredentialsPath) | ||
| - name: CRED_STORE | ||
| value: $(params.cred-store) | ||
| # FIXME: use a released version once something is released :) | ||
| image: ko://github.com/openshift-pipelines/tekton-caches/cmd/cache | ||
| args: ["$(params.patterns[*])"] | ||
| script: | | ||
| #!/usr/bin/env sh | ||
| set -x | ||
| if [[ ${PARAM_FORCE_CACHE_UPLOAD} == "false" && ${RESULT_CACHE_FETCHED} == "true" ]]; then | ||
| echo "no need to upload cache" | ||
| exit 0 | ||
| fi | ||
| PATTERN_FLAGS="" | ||
| echo "Patterns: $*" | ||
| for p in $*; do | ||
| PATTERN_FLAGS="${PATTERN_FLAGS} --pattern ${p}" | ||
| done | ||
| set -ex | ||
| /ko-app/cache upload ${PATTERN_FLAGS} \ | ||
| --target ${PARAM_TARGET} \ | ||
| --folder ${PARAM_CACHE_PATH} \ | ||
| --insecure ${PARAM_INSECURE} \ | ||
| --workingdir ${PARAM_WORKINGDIR} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.