diff --git a/docs/matrix.md b/docs/matrix.md index 463ae1c5215..9e7a4c62105 100644 --- a/docs/matrix.md +++ b/docs/matrix.md @@ -17,6 +17,8 @@ weight: 406 - [Parameters in Matrix.Include.Params](#parameters-in-matrixincludeparams) - [Specifying both `params` and `matrix` in a `PipelineTask`](#specifying-both-params-and-matrix-in-a-pipelinetask) - [Context Variables](#context-variables) + - [Access Matrix Combinations Length](#access-matrix-combinations-length) + - [Access Aggregated Results Length](#access-aggregated-results-length) - [Results](#results) - [Specifying Results in a Matrix](#specifying-results-in-a-matrix) - [Results in Matrix.Params](#results-in-matrixparams) @@ -291,6 +293,38 @@ Similarly to the `Parameters` in the `Params` field, the `Parameters` in the `Ma * `Pipeline` name * `PipelineTask` retries + +The following `context` variables allow users to access the `matrix` runtime data. Note: In order to create an ordering dependency, use `runAfter` or `taskResult` consumption as part of the same pipelineTask. + +#### Access Matrix Combinations Length + +The pipeline authors can access the total number of instances created as part of the `matrix` using the syntax: `tasks..matrix.length`. + +```yaml + - name: matrixed-echo-length + runAfter: + - matrix-emitting-results + params: + - name: matrixlength + value: $(tasks.matrix-emitting-results.matrix.length) +``` + +#### Access Aggregated Results Length + +The pipeline authors can access the length of the array of aggregated results that were +actually produced using the syntax: `tasks..matrix..length`. This will allow users to loop over the results produced. + +```yaml + - name: matrixed-echo-results-length + runAfter: + - matrix-emitting-results + params: + - name: matrixlength + value: $(tasks.matrix-emitting-results.matrix.a-result.length) +``` + +See the full example here: [pr-with-matrix-context-variables] + ## Results ### Specifying Results in a Matrix @@ -360,8 +394,51 @@ tasks: ### Results from fanned out Matrixed PipelineTasks -Emitting `Results` from fanned out `PipelineTasks` is not currently supported. -We plan to support emitting `Results` from fanned out `PipelineTasks` in the near future. +Emitting `Results` from fanned out `PipelineTasks` is now supported. Each fanned out +`TaskRun` that produces `Result` of type `string` will be aggregated into an `array` +of `Results` during reconciliation, in which the whole `array` of `Results` can be consumed by another `pipelineTask` using the star notion [*]. +Note: A known limitation is not being able to consume a singular result or specific +combinations of results produced by a previous fanned out `PipelineTask`. + +| Result Type in `taskRef` or `taskSpec` | Parameter Type of Consumer | Specification | +|----------------------------------------|----------------------------|-------------------------------------------------------| +| string | array | `$(tasks..results.[*])` | +| array | Not Supported | Not Supported | +| object | Not Supported | Not Supported | + +```yaml +apiVersion: tekton.dev/v1beta1 +kind: Pipeline +metadata: + name: platform-browser-tests +spec: + tasks: + - name: matrix-emitting-results + matrix: + params: + - name: platform + value: + - linux + - mac + - windows + - name: browser + value: + - chrome + - safari + - firefox + taskRef: + name: taskwithresults + kind: Task + - name: task-consuming-results + taskRef: + name: echoarrayurl + kind: Task + params: + - name: url + value: $(tasks.matrix-emitting-results.results.report-url[*]) + ... +``` +See the full example [pr-with-matrix-emitting-results] ## Retries @@ -851,4 +928,7 @@ status: [cel]: https://github.com/tektoncd/experimental/tree/1609827ea81d05c8d00f8933c5c9d6150cd36989/cel [pr-with-matrix]: ../examples/v1/pipelineruns/alpha/pipelinerun-with-matrix.yaml [pr-with-matrix-and-results]: ../examples/v1/pipelineruns/alpha/pipelinerun-with-matrix-and-results.yaml +[pr-with-matrix-context-variables]: ../examples/v1/pipelineruns/alpha/pipelinerun-with-matrix-context-variables.yaml +[pr-with-matrix-emitting-results]: ../examples/v1/pipelineruns/alpha/pipelinerun-with-matrix-emitting-results.yaml + [retries]: pipelines.md#using-the-retries-field diff --git a/examples/v1/pipelineruns/alpha/pipelinerun-with-matrix-context-variables.yaml b/examples/v1/pipelineruns/alpha/pipelinerun-with-matrix-context-variables.yaml new file mode 100644 index 00000000000..40a66ad7ea5 --- /dev/null +++ b/examples/v1/pipelineruns/alpha/pipelinerun-with-matrix-context-variables.yaml @@ -0,0 +1,95 @@ +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: echomatrixlength +spec: + params: + - name: matrixlength + type: string + steps: + - name: echo + image: alpine + script: echo $(params.matrixlength) +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: echomatrixresultslength +spec: + params: + - name: matrixresultslength + type: string + steps: + - name: echo + image: alpine + script: echo $(params.matrixresultslength) +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: taskwithresults +spec: + params: + - name: IMAGE + - name: DIGEST + default: "" + results: + - name: IMAGE-DIGEST + steps: + - name: produce-results + image: bash:latest + script: | + #!/usr/bin/env bash + echo "Building image for $(params.IMAGE)" + echo -n "$(params.DIGEST)" | sha256sum | tee $(results.IMAGE-DIGEST.path) +--- +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + generateName: matrix-emitting-results +spec: + serviceAccountName: "default" + pipelineSpec: + tasks: + - name: matrix-emitting-results + matrix: + include: + - name: build-1 + params: + - name: IMAGE + value: image-1 + - name: DOCKERFILE + value: path/to/Dockerfile1 + - name: build-2 + params: + - name: IMAGE + value: image-2 + - name: DOCKERFILE + value: path/to/Dockerfile2 + - name: build-3 + params: + - name: IMAGE + value: image-3 + - name: DOCKERFILE + value: path/to/Dockerfile3 + taskRef: + name: taskwithresults + kind: Task + - name: matrixed-echo-length + runAfter: + - matrix-emitting-results + params: + - name: matrixlength + value: $(tasks.matrix-emitting-results.matrix.length) + taskRef: + name: echomatrixlength + kind: Task + - name: matrixed-echo-results-length + runAfter: + - matrix-emitting-results + params: + - name: matrixresultslength + value: $(tasks.matrix-emitting-results.matrix.IMAGE-DIGEST.length) + taskRef: + name: echomatrixresultslength + kind: Task diff --git a/examples/v1/pipelineruns/alpha/pipelinerun-with-matrix-emitting-results.yaml b/examples/v1/pipelineruns/alpha/pipelinerun-with-matrix-emitting-results.yaml new file mode 100644 index 00000000000..091e89c8bc9 --- /dev/null +++ b/examples/v1/pipelineruns/alpha/pipelinerun-with-matrix-emitting-results.yaml @@ -0,0 +1,90 @@ +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: echostringurl +spec: + params: + - name: url + type: string + steps: + - name: echo + image: alpine + script: | + echo "$(params.url)" +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: echoarrayurl +spec: + params: + - name: url + type: array + steps: + - name: use-environments + image: bash:latest + args: ["$(params.url[*])"] + script: | + for arg in "$@"; do + echo "URL: $arg" + done +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: taskwithresults +spec: + params: + - name: platform + default: "" + - name: browser + default: "" + results: + - name: report-url + type: string + steps: + - name: produce-report-url + image: alpine + script: | + echo "Running tests on $(params.platform)-$(params.browser)" + echo -n "https://api.example/get-report/$(params.platform)-$(params.browser)" | tee $(results.report-url.path) +--- +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + generateName: platforms-with-results +spec: + serviceAccountName: "default" + pipelineSpec: + tasks: + - name: matrix-emitting-results + matrix: + params: + - name: platform + value: + - linux + - mac + - windows + - name: browser + value: + - chrome + - safari + - firefox + taskRef: + name: taskwithresults + kind: Task + - name: task-consuming-results + taskRef: + name: echoarrayurl + kind: Task + params: + - name: url + value: $(tasks.matrix-emitting-results.results.report-url[*]) + - name: matrix-consuming-results + taskRef: + name: echostringurl + kind: Task + matrix: + params: + - name: url + value: $(tasks.matrix-emitting-results.results.report-url[*]) diff --git a/pkg/apis/pipeline/v1/pipeline_validation.go b/pkg/apis/pipeline/v1/pipeline_validation.go index 71b002d9bcd..bba2930ec36 100644 --- a/pkg/apis/pipeline/v1/pipeline_validation.go +++ b/pkg/apis/pipeline/v1/pipeline_validation.go @@ -19,6 +19,7 @@ package v1 import ( "context" "fmt" + "regexp" "strings" "github.com/tektoncd/pipeline/pkg/apis/config" @@ -92,7 +93,7 @@ func (ps *PipelineSpec) Validate(ctx context.Context) (errs *apis.FieldError) { errs = errs.Also(validateWhenExpressions(ps.Tasks, ps.Finally)) errs = errs.Also(validateMatrix(ctx, ps.Tasks).ViaField("tasks")) errs = errs.Also(validateMatrix(ctx, ps.Finally).ViaField("finally")) - errs = errs.Also(validateResultsFromMatrixedPipelineTasksNotConsumed(ps.Tasks, ps.Finally)) + errs = errs.Also(validateResultsFromMatrixedPipelineTasksConsumed(ps.Tasks, ps.Finally)) return errs } @@ -260,15 +261,6 @@ func (pt PipelineTask) validateEmbeddedOrType() (errs *apis.FieldError) { return } -func (pt *PipelineTask) validateResultsFromMatrixedPipelineTasksNotConsumed(matrixedPipelineTasks sets.String) (errs *apis.FieldError) { - for _, ref := range PipelineTaskResultRefs(pt) { - if matrixedPipelineTasks.Has(ref.PipelineTask) { - errs = errs.Also(apis.ErrInvalidValue(fmt.Sprintf("consuming results from matrixed task %s is not allowed", ref.PipelineTask), "")) - } - } - return errs -} - func (pt *PipelineTask) validateWorkspaces(workspaceNames sets.String) (errs *apis.FieldError) { workspaceBindingNames := sets.NewString() for i, ws := range pt.Workspaces { @@ -771,18 +763,111 @@ func validateMatrix(ctx context.Context, tasks []PipelineTask) (errs *apis.Field return errs } -func validateResultsFromMatrixedPipelineTasksNotConsumed(tasks []PipelineTask, finally []PipelineTask) (errs *apis.FieldError) { - matrixedPipelineTasks := sets.String{} - for _, pt := range tasks { - if pt.IsMatrixed() { - matrixedPipelineTasks.Insert(pt.Name) +func createTaskMapping(tasks []PipelineTask) (taskMap map[string]PipelineTask) { + taskMapping := make(map[string]PipelineTask) + for _, task := range tasks { + taskMapping[task.Name] = task + } + return taskMapping +} + +// findAndValidateResultRefs checks that any result references to Matrixed PipelineTasks if consumed +// by another PieplineTask that the entire array of results produced by a matrix is consumed in aggregate +// since consuming a singular result produced by a matrix is currently not supported +func findAndValidateResultRefs(tasks []PipelineTask, taskMapping map[string]PipelineTask) (resultRefs []*ResultRef, errs *apis.FieldError) { + for _, t := range tasks { + for _, p := range t.Params { + if expressions, ok := p.GetVarSubstitutionExpressions(); ok { + if LooksLikeContainsResultRefs(expressions) { + errs := errs.Also(validateMatrixedPipelineTaskConsumed(expressions, taskMapping)) + if errs != nil { + return nil, errs + } + resultRefs = NewResultRefs(expressions) + } + } } } - for idx, pt := range tasks { - errs = errs.Also(pt.validateResultsFromMatrixedPipelineTasksNotConsumed(matrixedPipelineTasks).ViaFieldIndex("tasks", idx)) + return resultRefs, errs +} + +// validateMatrixedPipelineTaskConsumed checks that any Matrixed Pipeline Task that the is becing consumed is consumed in +// aggregate [*] since consuming a singular result produced by a matrix is currently not supported +func validateMatrixedPipelineTaskConsumed(expressions []string, taskMapping map[string]PipelineTask) (errs *apis.FieldError) { + for _, expression := range expressions { + subExpressions := strings.Split(expression, ".") + pipelineTask := subExpressions[1] + taskConsumed := taskMapping[pipelineTask] + if taskConsumed.IsMatrixed() { + regex := regexp.MustCompile("\\[(.*?)\\]") + matches := regex.FindAllStringSubmatch(expression, -1) + if len(matches) > 0 && len(matches[0]) > 1 && matches[0][1] != "*" { + errs = errs.Also(apis.ErrGeneric(fmt.Sprintf("A matrixed pipelineTask can only be consumed in aggregate using [*] notation, but is currently being indexed %s", expression))) + } + } + } + return errs +} + +// validateResultsFromMatrixedPipelineTasksConsumed checks that any Matrixed Pipeline Task that the is becing consumed is consumed in +// aggregate [*] since consuming a singular result produced by a matrix is currently not supported. +// It also validates that a matrix emitting results can only emit results with the underlying type string +// if those results are being consumed by another PipelineTask. +func validateResultsFromMatrixedPipelineTasksConsumed(tasks []PipelineTask, finally []PipelineTask) (errs *apis.FieldError) { + errs = errs.Also(validateTaskResultsFromMatrixedPipelineTasksConsumed(tasks)) + errs = errs.Also(validateFinallyResultsFromMatrixedPipelineTasksConsumed(finally)) + return errs +} + +func validateTaskResultsFromMatrixedPipelineTasksConsumed(tasks []PipelineTask) (errs *apis.FieldError) { + taskMapping := createTaskMapping(tasks) + resultRefs, errs := findAndValidateResultRefs(tasks, taskMapping) + if errs != nil { + return errs + } + + errs = errs.Also(validateMatrixEmittingStringResults(resultRefs, taskMapping)) + return errs +} + +func validateFinallyResultsFromMatrixedPipelineTasksConsumed(finally []PipelineTask) (errs *apis.FieldError) { + taskMapping := createTaskMapping(finally) + resultRefs, errs := findAndValidateResultRefs(finally, taskMapping) + if errs != nil { + return errs } - for idx, pt := range finally { - errs = errs.Also(pt.validateResultsFromMatrixedPipelineTasksNotConsumed(matrixedPipelineTasks).ViaFieldIndex("finally", idx)) + + errs = errs.Also(validateMatrixEmittingStringResults(resultRefs, taskMapping)) + return errs +} + +// validateMatrixEmittingStringResults checks a matrix emitting results can only emit results with the underlying type string +// if those results are being consumed by another PipelineTask. +func validateMatrixEmittingStringResults(resultRefs []*ResultRef, taskMapping map[string]PipelineTask) (errs *apis.FieldError) { + for _, resultRef := range resultRefs { + task := taskMapping[resultRef.PipelineTask] + resultName := resultRef.Result + if task.TaskRef != nil { + referencedTaskName := task.TaskRef.Name + referencedTask := taskMapping[referencedTaskName] + if referencedTask.TaskSpec != nil { + for _, result := range referencedTask.TaskSpec.Results { + if result.Name == resultName { + if result.Type != ResultsTypeString { + errs = errs.Also(apis.ErrGeneric(fmt.Sprintf("Matrixed PipelineTasks emitting results must have an underlying type string, but result \"%s\" has type \"%s\" in pipelineTask \"%s\"", resultName, string(result.Type), task.Name))) + } + } + } + } + } else if task.TaskSpec != nil { + for _, result := range task.TaskSpec.Results { + if result.Name == resultName { + if result.Type != ResultsTypeString { + errs = errs.Also(apis.ErrGeneric(fmt.Sprintf("Matrixed PipelineTasks emitting results must have an underlying type string, but result \"%s\" has type \"%s\" in pipelineTask \"%s\"", resultName, string(result.Type), task.Name))) + } + } + } + } } return errs } diff --git a/pkg/apis/pipeline/v1/pipeline_validation_test.go b/pkg/apis/pipeline/v1/pipeline_validation_test.go index c80b51a4b03..912b641acad 100644 --- a/pkg/apis/pipeline/v1/pipeline_validation_test.go +++ b/pkg/apis/pipeline/v1/pipeline_validation_test.go @@ -3575,6 +3575,7 @@ func Test_validateMatrix(t *testing.T) { tests := []struct { name string tasks []PipelineTask + finally []PipelineTask wantErrs *apis.FieldError }{{ name: "parameter in both matrix and params", @@ -3630,35 +3631,7 @@ func Test_validateMatrix(t *testing.T) { Name: "a-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.foo-task.results.a-task-results[*])"}}, }}}, }}, - }} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - featureFlags, _ := config.NewFeatureFlagsFromMap(map[string]string{ - "enable-api-fields": "alpha", - }) - defaults := &config.Defaults{ - DefaultMaxMatrixCombinationsCount: 4, - } - cfg := &config.Config{ - FeatureFlags: featureFlags, - Defaults: defaults, - } - - ctx := config.ToContext(context.Background(), cfg) - if d := cmp.Diff(tt.wantErrs.Error(), validateMatrix(ctx, tt.tasks).Error()); d != "" { - t.Errorf("validateMatrix() errors diff %s", diff.PrintWantGot(d)) - } - }) - } -} - -func Test_validateResultsFromMatrixedPipelineTasksNotConsumed(t *testing.T) { - tests := []struct { - name string - tasks []PipelineTask - finally []PipelineTask - wantErrs *apis.FieldError - }{{ + }, { name: "results from matrixed task consumed in tasks through parameters", tasks: PipelineTaskList{{ Name: "a-task", @@ -3674,10 +3647,6 @@ func Test_validateResultsFromMatrixedPipelineTasksNotConsumed(t *testing.T) { Name: "b-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.a-task.results.a-result)"}}, }}, }}, - wantErrs: &apis.FieldError{ - Message: "invalid value: consuming results from matrixed task a-task is not allowed", - Paths: []string{"tasks[1]"}, - }, }, { name: "results from matrixed task consumed in finally through parameters", tasks: PipelineTaskList{{ @@ -3695,10 +3664,6 @@ func Test_validateResultsFromMatrixedPipelineTasksNotConsumed(t *testing.T) { Name: "b-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.a-task.results.a-result)"}}, }}, }}, - wantErrs: &apis.FieldError{ - Message: "invalid value: consuming results from matrixed task a-task is not allowed", - Paths: []string{"finally[0]"}, - }, }, { name: "results from matrixed task consumed in tasks and finally through parameters", tasks: PipelineTaskList{{ @@ -3722,10 +3687,6 @@ func Test_validateResultsFromMatrixedPipelineTasksNotConsumed(t *testing.T) { Name: "b-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.a-task.results.a-result)"}}, }}, }}, - wantErrs: &apis.FieldError{ - Message: "invalid value: consuming results from matrixed task a-task is not allowed", - Paths: []string{"tasks[1]", "finally[0]"}, - }, }, { name: "results from matrixed task consumed in tasks through when expressions", tasks: PipelineTaskList{{ @@ -3744,10 +3705,6 @@ func Test_validateResultsFromMatrixedPipelineTasksNotConsumed(t *testing.T) { Values: []string{"$(tasks.a-task.results.a-result)"}, }}, }}, - wantErrs: &apis.FieldError{ - Message: "invalid value: consuming results from matrixed task a-task is not allowed", - Paths: []string{"tasks[1]"}, - }, }, { name: "results from matrixed task consumed in finally through when expressions", tasks: PipelineTaskList{{ @@ -3767,10 +3724,6 @@ func Test_validateResultsFromMatrixedPipelineTasksNotConsumed(t *testing.T) { Values: []string{"foo", "bar"}, }}, }}, - wantErrs: &apis.FieldError{ - Message: "invalid value: consuming results from matrixed task a-task is not allowed", - Paths: []string{"finally[0]"}, - }, }, { name: "results from matrixed task consumed in tasks and finally through when expressions", tasks: PipelineTaskList{{ @@ -3798,15 +3751,23 @@ func Test_validateResultsFromMatrixedPipelineTasksNotConsumed(t *testing.T) { Values: []string{"$(tasks.a-task.results.a-result)"}, }}, }}, - wantErrs: &apis.FieldError{ - Message: "invalid value: consuming results from matrixed task a-task is not allowed", - Paths: []string{"tasks[1]", "finally[0]"}, - }, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if d := cmp.Diff(tt.wantErrs.Error(), validateResultsFromMatrixedPipelineTasksNotConsumed(tt.tasks, tt.finally).Error()); d != "" { - t.Errorf("validateResultsFromMatrixedPipelineTasksNotConsumed() errors diff %s", diff.PrintWantGot(d)) + featureFlags, _ := config.NewFeatureFlagsFromMap(map[string]string{ + "enable-api-fields": "alpha", + }) + defaults := &config.Defaults{ + DefaultMaxMatrixCombinationsCount: 4, + } + cfg := &config.Config{ + FeatureFlags: featureFlags, + Defaults: defaults, + } + + ctx := config.ToContext(context.Background(), cfg) + if d := cmp.Diff(tt.wantErrs.Error(), validateMatrix(ctx, tt.tasks).Error()); d != "" { + t.Errorf("validateMatrix() errors diff %s", diff.PrintWantGot(d)) } }) } @@ -4172,3 +4133,259 @@ func TestGetIndexingReferencesToArrayParams(t *testing.T) { }) } } + +func Test_validateTaskResultsFromMatrixedPipelineTaskConsumed(t *testing.T) { + + tasks := PipelineTaskList{{ + Name: "matrix-emitting-results", + TaskRef: &TaskRef{Name: "taskwithresult"}, + Matrix: &Matrix{ + Params: Params{{ + Name: "platform", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"linux", "mac", "windows"}}, + }, { + Name: "browser", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"chrome", "safari", "firefox"}}, + }}}, + }, { + Name: "echoarrayurl", + TaskSpec: &EmbeddedTask{TaskSpec: TaskSpec{ + Params: ParamSpecs{{ + Name: "url", Type: "array", + }}, + Steps: []Step{{ + Name: "use-environments", + Image: "bash:latest", + Args: []string{"$(params.url[*])"}, + Script: `for arg in "$@"; do + echo "URL: $arg" + done`, + }}, + }}, + }} + + tests := []struct { + name string + tasks []PipelineTask + finally []PipelineTask + wantErrs *apis.FieldError + }{{ + name: "valid matrix emitting string results consumed in aggregate by another pipelineTask ", + finally: PipelineTaskList{{ + Name: "matrix-emitting-results", + TaskRef: &TaskRef{Name: "taskwithresult"}, + Matrix: &Matrix{ + Params: Params{{ + Name: "platform", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"linux", "mac", "windows"}}, + }, { + Name: "browser", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"chrome", "safari", "firefox"}}, + }}}, + }, { + Name: "echoarrayurl", + TaskSpec: &EmbeddedTask{TaskSpec: TaskSpec{ + Params: ParamSpecs{{ + Name: "url", Type: "array", + }}, + Steps: []Step{{ + Name: "use-environments", + Image: "bash:latest", + Args: []string{"$(params.url[*])"}, + Script: `for arg in "$@"; do + echo "URL: $arg" + done`, + }}, + }}, + }, { + Name: "taskwithresult", + TaskSpec: &EmbeddedTask{TaskSpec: TaskSpec{ + Params: ParamSpecs{{ + Name: "platform", + }, { + Name: "browser"}}, + Results: []TaskResult{{ + Name: "report-url", + Type: ResultsTypeString, + }}, + Steps: []Step{{ + Name: "produce-report-url", + Image: "alpine", + Script: ` | + echo -n "https://api.example/get-report/$(params.platform)-$(params.browser)" | tee $(results.report-url.path)`}}, + }}, + }, { + Name: "task-consuming-results", + TaskRef: &TaskRef{Name: "echoarrayurl"}, + Params: Params{{ + Name: "b-param", Value: ParamValue{Type: ParamTypeString, StringVal: "$(tasks.matrix-emitting-results.results.report-url[*])"}, + }}, + }}, + }, { + name: "valid matrix emitting string results consumed in aggregate by another pipelineTask (embedded taskSpec)", + tasks: PipelineTaskList{{ + Name: "taskwithresult", + TaskSpec: &EmbeddedTask{TaskSpec: TaskSpec{ + Params: ParamSpecs{{ + Name: "platform", + }, { + Name: "browser"}}, + Results: []TaskResult{{ + Name: "report-url", + Type: ResultsTypeString, + }}, + Steps: []Step{{ + Name: "produce-report-url", + Image: "alpine", + Script: ` | + echo -n "https://api.example/get-report/$(params.platform)-$(params.browser)" | tee $(results.report-url.path)`}}, + }}, + }, { + Name: "task-consuming-results", + TaskSpec: &EmbeddedTask{TaskSpec: TaskSpec{ + Params: ParamSpecs{{ + Name: "url", Type: "array", + }}, + Steps: []Step{{ + Name: "use-environments", + Image: "bash:latest", + Args: []string{"$(params.url[*])"}, + Script: `for arg in "$@"; do + echo "URL: $arg" + done`, + }}, + }}, + Params: Params{{ + Name: "b-param", Value: ParamValue{Type: ParamTypeString, StringVal: "$(tasks.matrix-emitting-results.results.report-url[*])"}, + }}, + }}, + }, { + name: "invalid matrix emitting stings results consumed using array indexing by another pipelineTask", + tasks: PipelineTaskList{{ + Name: "taskwithresult", + TaskSpec: &EmbeddedTask{TaskSpec: TaskSpec{ + Params: ParamSpecs{{ + Name: "platform", + }, { + Name: "browser"}}, + Results: []TaskResult{{ + Name: "report-url", + Type: ResultsTypeString, + }}, + Steps: []Step{{ + Name: "produce-report-url", + Image: "alpine", + Script: ` | + echo -n "https://api.example/get-report/$(params.platform)-$(params.browser)" | tee $(results.report-url.path)`}}, + }}, + }, { + Name: "task-consuming-results", + TaskRef: &TaskRef{Name: "echoarrayurl"}, + Params: Params{{ + Name: "b-param", Value: ParamValue{Type: ParamTypeString, StringVal: "$(tasks.matrix-emitting-results.results.report-url[0])"}, + }}, + }}, + wantErrs: apis.ErrGeneric("A matrixed pipelineTask can only be consumed in aggregate using [*] notation, but is currently being indexed tasks.matrix-emitting-results.results.report-url[0]"), + }, { + name: "invalid matrix emitting array results consumed in aggregate by another pipelineTask ", + tasks: PipelineTaskList{{ + Name: "taskwithresult", + TaskSpec: &EmbeddedTask{TaskSpec: TaskSpec{ + Params: ParamSpecs{{ + Name: "platform", + }, { + Name: "browser"}}, + Results: []TaskResult{{ + Name: "array-result", + Type: ResultsTypeArray, + }}, + Steps: []Step{{ + Name: "produce-array-result", + Image: "alpine", + Script: ` | + echo -n "[\"${params.platform}\",\"${params.browser}\"]" | tee $(results.array-result.path)`}}, + }}, + }, { + Name: "task-consuming-results", + TaskRef: &TaskRef{Name: "echoarrayurl"}, + Params: Params{{ + Name: "b-param", Value: ParamValue{Type: ParamTypeString, StringVal: "$(tasks.matrix-emitting-results.results.array-result[*])"}, + }}, + }}, + wantErrs: apis.ErrGeneric("Matrixed PipelineTasks emitting results must have an underlying type string, but result \"array-result\" has type \"array\" in pipelineTask \"matrix-emitting-results\""), + }, { + name: "invalid matrix emitting stings results consumed using array indexing by another pipelineTask (embedded TaskSpec)", + tasks: PipelineTaskList{{ + Name: "taskwithresult", + TaskSpec: &EmbeddedTask{TaskSpec: TaskSpec{ + Params: ParamSpecs{{ + Name: "platform", + }, { + Name: "browser"}}, + Results: []TaskResult{{ + Name: "report-url", + Type: ResultsTypeString, + }}, + Steps: []Step{{ + Name: "produce-report-url", + Image: "alpine", + Script: ` | + echo -n "https://api.example/get-report/$(params.platform)-$(params.browser)" | tee $(results.report-url.path)`}}, + }}, + }, { + Name: "task-consuming-results", + TaskSpec: &EmbeddedTask{TaskSpec: TaskSpec{ + Params: ParamSpecs{{ + Name: "platform", + }, { + Name: "browser"}}, + Results: []TaskResult{{ + Name: "report-url", + Type: ResultsTypeString, + }}, + Steps: []Step{{ + Name: "produce-report-url", + Image: "alpine", + Script: ` | + echo -n "https://api.example/get-report/$(params.platform)-$(params.browser)" | tee $(results.report-url.path)`}}, + }}, + Params: Params{{ + Name: "b-param", Value: ParamValue{Type: ParamTypeString, StringVal: "$(tasks.matrix-emitting-results.results.report-url[0])"}, + }}, + }}, + wantErrs: apis.ErrGeneric("A matrixed pipelineTask can only be consumed in aggregate using [*] notation, but is currently being indexed tasks.matrix-emitting-results.results.report-url[0]"), + }, { + name: "invalid matrix emitting array results consumed in aggregate by another pipelineTask ", + tasks: PipelineTaskList{{ + Name: "taskwithresult", + TaskSpec: &EmbeddedTask{TaskSpec: TaskSpec{ + Params: ParamSpecs{{ + Name: "platform", + }, { + Name: "browser"}}, + Results: []TaskResult{{ + Name: "array-result", + Type: ResultsTypeArray, + }}, + Steps: []Step{{ + Name: "produce-array-result", + Image: "alpine", + Script: ` | + echo -n "[\"${params.platform}\",\"${params.browser}\"]" | tee $(results.array-result.path)`}}, + }}, + }, { + Name: "task-consuming-results", + TaskRef: &TaskRef{Name: "echoarrayurl"}, + Params: Params{{ + Name: "b-param", Value: ParamValue{Type: ParamTypeString, StringVal: "$(tasks.matrix-emitting-results.results.array-result[*])"}, + }}, + }}, + wantErrs: apis.ErrGeneric("Matrixed PipelineTasks emitting results must have an underlying type string, but result \"array-result\" has type \"array\" in pipelineTask \"matrix-emitting-results\""), + }} + for _, tt := range tests { + var tasksForTest []PipelineTask + t.Run(tt.name, func(t *testing.T) { + tasksForTest := append(tasksForTest, tasks...) + tasksForTest = append(tasksForTest, tt.tasks...) + if d := cmp.Diff(tt.wantErrs.Error(), validateTaskResultsFromMatrixedPipelineTasksConsumed(tasksForTest).Error()); d != "" { + t.Errorf("validateTaskResultsFromMatrixedPipelineTasksConsumed Consumed() errors diff %s", diff.PrintWantGot(d)) + } + }) + } +} diff --git a/pkg/apis/pipeline/v1beta1/pipeline_validation.go b/pkg/apis/pipeline/v1beta1/pipeline_validation.go index 7529c273b4f..4adffcd2030 100644 --- a/pkg/apis/pipeline/v1beta1/pipeline_validation.go +++ b/pkg/apis/pipeline/v1beta1/pipeline_validation.go @@ -19,6 +19,7 @@ package v1beta1 import ( "context" "fmt" + "regexp" "strings" "github.com/google/go-containerregistry/pkg/name" @@ -90,7 +91,7 @@ func (ps *PipelineSpec) Validate(ctx context.Context) (errs *apis.FieldError) { errs = errs.Also(validateWhenExpressions(ps.Tasks, ps.Finally)) errs = errs.Also(validateMatrix(ctx, ps.Tasks).ViaField("tasks")) errs = errs.Also(validateMatrix(ctx, ps.Finally).ViaField("finally")) - errs = errs.Also(validateResultsFromMatrixedPipelineTasksNotConsumed(ps.Tasks, ps.Finally)) + errs = errs.Also(validateResultsFromMatrixedPipelineTasksConsumed(ps.Tasks, ps.Finally)) return errs } @@ -212,15 +213,6 @@ func (pt PipelineTask) validateEmbeddedOrType() (errs *apis.FieldError) { return } -func (pt *PipelineTask) validateResultsFromMatrixedPipelineTasksNotConsumed(matrixedPipelineTasks sets.String) (errs *apis.FieldError) { - for _, ref := range PipelineTaskResultRefs(pt) { - if matrixedPipelineTasks.Has(ref.PipelineTask) { - errs = errs.Also(apis.ErrInvalidValue(fmt.Sprintf("consuming results from matrixed task %s is not allowed", ref.PipelineTask), "")) - } - } - return errs -} - func (pt *PipelineTask) validateWorkspaces(workspaceNames sets.String) (errs *apis.FieldError) { workspaceBindingNames := sets.NewString() for i, ws := range pt.Workspaces { @@ -734,18 +726,111 @@ func validateMatrix(ctx context.Context, tasks []PipelineTask) (errs *apis.Field return errs } -func validateResultsFromMatrixedPipelineTasksNotConsumed(tasks []PipelineTask, finally []PipelineTask) (errs *apis.FieldError) { - matrixedPipelineTasks := sets.String{} - for _, pt := range tasks { - if pt.IsMatrixed() { - matrixedPipelineTasks.Insert(pt.Name) +func createTaskMapping(tasks []PipelineTask) (taskMap map[string]PipelineTask) { + taskMapping := make(map[string]PipelineTask) + for _, task := range tasks { + taskMapping[task.Name] = task + } + return taskMapping +} + +// findAndValidateResultRefs checks that any result references to Matrixed PipelineTasks if consumed +// by another PieplineTask that the entire array of results produced by a matrix is consumed in aggregate +// since consuming a singular result produced by a matrix is currently not supported +func findAndValidateResultRefs(tasks []PipelineTask, taskMapping map[string]PipelineTask) (resultRefs []*ResultRef, errs *apis.FieldError) { + for _, t := range tasks { + for _, p := range t.Params { + if expressions, ok := GetVarSubstitutionExpressionsForParam(p); ok { + if LooksLikeContainsResultRefs(expressions) { + errs := errs.Also(validateMatrixedPipelineTaskConsumed(expressions, taskMapping)) + if errs != nil { + return nil, errs + } + resultRefs = NewResultRefs(expressions) + } + } } } - for idx, pt := range tasks { - errs = errs.Also(pt.validateResultsFromMatrixedPipelineTasksNotConsumed(matrixedPipelineTasks).ViaFieldIndex("tasks", idx)) + return resultRefs, errs +} + +// validateMatrixedPipelineTaskConsumed checks that any Matrixed Pipeline Task that the is becing consumed is consumed in +// aggregate [*] since consuming a singular result produced by a matrix is currently not supported +func validateMatrixedPipelineTaskConsumed(expressions []string, taskMapping map[string]PipelineTask) (errs *apis.FieldError) { + for _, expression := range expressions { + subExpressions := strings.Split(expression, ".") + pipelineTask := subExpressions[1] + taskConsumed := taskMapping[pipelineTask] + if taskConsumed.IsMatrixed() { + regex := regexp.MustCompile("\\[(.*?)\\]") + matches := regex.FindAllStringSubmatch(expression, -1) + if len(matches) > 0 && len(matches[0]) > 1 && matches[0][1] != "*" { + errs = errs.Also(apis.ErrGeneric(fmt.Sprintf("A matrixed pipelineTask can only be consumed in aggregate using [*] notation, but is currently being indexed %s", expression))) + } + } + } + return errs +} + +// validateResultsFromMatrixedPipelineTasksConsumed checks that any Matrixed Pipeline Task that the is becing consumed is consumed in +// aggregate [*] since consuming a singular result produced by a matrix is currently not supported. +// It also validates that a matrix emitting results can only emit results with the underlying type string +// if those results are being consumed by another PipelineTask. +func validateResultsFromMatrixedPipelineTasksConsumed(tasks []PipelineTask, finally []PipelineTask) (errs *apis.FieldError) { + errs = errs.Also(validateTaskResultsFromMatrixedPipelineTasksConsumed(tasks)) + errs = errs.Also(validateFinallyResultsFromMatrixedPipelineTasksConsumed(finally)) + return errs +} + +func validateTaskResultsFromMatrixedPipelineTasksConsumed(tasks []PipelineTask) (errs *apis.FieldError) { + taskMapping := createTaskMapping(tasks) + resultRefs, errs := findAndValidateResultRefs(tasks, taskMapping) + if errs != nil { + return errs + } + + errs = errs.Also(validateMatrixEmittingStringResults(resultRefs, taskMapping)) + return errs +} + +func validateFinallyResultsFromMatrixedPipelineTasksConsumed(finally []PipelineTask) (errs *apis.FieldError) { + taskMapping := createTaskMapping(finally) + resultRefs, errs := findAndValidateResultRefs(finally, taskMapping) + if errs != nil { + return errs } - for idx, pt := range finally { - errs = errs.Also(pt.validateResultsFromMatrixedPipelineTasksNotConsumed(matrixedPipelineTasks).ViaFieldIndex("finally", idx)) + + errs = errs.Also(validateMatrixEmittingStringResults(resultRefs, taskMapping)) + return errs +} + +// validateMatrixEmittingStringResults checks a matrix emitting results can only emit results with the underlying type string +// if those results are being consumed by another PipelineTask. +func validateMatrixEmittingStringResults(resultRefs []*ResultRef, taskMapping map[string]PipelineTask) (errs *apis.FieldError) { + for _, resultRef := range resultRefs { + task := taskMapping[resultRef.PipelineTask] + resultName := resultRef.Result + if task.TaskRef != nil { + referencedTaskName := task.TaskRef.Name + referencedTask := taskMapping[referencedTaskName] + if referencedTask.TaskSpec != nil { + for _, result := range referencedTask.TaskSpec.Results { + if result.Name == resultName { + if result.Type != ResultsTypeString { + errs = errs.Also(apis.ErrGeneric(fmt.Sprintf("Matrixed PipelineTasks emitting results must have an underlying type string, but result \"%s\" has type \"%s\" in pipelineTask \"%s\"", resultName, string(result.Type), task.Name))) + } + } + } + } + } else if task.TaskSpec != nil { + for _, result := range task.TaskSpec.Results { + if result.Name == resultName { + if result.Type != ResultsTypeString { + errs = errs.Also(apis.ErrGeneric(fmt.Sprintf("Matrixed PipelineTasks emitting results must have an underlying type string, but result \"%s\" has type \"%s\" in pipelineTask \"%s\"", resultName, string(result.Type), task.Name))) + } + } + } + } } return errs } diff --git a/pkg/apis/pipeline/v1beta1/pipeline_validation_test.go b/pkg/apis/pipeline/v1beta1/pipeline_validation_test.go index 1ebd0604678..d01e4269fa3 100644 --- a/pkg/apis/pipeline/v1beta1/pipeline_validation_test.go +++ b/pkg/apis/pipeline/v1beta1/pipeline_validation_test.go @@ -3618,6 +3618,7 @@ func Test_validateMatrix(t *testing.T) { tests := []struct { name string tasks []PipelineTask + finally []PipelineTask wantErrs *apis.FieldError }{{ name: "parameter in both matrix and params", @@ -3673,35 +3674,7 @@ func Test_validateMatrix(t *testing.T) { Name: "a-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.foo-task.results.a-task-results[*])"}}, }}}, }}, - }} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - featureFlags, _ := config.NewFeatureFlagsFromMap(map[string]string{ - "enable-api-fields": "alpha", - }) - defaults := &config.Defaults{ - DefaultMaxMatrixCombinationsCount: 4, - } - cfg := &config.Config{ - FeatureFlags: featureFlags, - Defaults: defaults, - } - - ctx := config.ToContext(context.Background(), cfg) - if d := cmp.Diff(tt.wantErrs.Error(), validateMatrix(ctx, tt.tasks).Error()); d != "" { - t.Errorf("validateMatrix() errors diff %s", diff.PrintWantGot(d)) - } - }) - } -} - -func Test_validateResultsFromMatrixedPipelineTasksNotConsumed(t *testing.T) { - tests := []struct { - name string - tasks []PipelineTask - finally []PipelineTask - wantErrs *apis.FieldError - }{{ + }, { name: "results from matrixed task consumed in tasks through parameters", tasks: PipelineTaskList{{ Name: "a-task", @@ -3717,10 +3690,6 @@ func Test_validateResultsFromMatrixedPipelineTasksNotConsumed(t *testing.T) { Name: "b-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.a-task.results.a-result)"}}, }}, }}, - wantErrs: &apis.FieldError{ - Message: "invalid value: consuming results from matrixed task a-task is not allowed", - Paths: []string{"tasks[1]"}, - }, }, { name: "results from matrixed task consumed in finally through parameters", tasks: PipelineTaskList{{ @@ -3738,10 +3707,6 @@ func Test_validateResultsFromMatrixedPipelineTasksNotConsumed(t *testing.T) { Name: "b-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.a-task.results.a-result)"}}, }}, }}, - wantErrs: &apis.FieldError{ - Message: "invalid value: consuming results from matrixed task a-task is not allowed", - Paths: []string{"finally[0]"}, - }, }, { name: "results from matrixed task consumed in tasks and finally through parameters", tasks: PipelineTaskList{{ @@ -3765,10 +3730,6 @@ func Test_validateResultsFromMatrixedPipelineTasksNotConsumed(t *testing.T) { Name: "b-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.a-task.results.a-result)"}}, }}, }}, - wantErrs: &apis.FieldError{ - Message: "invalid value: consuming results from matrixed task a-task is not allowed", - Paths: []string{"tasks[1]", "finally[0]"}, - }, }, { name: "results from matrixed task consumed in tasks through when expressions", tasks: PipelineTaskList{{ @@ -3787,10 +3748,6 @@ func Test_validateResultsFromMatrixedPipelineTasksNotConsumed(t *testing.T) { Values: []string{"$(tasks.a-task.results.a-result)"}, }}, }}, - wantErrs: &apis.FieldError{ - Message: "invalid value: consuming results from matrixed task a-task is not allowed", - Paths: []string{"tasks[1]"}, - }, }, { name: "results from matrixed task consumed in finally through when expressions", tasks: PipelineTaskList{{ @@ -3810,10 +3767,6 @@ func Test_validateResultsFromMatrixedPipelineTasksNotConsumed(t *testing.T) { Values: []string{"foo", "bar"}, }}, }}, - wantErrs: &apis.FieldError{ - Message: "invalid value: consuming results from matrixed task a-task is not allowed", - Paths: []string{"finally[0]"}, - }, }, { name: "results from matrixed task consumed in tasks and finally through when expressions", tasks: PipelineTaskList{{ @@ -3841,15 +3794,23 @@ func Test_validateResultsFromMatrixedPipelineTasksNotConsumed(t *testing.T) { Values: []string{"$(tasks.a-task.results.a-result)"}, }}, }}, - wantErrs: &apis.FieldError{ - Message: "invalid value: consuming results from matrixed task a-task is not allowed", - Paths: []string{"tasks[1]", "finally[0]"}, - }, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if d := cmp.Diff(tt.wantErrs.Error(), validateResultsFromMatrixedPipelineTasksNotConsumed(tt.tasks, tt.finally).Error()); d != "" { - t.Errorf("validateResultsFromMatrixedPipelineTasksNotConsumed() errors diff %s", diff.PrintWantGot(d)) + featureFlags, _ := config.NewFeatureFlagsFromMap(map[string]string{ + "enable-api-fields": "alpha", + }) + defaults := &config.Defaults{ + DefaultMaxMatrixCombinationsCount: 4, + } + cfg := &config.Config{ + FeatureFlags: featureFlags, + Defaults: defaults, + } + + ctx := config.ToContext(context.Background(), cfg) + if d := cmp.Diff(tt.wantErrs.Error(), validateMatrix(ctx, tt.tasks).Error()); d != "" { + t.Errorf("validateMatrix() errors diff %s", diff.PrintWantGot(d)) } }) } @@ -4031,3 +3992,259 @@ func TestGetIndexingReferencesToArrayParams(t *testing.T) { }) } } + +func Test_validateTaskResultsFromMatrixedPipelineTaskConsumed(t *testing.T) { + + tasks := PipelineTaskList{{ + Name: "matrix-emitting-results", + TaskRef: &TaskRef{Name: "taskwithresult"}, + Matrix: &Matrix{ + Params: Params{{ + Name: "platform", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"linux", "mac", "windows"}}, + }, { + Name: "browser", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"chrome", "safari", "firefox"}}, + }}}, + }, { + Name: "echoarrayurl", + TaskSpec: &EmbeddedTask{TaskSpec: TaskSpec{ + Params: ParamSpecs{{ + Name: "url", Type: "array", + }}, + Steps: []Step{{ + Name: "use-environments", + Image: "bash:latest", + Args: []string{"$(params.url[*])"}, + Script: `for arg in "$@"; do + echo "URL: $arg" + done`, + }}, + }}, + }} + + tests := []struct { + name string + tasks []PipelineTask + finally []PipelineTask + wantErrs *apis.FieldError + }{{ + name: "valid matrix emitting string results consumed in aggregate by another pipelineTask ", + finally: PipelineTaskList{{ + Name: "matrix-emitting-results", + TaskRef: &TaskRef{Name: "taskwithresult"}, + Matrix: &Matrix{ + Params: Params{{ + Name: "platform", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"linux", "mac", "windows"}}, + }, { + Name: "browser", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"chrome", "safari", "firefox"}}, + }}}, + }, { + Name: "echoarrayurl", + TaskSpec: &EmbeddedTask{TaskSpec: TaskSpec{ + Params: ParamSpecs{{ + Name: "url", Type: "array", + }}, + Steps: []Step{{ + Name: "use-environments", + Image: "bash:latest", + Args: []string{"$(params.url[*])"}, + Script: `for arg in "$@"; do + echo "URL: $arg" + done`, + }}, + }}, + }, { + Name: "taskwithresult", + TaskSpec: &EmbeddedTask{TaskSpec: TaskSpec{ + Params: ParamSpecs{{ + Name: "platform", + }, { + Name: "browser"}}, + Results: []TaskResult{{ + Name: "report-url", + Type: ResultsTypeString, + }}, + Steps: []Step{{ + Name: "produce-report-url", + Image: "alpine", + Script: ` | + echo -n "https://api.example/get-report/$(params.platform)-$(params.browser)" | tee $(results.report-url.path)`}}, + }}, + }, { + Name: "task-consuming-results", + TaskRef: &TaskRef{Name: "echoarrayurl"}, + Params: Params{{ + Name: "b-param", Value: ParamValue{Type: ParamTypeString, StringVal: "$(tasks.matrix-emitting-results.results.report-url[*])"}, + }}, + }}, + }, { + name: "valid matrix emitting string results consumed in aggregate by another pipelineTask (embedded taskSpec)", + tasks: PipelineTaskList{{ + Name: "taskwithresult", + TaskSpec: &EmbeddedTask{TaskSpec: TaskSpec{ + Params: ParamSpecs{{ + Name: "platform", + }, { + Name: "browser"}}, + Results: []TaskResult{{ + Name: "report-url", + Type: ResultsTypeString, + }}, + Steps: []Step{{ + Name: "produce-report-url", + Image: "alpine", + Script: ` | + echo -n "https://api.example/get-report/$(params.platform)-$(params.browser)" | tee $(results.report-url.path)`}}, + }}, + }, { + Name: "task-consuming-results", + TaskSpec: &EmbeddedTask{TaskSpec: TaskSpec{ + Params: ParamSpecs{{ + Name: "url", Type: "array", + }}, + Steps: []Step{{ + Name: "use-environments", + Image: "bash:latest", + Args: []string{"$(params.url[*])"}, + Script: `for arg in "$@"; do + echo "URL: $arg" + done`, + }}, + }}, + Params: Params{{ + Name: "b-param", Value: ParamValue{Type: ParamTypeString, StringVal: "$(tasks.matrix-emitting-results.results.report-url[*])"}, + }}, + }}, + }, { + name: "invalid matrix emitting stings results consumed using array indexing by another pipelineTask", + tasks: PipelineTaskList{{ + Name: "taskwithresult", + TaskSpec: &EmbeddedTask{TaskSpec: TaskSpec{ + Params: ParamSpecs{{ + Name: "platform", + }, { + Name: "browser"}}, + Results: []TaskResult{{ + Name: "report-url", + Type: ResultsTypeString, + }}, + Steps: []Step{{ + Name: "produce-report-url", + Image: "alpine", + Script: ` | + echo -n "https://api.example/get-report/$(params.platform)-$(params.browser)" | tee $(results.report-url.path)`}}, + }}, + }, { + Name: "task-consuming-results", + TaskRef: &TaskRef{Name: "echoarrayurl"}, + Params: Params{{ + Name: "b-param", Value: ParamValue{Type: ParamTypeString, StringVal: "$(tasks.matrix-emitting-results.results.report-url[0])"}, + }}, + }}, + wantErrs: apis.ErrGeneric("A matrixed pipelineTask can only be consumed in aggregate using [*] notation, but is currently being indexed tasks.matrix-emitting-results.results.report-url[0]"), + }, { + name: "invalid matrix emitting array results consumed in aggregate by another pipelineTask ", + tasks: PipelineTaskList{{ + Name: "taskwithresult", + TaskSpec: &EmbeddedTask{TaskSpec: TaskSpec{ + Params: ParamSpecs{{ + Name: "platform", + }, { + Name: "browser"}}, + Results: []TaskResult{{ + Name: "array-result", + Type: ResultsTypeArray, + }}, + Steps: []Step{{ + Name: "produce-array-result", + Image: "alpine", + Script: ` | + echo -n "[\"${params.platform}\",\"${params.browser}\"]" | tee $(results.array-result.path)`}}, + }}, + }, { + Name: "task-consuming-results", + TaskRef: &TaskRef{Name: "echoarrayurl"}, + Params: Params{{ + Name: "b-param", Value: ParamValue{Type: ParamTypeString, StringVal: "$(tasks.matrix-emitting-results.results.array-result[*])"}, + }}, + }}, + wantErrs: apis.ErrGeneric("Matrixed PipelineTasks emitting results must have an underlying type string, but result \"array-result\" has type \"array\" in pipelineTask \"matrix-emitting-results\""), + }, { + name: "invalid matrix emitting stings results consumed using array indexing by another pipelineTask (embedded TaskSpec)", + tasks: PipelineTaskList{{ + Name: "taskwithresult", + TaskSpec: &EmbeddedTask{TaskSpec: TaskSpec{ + Params: ParamSpecs{{ + Name: "platform", + }, { + Name: "browser"}}, + Results: []TaskResult{{ + Name: "report-url", + Type: ResultsTypeString, + }}, + Steps: []Step{{ + Name: "produce-report-url", + Image: "alpine", + Script: ` | + echo -n "https://api.example/get-report/$(params.platform)-$(params.browser)" | tee $(results.report-url.path)`}}, + }}, + }, { + Name: "task-consuming-results", + TaskSpec: &EmbeddedTask{TaskSpec: TaskSpec{ + Params: ParamSpecs{{ + Name: "platform", + }, { + Name: "browser"}}, + Results: []TaskResult{{ + Name: "report-url", + Type: ResultsTypeString, + }}, + Steps: []Step{{ + Name: "produce-report-url", + Image: "alpine", + Script: ` | + echo -n "https://api.example/get-report/$(params.platform)-$(params.browser)" | tee $(results.report-url.path)`}}, + }}, + Params: Params{{ + Name: "b-param", Value: ParamValue{Type: ParamTypeString, StringVal: "$(tasks.matrix-emitting-results.results.report-url[0])"}, + }}, + }}, + wantErrs: apis.ErrGeneric("A matrixed pipelineTask can only be consumed in aggregate using [*] notation, but is currently being indexed tasks.matrix-emitting-results.results.report-url[0]"), + }, { + name: "invalid matrix emitting array results consumed in aggregate by another pipelineTask ", + tasks: PipelineTaskList{{ + Name: "taskwithresult", + TaskSpec: &EmbeddedTask{TaskSpec: TaskSpec{ + Params: ParamSpecs{{ + Name: "platform", + }, { + Name: "browser"}}, + Results: []TaskResult{{ + Name: "array-result", + Type: ResultsTypeArray, + }}, + Steps: []Step{{ + Name: "produce-array-result", + Image: "alpine", + Script: ` | + echo -n "[\"${params.platform}\",\"${params.browser}\"]" | tee $(results.array-result.path)`}}, + }}, + }, { + Name: "task-consuming-results", + TaskRef: &TaskRef{Name: "echoarrayurl"}, + Params: Params{{ + Name: "b-param", Value: ParamValue{Type: ParamTypeString, StringVal: "$(tasks.matrix-emitting-results.results.array-result[*])"}, + }}, + }}, + wantErrs: apis.ErrGeneric("Matrixed PipelineTasks emitting results must have an underlying type string, but result \"array-result\" has type \"array\" in pipelineTask \"matrix-emitting-results\""), + }} + for _, tt := range tests { + var tasksForTest []PipelineTask + t.Run(tt.name, func(t *testing.T) { + tasksForTest := append(tasksForTest, tasks...) + tasksForTest = append(tasksForTest, tt.tasks...) + if d := cmp.Diff(tt.wantErrs.Error(), validateTaskResultsFromMatrixedPipelineTasksConsumed(tasksForTest).Error()); d != "" { + t.Errorf("validateTaskResultsFromMatrixedPipelineTasksConsumed Consumed() errors diff %s", diff.PrintWantGot(d)) + } + }) + } +} diff --git a/pkg/reconciler/pipelinerun/pipelinerun.go b/pkg/reconciler/pipelinerun/pipelinerun.go index eef81a2e16c..f4a81af92c7 100644 --- a/pkg/reconciler/pipelinerun/pipelinerun.go +++ b/pkg/reconciler/pipelinerun/pipelinerun.go @@ -835,7 +835,7 @@ func (c *Reconciler) createTaskRun(ctx context.Context, taskRunName string, para ctx, span := c.tracerProvider.Tracer(TracerName).Start(ctx, "createTaskRun") defer span.End() logger := logging.FromContext(ctx) - rpt.PipelineTask = resources.ApplyPipelineTaskContexts(rpt.PipelineTask) + rpt.PipelineTask = resources.ApplyPipelineTaskContexts(rpt.PipelineTask, pr.Status) taskRunSpec := pr.GetTaskRunSpec(rpt.PipelineTask.Name) params = append(params, rpt.PipelineTask.Params...) tr := &v1.TaskRun{ @@ -935,7 +935,7 @@ func (c *Reconciler) createCustomRun(ctx context.Context, runName string, params ctx, span := c.tracerProvider.Tracer(TracerName).Start(ctx, "createCustomRun") defer span.End() logger := logging.FromContext(ctx) - rpt.PipelineTask = resources.ApplyPipelineTaskContexts(rpt.PipelineTask) + rpt.PipelineTask = resources.ApplyPipelineTaskContexts(rpt.PipelineTask, pr.Status) taskRunSpec := pr.GetTaskRunSpec(rpt.PipelineTask.Name) params = append(params, rpt.PipelineTask.Params...) diff --git a/pkg/reconciler/pipelinerun/pipelinerun_test.go b/pkg/reconciler/pipelinerun/pipelinerun_test.go index ab35dc88702..e46a411b56d 100644 --- a/pkg/reconciler/pipelinerun/pipelinerun_test.go +++ b/pkg/reconciler/pipelinerun/pipelinerun_test.go @@ -1196,6 +1196,17 @@ status: kind: TaskRun name: test-pipeline-missing-results-task1 pipelineTaskName: task1 + provenance: + featureFlags: + RunningInEnvWithInjectedSidecars: true + EnableAPIFields: "beta" + EnforceNonfalsifiability: "none" + AwaitSidecarReadiness: true + VerificationNoMatchPolicy: "ignore" + EnableProvenanceInStatus: true + ResultExtractionMethod: "termination-message" + MaxResultSize: 4096 + Coschedule: "workspaces" `) d := test.Data{ PipelineRuns: prs, @@ -12340,6 +12351,1787 @@ spec: } } +func TestReconciler_PipelineTaskExplicitCombosWithResults(t *testing.T) { + names.TestingSeed() + task := parse.MustParseV1Task(t, ` +metadata: + name: mytask + namespace: foo +spec: + params: + - name: IMAGE + - name: DOCKERFILE + steps: + - name: echo + image: alpine + script: | + echo "$(params.IMAGE) and $(params.DOCKERFILE)" +`) + expectedTaskRuns := []*v1.TaskRun{ + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-include-0", "foo", + "pr", "p", "matrix-include", false), + ` +spec: + params: + - name: DOCKERFILE + value: path/to/Dockerfile1 + - name: IMAGE + value: image-1 + serviceAccountName: test-sa + taskRef: + name: mytask + kind: Task +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-include-1", "foo", + "pr", "p", "matrix-include", false), + ` +spec: + params: + - name: DOCKERFILE + value: path/to/Dockerfile2 + - name: IMAGE + value: image-2 + serviceAccountName: test-sa + taskRef: + name: mytask + kind: Task +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-include-2", "foo", + "pr", "p", "matrix-include", false), + ` +spec: + params: + - name: DOCKERFILE + value: path/to/Dockerfile3 + - name: IMAGE + value: image-3 + serviceAccountName: test-sa + taskRef: + name: mytask + kind: Task +`), + } + cms := []*corev1.ConfigMap{withEnabledAlphaAPIFields(newFeatureFlagsConfigMap())} + cms = append(cms, withMaxMatrixCombinationsCount(newDefaultsConfigMap(), 10)) + tests := []struct { + name string + memberOf string + p *v1.Pipeline + tr *v1.TaskRun + expectedPipelineRun *v1.PipelineRun + }{{ + name: "p-dag", + memberOf: "tasks", + p: parse.MustParseV1Pipeline(t, fmt.Sprintf(` +metadata: + name: %s + namespace: foo +spec: + tasks: + - name: matrix-include + taskRef: + name: mytask + matrix: + include: + - name: build-1 + params: + - name: IMAGE + value: image-1 + - name: DOCKERFILE + value: path/to/Dockerfile1 + - name: build-2 + params: + - name: IMAGE + value: image-2 + - name: DOCKERFILE + value: path/to/Dockerfile2 + - name: build-3 + params: + - name: IMAGE + value: image-3 + - name: DOCKERFILE + value: path/to/Dockerfile3 + steps: + - name: echo + image: alpine + script: | + echo -n "$(params.IMAGE)" | sha256sum | tee $(results.IMAGE-DIGEST.path) + results: + - name: IMAGE-DIGEST + type: string +`, "p-dag")), + expectedPipelineRun: parse.MustParseV1PipelineRun(t, ` +metadata: + name: pr + namespace: foo + annotations: {} + labels: + tekton.dev/pipeline: p-dag +spec: + taskRunTemplate: + serviceAccountName: test-sa + pipelineRef: + name: p-dag +status: + pipelineSpec: + tasks: + - name: matrix-include + taskRef: + name: mytask + kind: Task + matrix: + include: + - name: build-1 + params: + - name: IMAGE + value: image-1 + - name: DOCKERFILE + value: path/to/Dockerfile1 + - name: build-2 + params: + - name: IMAGE + value: image-2 + - name: DOCKERFILE + value: path/to/Dockerfile2 + - name: build-3 + params: + - name: IMAGE + value: image-3 + - name: DOCKERFILE + value: path/to/Dockerfile3 + conditions: + - type: Succeeded + status: "Unknown" + reason: "Running" + message: "Tasks Completed: 0 (Failed: 0, Cancelled 0), Incomplete: 1, Skipped: 0" + childReferences: + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-include-0 + pipelineTaskName: matrix-include + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-include-1 + pipelineTaskName: matrix-include + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-include-2 + pipelineTaskName: matrix-include +`), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pr := parse.MustParseV1PipelineRun(t, fmt.Sprintf(` +metadata: + name: pr + namespace: foo +spec: + taskRunTemplate: + serviceAccountName: test-sa + pipelineRef: + name: %s +`, tt.name)) + d := test.Data{ + PipelineRuns: []*v1.PipelineRun{pr}, + Pipelines: []*v1.Pipeline{tt.p}, + Tasks: []*v1.Task{task}, + ConfigMaps: cms, + } + if tt.tr != nil { + d.TaskRuns = []*v1.TaskRun{tt.tr} + } + prt := newPipelineRunTest(t, d) + defer prt.Cancel() + _, clients := prt.reconcileRun("foo", "pr", []string{}, false) + taskRuns, err := clients.Pipeline.TektonV1().TaskRuns("foo").List(prt.TestAssets.Ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("tekton.dev/pipelineRun=pr,tekton.dev/pipeline=%s,tekton.dev/pipelineTask=matrix-include", tt.name), + Limit: 1, + }) + if err != nil { + t.Fatalf("Failure to list TaskRun's %s", err) + } + if len(taskRuns.Items) != 3 { + t.Fatalf("Expected 3 TaskRuns got %d", len(taskRuns.Items)) + } + for i := range taskRuns.Items { + expectedTaskRun := expectedTaskRuns[i] + expectedTaskRun.Labels["tekton.dev/pipeline"] = tt.name + expectedTaskRun.Labels["tekton.dev/memberOf"] = tt.memberOf + if d := cmp.Diff(expectedTaskRun, &taskRuns.Items[i], ignoreResourceVersion, ignoreTypeMeta); d != "" { + t.Errorf("expected to see TaskRun %v created. Diff %s", expectedTaskRuns[i].Name, diff.PrintWantGot(d)) + } + } + pipelineRun, err := clients.Pipeline.TektonV1().PipelineRuns("foo").Get(prt.TestAssets.Ctx, "pr", metav1.GetOptions{}) + if err != nil { + t.Fatalf("Got an error getting reconciled run out of fake client: %s", err) + } + if d := cmp.Diff(tt.expectedPipelineRun, pipelineRun, ignoreResourceVersion, ignoreTypeMeta, ignoreLastTransitionTime, ignoreStartTime, ignoreFinallyStartTime, ignoreProvenance, cmpopts.EquateEmpty()); d != "" { + t.Errorf("expected PipelineRun was not created. Diff %s", diff.PrintWantGot(d)) + } + }) + } +} + +func TestReconciler_PipelineTaskMatrixConsumingResults(t *testing.T) { + names.TestingSeed() + task1 := parse.MustParseV1Task(t, ` +metadata: + name: arraytask + namespace: foo +spec: + params: + - name: DIGEST + type: array + steps: + - name: use-environments + image: bash:latest + args: ["$(params.DIGEST[*])"] + script: | + for arg in "$@"; do + echo "Arg: $arg" + done +`) + task2 := parse.MustParseV1Task(t, ` +metadata: + name: stringtask + namespace: foo +spec: + params: + - name: DIGEST + type: string + steps: + - name: echo + image: alpine + script: | + echo "$(params.DIGEST)" +`) + task3 := parse.MustParseV1Task(t, ` +metadata: + name: echomatrixlength + namespace: foo +spec: + params: + - name: matrixlength + type: string + steps: + - name: echo + image: alpine + script: echo $(params.matrixlength) +`) + task4 := parse.MustParseV1Task(t, ` +metadata: + name: echomatrixresultslength + namespace: foo +spec: + params: + - name: matrixresultslength + type: string + steps: + - name: echo + image: alpine + script: echo $(params.matrixresultslength) +`) + + taskwithresults := parse.MustParseV1Task(t, ` +metadata: + name: taskwithresults + namespace: foo +spec: + params: + - name: IMAGE + - name: DIGEST + default: "" + results: + - name: IMAGE-DIGEST + steps: + - name: produce-results + image: bash:latest + script: | + #!/usr/bin/env bash + echo "Building image for $(params.IMAGE)" + echo -n "$(params.DIGEST)" | sha256sum | tee $(results.IMAGE-DIGEST.path) +`) + taskRuns := []*v1.TaskRun{ + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-emitting-results-0", "foo", + "pr", "p", "matrix-emitting-results", false), + ` +spec: + params: + - name: DOCKERFILE + value: path/to/Dockerfile1 + - name: IMAGE + value: image-1 + serviceAccountName: test-sa + taskRef: + name: arraytask + kind: Task +status: + conditions: + - type: Succeeded + status: "True" + reason: Succeeded + message: All Tasks have completed executing + results: + - name: IMAGE-DIGEST + value: 0cf457e24a479f02fd4d34540389f720f0807dcff92a7562108165b2637ea82f + - name: IMAGE-NAME + value: image-1 +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-emitting-results-1", "foo", + "pr", "p", "matrix-emitting-results", false), + ` +spec: + params: + - name: DOCKERFILE + value: path/to/Dockerfile2 + - name: IMAGE + value: image-2 + serviceAccountName: test-sa + taskRef: + name: arraytask + kind: Task +status: + conditions: + - type: Succeeded + status: "True" + reason: Succeeded + message: All Tasks have completed executing + results: + - name: IMAGE-DIGEST + value: 5a0717cb6596468ea1dffa86011f9b0f497348d80421835b51799f9aeb455642 + - name: IMAGE-NAME + value: image-2 +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-emitting-results-2", "foo", + "pr", "p", "matrix-emitting-results", false), + ` +spec: + params: + - name: DOCKERFILE + value: path/to/Dockerfile3 + - name: IMAGE + value: image-3 + serviceAccountName: test-sa + taskRef: + name: arraytask + kind: Task +status: + conditions: + - type: Succeeded + status: "True" + reason: Succeeded + message: All Tasks have completed executing + results: + - name: IMAGE-DIGEST + value: d9f313aef2d97e58def0511fdc17512d53e6b30d578860ae04b5288c6a239010 + - name: IMAGE-NAME + value: image-3 +`), + } + cms := []*corev1.ConfigMap{withEnabledAlphaAPIFields(newFeatureFlagsConfigMap())} + cms = append(cms, withMaxMatrixCombinationsCount(newDefaultsConfigMap(), 10)) + tests := []struct { + name string + memberOf string + p *v1.Pipeline + expectedTaskRuns []*v1.TaskRun + expectedPipelineRun *v1.PipelineRun + }{{ + name: "p-dag", + memberOf: "tasks", + p: parse.MustParseV1Pipeline(t, fmt.Sprintf(` +metadata: + name: %s + namespace: foo +spec: + tasks: + - name: matrix-emitting-results + matrix: + include: + - name: build-1 + params: + - name: IMAGE + value: image-1 + - name: DOCKERFILE + value: path/to/Dockerfile1 + - name: build-2 + params: + - name: IMAGE + value: image-2 + - name: DOCKERFILE + value: path/to/Dockerfile2 + - name: build-3 + params: + - name: IMAGE + value: image-3 + - name: DOCKERFILE + value: path/to/Dockerfile3 + taskRef: + name: taskwithresults + kind: Task + - name: task-consuming-results + taskRef: + name: arraytask + kind: Task + params: + - name: NAME + value: $(tasks.matrix-emitting-results.results.IMAGE-NAME[*]) + - name: DIGEST + value: $(tasks.matrix-emitting-results.results.IMAGE-DIGEST[*]) +`, "p-dag")), + expectedTaskRuns: []*v1.TaskRun{ + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-task-consuming-results", "foo", + "pr", "p", "task-consuming-results", false), + ` +spec: + params: + - name: NAME + value: + - image-1 + - image-2 + - image-3 + - name: DIGEST + value: + - 0cf457e24a479f02fd4d34540389f720f0807dcff92a7562108165b2637ea82f + - 5a0717cb6596468ea1dffa86011f9b0f497348d80421835b51799f9aeb455642 + - d9f313aef2d97e58def0511fdc17512d53e6b30d578860ae04b5288c6a239010 + serviceAccountName: test-sa + taskRef: + name: arraytask + kind: Task +`), + }, + expectedPipelineRun: parse.MustParseV1PipelineRun(t, ` +metadata: + name: pr + namespace: foo + annotations: {} + labels: + tekton.dev/pipeline: p-dag +spec: + taskRunTemplate: + serviceAccountName: test-sa + pipelineRef: + name: p-dag +status: + pipelineSpec: + tasks: + - name: matrix-emitting-results + matrix: + include: + - name: build-1 + params: + - name: IMAGE + value: image-1 + - name: DOCKERFILE + value: path/to/Dockerfile1 + - name: build-2 + params: + - name: IMAGE + value: image-2 + - name: DOCKERFILE + value: path/to/Dockerfile2 + - name: build-3 + params: + - name: IMAGE + value: image-3 + - name: DOCKERFILE + value: path/to/Dockerfile3 + taskRef: + kind: Task + name: taskwithresults + - name: task-consuming-results + taskRef: + name: arraytask + kind: Task + params: + - name: NAME + value: $(tasks.matrix-emitting-results.results.IMAGE-NAME[*]) + - name: DIGEST + value: $(tasks.matrix-emitting-results.results.IMAGE-DIGEST[*]) + conditions: + - type: Succeeded + status: "Unknown" + reason: "Running" + message: "Tasks Completed: 1 (Failed: 0, Cancelled 0), Incomplete: 1, Skipped: 0" + childReferences: + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-0 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-1 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-2 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-task-consuming-results + pipelineTaskName: task-consuming-results +`), + }, { + name: "p-dag-2", + memberOf: "tasks", + p: parse.MustParseV1Pipeline(t, fmt.Sprintf(` +metadata: + name: %s + namespace: foo +spec: + tasks: + - name: matrix-emitting-results + matrix: + include: + - name: build-1 + params: + - name: IMAGE + value: image-1 + - name: DOCKERFILE + value: path/to/Dockerfile1 + - name: build-2 + params: + - name: IMAGE + value: image-2 + - name: DOCKERFILE + value: path/to/Dockerfile2 + - name: build-3 + params: + - name: IMAGE + value: image-3 + - name: DOCKERFILE + value: path/to/Dockerfile3 + taskRef: + name: taskwithresults + kind: Task + - name: matrix-task-consuming-results + taskRef: + name: stringtask + kind: Task + matrix: + params: + - name: DIGEST + value: $(tasks.matrix-emitting-results.results.IMAGE-DIGEST[*]) +`, "p-dag-2")), + expectedTaskRuns: []*v1.TaskRun{ + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-task-consuming-results-0", "foo", + "pr", "p", "matrix-task-consuming-results", false), + ` +spec: + params: + - name: DIGEST + value: 0cf457e24a479f02fd4d34540389f720f0807dcff92a7562108165b2637ea82f + serviceAccountName: test-sa + taskRef: + name: stringtask + kind: Task +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-task-consuming-results-1", "foo", + "pr", "p", "matrix-task-consuming-results", false), + ` +spec: + params: + - name: DIGEST + value: 5a0717cb6596468ea1dffa86011f9b0f497348d80421835b51799f9aeb455642 + serviceAccountName: test-sa + taskRef: + name: stringtask + kind: Task +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-task-consuming-results-2", "foo", + "pr", "p", "matrix-task-consuming-results", false), + ` +spec: + params: + - name: DIGEST + value: d9f313aef2d97e58def0511fdc17512d53e6b30d578860ae04b5288c6a239010 + serviceAccountName: test-sa + taskRef: + name: stringtask + kind: Task +`), + }, + expectedPipelineRun: parse.MustParseV1PipelineRun(t, ` +metadata: + name: pr + namespace: foo + annotations: {} + labels: + tekton.dev/pipeline: p-dag-2 +spec: + taskRunTemplate: + serviceAccountName: test-sa + pipelineRef: + name: p-dag-2 +status: + pipelineSpec: + tasks: + - name: matrix-emitting-results + matrix: + include: + - name: build-1 + params: + - name: IMAGE + value: image-1 + - name: DOCKERFILE + value: path/to/Dockerfile1 + - name: build-2 + params: + - name: IMAGE + value: image-2 + - name: DOCKERFILE + value: path/to/Dockerfile2 + - name: build-3 + params: + - name: IMAGE + value: image-3 + - name: DOCKERFILE + value: path/to/Dockerfile3 + taskRef: + kind: Task + name: taskwithresults + - name: matrix-task-consuming-results + taskRef: + name: stringtask + kind: Task + matrix: + params: + - name: DIGEST + value: $(tasks.matrix-emitting-results.results.IMAGE-DIGEST[*]) + conditions: + - type: Succeeded + status: "Unknown" + reason: "Running" + message: "Tasks Completed: 1 (Failed: 0, Cancelled 0), Incomplete: 1, Skipped: 0" + childReferences: + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-0 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-1 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-2 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-task-consuming-results-0 + pipelineTaskName: matrix-task-consuming-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-task-consuming-results-1 + pipelineTaskName: matrix-task-consuming-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-task-consuming-results-2 + pipelineTaskName: matrix-task-consuming-results +`), + }, { + name: "p-dag-3", + memberOf: "tasks", + p: parse.MustParseV1Pipeline(t, fmt.Sprintf(` +metadata: + name: %s + namespace: foo +spec: + tasks: + - name: matrix-emitting-results + matrix: + include: + - name: build-1 + params: + - name: IMAGE + value: image-1 + - name: DOCKERFILE + value: path/to/Dockerfile1 + - name: build-2 + params: + - name: IMAGE + value: image-2 + - name: DOCKERFILE + value: path/to/Dockerfile2 + - name: build-3 + params: + - name: IMAGE + value: image-3 + - name: DOCKERFILE + value: path/to/Dockerfile3 + taskRef: + name: taskwithresults + kind: Task + - name: matrixed-echo-length + params: + - name: matrixlength + value: $(tasks.matrix-emitting-results.matrix.length) + taskRef: + name: echomatrixlength + kind: Task + - name: matrixed-echo-results-length + params: + - name: matrixresultslength + value: $(tasks.matrix-emitting-results.matrix.IMAGE-DIGEST.length) + taskRef: + name: echomatrixresultslength + kind: Task +`, "p-dag-3")), + expectedTaskRuns: []*v1.TaskRun{ + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrixed-echo-length", "foo", + "pr", "p", "matrixed-echo-length", false), + ` +spec: + params: + - name: matrixlength + value: 3 + serviceAccountName: test-sa + taskRef: + name: echomatrixlength + kind: Task +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrixed-echo-results-length", "foo", + "pr", "p", "matrixed-echo-results-length", false), + ` +spec: + params: + - name: matrixresultslength + value: 3 + serviceAccountName: test-sa + taskRef: + name: echomatrixresultslength + kind: Task +`), + }, + expectedPipelineRun: parse.MustParseV1PipelineRun(t, ` +metadata: + name: pr + namespace: foo + annotations: {} + labels: + tekton.dev/pipeline: p-dag-3 +spec: + taskRunTemplate: + serviceAccountName: test-sa + pipelineRef: + name: p-dag-3 +status: + pipelineSpec: + tasks: + - name: matrix-emitting-results + matrix: + include: + - name: build-1 + params: + - name: IMAGE + value: image-1 + - name: DOCKERFILE + value: path/to/Dockerfile1 + - name: build-2 + params: + - name: IMAGE + value: image-2 + - name: DOCKERFILE + value: path/to/Dockerfile2 + - name: build-3 + params: + - name: IMAGE + value: image-3 + - name: DOCKERFILE + value: path/to/Dockerfile3 + taskRef: + kind: Task + name: taskwithresults + - name: matrixed-echo-length + params: + - name: matrixlength + value: $(tasks.matrix-emitting-results.matrix.length) + taskRef: + name: echomatrixlength + kind: Task + - name: matrixed-echo-results-length + params: + - name: matrixresultslength + value: $(tasks.matrix-emitting-results.matrix.IMAGE-DIGEST.length) + taskRef: + name: echomatrixresultslength + kind: Task + conditions: + - type: Succeeded + status: "Unknown" + reason: "Running" + message: "Tasks Completed: 1 (Failed: 0, Cancelled 0), Incomplete: 2, Skipped: 0" + childReferences: + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-0 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-1 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-2 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrixed-echo-length + pipelineTaskName: matrixed-echo-length + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrixed-echo-results-length + pipelineTaskName: matrixed-echo-results-length +`), + }, { + name: "p-finally", + memberOf: "finally", + p: parse.MustParseV1Pipeline(t, fmt.Sprintf(` +metadata: + name: %s + namespace: foo +spec: + tasks: + - name: matrix-emitting-results + matrix: + include: + - name: build-1 + params: + - name: IMAGE + value: image-1 + - name: DOCKERFILE + value: path/to/Dockerfile1 + - name: build-2 + params: + - name: IMAGE + value: image-2 + - name: DOCKERFILE + value: path/to/Dockerfile2 + - name: build-3 + params: + - name: IMAGE + value: image-3 + - name: DOCKERFILE + value: path/to/Dockerfile3 + taskRef: + name: taskwithresults + kind: Task + finally: + - name: matrixed-echo-length + params: + - name: matrixlength + value: $(tasks.matrix-emitting-results.matrix.length) + taskRef: + name: echomatrixlength + kind: Task + - name: matrixed-echo-results-length + params: + - name: matrixresultslength + value: $(tasks.matrix-emitting-results.matrix.IMAGE-DIGEST.length) + taskRef: + name: echomatrixresultslength + kind: Task +`, "p-finally")), + + expectedTaskRuns: []*v1.TaskRun{ + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrixed-echo-length", "foo", + "pr", "p-finally", "matrixed-echo-length", false), + ` +spec: + params: + - name: matrixlength + value: 3 + serviceAccountName: test-sa + taskRef: + name: echomatrixlength + kind: Task +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrixed-echo-results-length", "foo", + "pr", "p-finally", "matrixed-echo-results-length", false), + ` +spec: + params: + - name: matrixresultslength + value: 3 + serviceAccountName: test-sa + taskRef: + name: echomatrixresultslength + kind: Task +`), + }, + expectedPipelineRun: parse.MustParseV1PipelineRun(t, ` +metadata: + name: pr + namespace: foo + annotations: {} + labels: + tekton.dev/pipeline: p-finally +spec: + taskRunTemplate: + serviceAccountName: test-sa + pipelineRef: + name: p-finally +status: + pipelineSpec: + tasks: + - name: matrix-emitting-results + matrix: + include: + - name: build-1 + params: + - name: IMAGE + value: image-1 + - name: DOCKERFILE + value: path/to/Dockerfile1 + - name: build-2 + params: + - name: IMAGE + value: image-2 + - name: DOCKERFILE + value: path/to/Dockerfile2 + - name: build-3 + params: + - name: IMAGE + value: image-3 + - name: DOCKERFILE + value: path/to/Dockerfile3 + taskRef: + name: taskwithresults + kind: Task + finally: + - name: matrixed-echo-length + params: + - name: matrixlength + value: $(tasks.matrix-emitting-results.matrix.length) + taskRef: + name: echomatrixlength + kind: Task + - name: matrixed-echo-results-length + params: + - name: matrixresultslength + value: $(tasks.matrix-emitting-results.matrix.IMAGE-DIGEST.length) + taskRef: + name: echomatrixresultslength + kind: Task + conditions: + - type: Succeeded + status: "Unknown" + reason: "Running" + message: "Tasks Completed: 1 (Failed: 0, Cancelled 0), Incomplete: 2, Skipped: 0" + childReferences: + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-0 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-1 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-2 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrixed-echo-length + pipelineTaskName: matrixed-echo-length + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrixed-echo-results-length + pipelineTaskName: matrixed-echo-results-length +`), + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pr := parse.MustParseV1PipelineRun(t, fmt.Sprintf(` +metadata: + name: pr + namespace: foo +spec: + taskRunTemplate: + serviceAccountName: test-sa + pipelineRef: + name: %s +`, tt.name)) + d := test.Data{ + PipelineRuns: []*v1.PipelineRun{pr}, + Pipelines: []*v1.Pipeline{tt.p}, + Tasks: []*v1.Task{task1, task2, task3, task4, taskwithresults}, + TaskRuns: taskRuns, + ConfigMaps: cms, + } + prt := newPipelineRunTest(t, d) + defer prt.Cancel() + _, clients := prt.reconcileRun("foo", "pr", []string{}, false) + taskRuns, err := clients.Pipeline.TektonV1().TaskRuns("foo").List(prt.TestAssets.Ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("tekton.dev/pipelineRun=pr,tekton.dev/pipeline=%s", tt.name), + Limit: 1, + }) + if err != nil { + t.Fatalf("Failure to list TaskRun's %s", err) + } + if len(taskRuns.Items) != len(tt.expectedTaskRuns) { + t.Fatalf("Expected %d TaskRuns got %d", len(tt.expectedTaskRuns), len(taskRuns.Items)) + } + for i := range taskRuns.Items { + expectedTaskRun := tt.expectedTaskRuns[i] + expectedTaskRun.Labels["tekton.dev/pipeline"] = tt.name + expectedTaskRun.Labels["tekton.dev/memberOf"] = tt.memberOf + if d := cmp.Diff(expectedTaskRun, &taskRuns.Items[i], ignoreResourceVersion, ignoreTypeMeta); d != "" { + t.Errorf("expected to see TaskRun %v created. Diff %s", tt.expectedTaskRuns[i].Name, diff.PrintWantGot(d)) + } + } + pipelineRun, err := clients.Pipeline.TektonV1().PipelineRuns("foo").Get(prt.TestAssets.Ctx, "pr", metav1.GetOptions{}) + if err != nil { + t.Fatalf("Got an error getting reconciled run out of fake client: %s", err) + } + if d := cmp.Diff(tt.expectedPipelineRun, pipelineRun, ignoreResourceVersion, ignoreTypeMeta, ignoreLastTransitionTime, ignoreStartTime, ignoreFinallyStartTime, ignoreProvenance, cmpopts.EquateEmpty(), cmpopts.SortSlices(lessChildReferences)); d != "" { + t.Errorf("expected PipelineRun was not created. Diff %s", diff.PrintWantGot(d)) + } + }) + } +} + +func TestReconciler_PipelineTaskMatrixConsumingResults2(t *testing.T) { + names.TestingSeed() + task1 := parse.MustParseV1Task(t, ` +metadata: + name: arraytask + namespace: foo +spec: + params: + - name: result + type: array + steps: + - name: echo + image: alpine + script: | + echo "$(params.result)" +`) + + task2 := parse.MustParseV1Task(t, ` +metadata: + name: stringtask + namespace: foo +spec: + params: + - name: result + type: string + steps: + - name: echo + image: alpine + script: | + echo "$(params.result)" +`) + taskwithresults := parse.MustParseV1Task(t, ` +metadata: + name: taskwithresults + namespace: foo +spec: + params: + - name: platform + default: "" + - name: browser + default: "" + results: + - name: report-url + type: string + steps: + - name: produce-results + image: alpine + script: | + #!/usr/bin/env bash + echo "https://api.example/get-report/$(params.platform)-$(params.browser)" | tee $(results.report-url.path) +`) + trs := []*v1.TaskRun{ + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-emitting-results-0", "foo", + "pr", "p", "matrix-emitting-results", false), + ` +spec: + params: + - name: result + value: https://api.example/get-report/linux-chrome + serviceAccountName: test-sa + taskRef: + name: arraytask + kind: Task +status: + conditions: + - type: Succeeded + status: "True" + reason: Succeeded + message: All Tasks have completed executing + results: + - name: report-url + value: https://api.example/get-report/linux-chrome +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-emitting-results-1", "foo", + "pr", "p", "matrix-emitting-results", false), + ` +spec: + params: + - name: result + value: https://api.example/get-report/mac-chrome + serviceAccountName: test-sa + taskRef: + name: arraytask + kind: Task +status: + conditions: + - type: Succeeded + status: "True" + reason: Succeeded + message: All Tasks have completed executing + results: + - name: report-url + value: https://api.example/get-report/mac-chrome +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-emitting-results-2", "foo", + "pr", "p", "matrix-emitting-results", false), + ` +spec: + params: + - name: result + value: https://api.example/get-report/windows-chrome + serviceAccountName: test-sa + taskRef: + name: arraytask + kind: Task +status: + conditions: + - type: Succeeded + status: "True" + reason: Succeeded + message: All Tasks have completed executing + results: + - name: report-url + value: https://api.example/get-report/windows-chrome +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-emitting-results-3", "foo", + "pr", "p", "matrix-emitting-results", false), + ` +spec: + params: + - name: result + value: https://api.example/get-report/linux-safari + serviceAccountName: test-sa + taskRef: + name: arraytask + kind: Task +status: + conditions: + - type: Succeeded + status: "True" + reason: Succeeded + message: All Tasks have completed executing + results: + - name: report-url + value: https://api.example/get-report/linux-safari +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-emitting-results-4", "foo", + "pr", "p", "matrix-emitting-results", false), + ` +spec: + params: + - name: result + value: https://api.example/get-report/mac-safari + serviceAccountName: test-sa + taskRef: + name: arraytask + kind: Task +status: + conditions: + - type: Succeeded + status: "True" + reason: Succeeded + message: All Tasks have completed executing + results: + - name: report-url + value: https://api.example/get-report/mac-safari +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-emitting-results-5", "foo", + "pr", "p", "matrix-emitting-results", false), + ` +spec: + params: + - name: result + value: https://api.example/get-report/windows-safari + serviceAccountName: test-sa + taskRef: + name: arraytask + kind: Task +status: + conditions: + - type: Succeeded + status: "True" + reason: Succeeded + message: All Tasks have completed executing + results: + - name: report-url + value: https://api.example/get-report/windows-safari +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-emitting-results-6", "foo", + "pr", "p", "matrix-emitting-results", false), + ` +spec: + params: + - name: result + value: https://api.example/get-report/linux-firefox + serviceAccountName: test-sa + taskRef: + name: arraytask + kind: Task +status: + conditions: + - type: Succeeded + status: "True" + reason: Succeeded + message: All Tasks have completed executing + results: + - name: report-url + value: https://api.example/get-report/linux-firefox +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-emitting-results-7", "foo", + "pr", "p", "matrix-emitting-results", false), + ` +spec: + params: + - name: result + value: https://api.example/get-report/mac-firefox + serviceAccountName: test-sa + taskRef: + name: arraytask + kind: Task +status: + conditions: + - type: Succeeded + status: "True" + reason: Succeeded + message: All Tasks have completed executing + results: + - name: report-url + value: https://api.example/get-report/mac-firefox +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-emitting-results-8", "foo", + "pr", "p", "matrix-emitting-results", false), + ` +spec: + params: + - name: result + value: https://api.example/get-report/windows-firefox + serviceAccountName: test-sa + taskRef: + name: arraytask + kind: Task +status: + conditions: + - type: Succeeded + status: "True" + reason: Succeeded + message: All Tasks have completed executing + results: + - name: report-url + value: https://api.example/get-report/windows-firefox +`), + } + cms := []*corev1.ConfigMap{withEnabledAlphaAPIFields(newFeatureFlagsConfigMap())} + cms = append(cms, withMaxMatrixCombinationsCount(newDefaultsConfigMap(), 10)) + tests := []struct { + name string + memberOf string + p *v1.Pipeline + expectedTaskRuns []*v1.TaskRun + expectedPipelineRun *v1.PipelineRun + }{{ + name: "p-dag", + memberOf: "tasks", + p: parse.MustParseV1Pipeline(t, fmt.Sprintf(` +metadata: + name: %s + namespace: foo +spec: + tasks: + - name: matrix-emitting-results + matrix: + params: + - name: platform + value: + - linux + - mac + - windows + - name: browser + value: + - chrome + - safari + - firefox + taskRef: + name: taskwithresults + kind: Task + - name: task-consuming-results + taskRef: + name: arraytask + kind: Task + params: + - name: result + value: $(tasks.matrix-emitting-results.results.report-url[*]) +`, "p-dag")), + expectedTaskRuns: []*v1.TaskRun{ + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-task-consuming-results", "foo", + "pr", "p", "task-consuming-results", false), + ` +spec: + params: + - name: result + value: + - https://api.example/get-report/linux-chrome + - https://api.example/get-report/mac-chrome + - https://api.example/get-report/windows-chrome + - https://api.example/get-report/linux-safari + - https://api.example/get-report/mac-safari + - https://api.example/get-report/windows-safari + - https://api.example/get-report/linux-firefox + - https://api.example/get-report/mac-firefox + - https://api.example/get-report/windows-firefox + serviceAccountName: test-sa + taskRef: + name: arraytask + kind: Task +`), + }, + expectedPipelineRun: parse.MustParseV1PipelineRun(t, ` +metadata: + name: pr + namespace: foo + annotations: {} + labels: + tekton.dev/pipeline: p-dag +spec: + taskRunTemplate: + serviceAccountName: test-sa + pipelineRef: + name: p-dag +status: + pipelineSpec: + tasks: + - name: matrix-emitting-results + matrix: + params: + - name: platform + value: + - linux + - mac + - windows + - name: browser + value: + - chrome + - safari + - firefox + taskRef: + name: taskwithresults + kind: Task + - name: task-consuming-results + taskRef: + name: arraytask + kind: Task + params: + - name: result + value: $(tasks.matrix-emitting-results.results.report-url[*]) + conditions: + - type: Succeeded + status: "Unknown" + reason: "Running" + message: "Tasks Completed: 1 (Failed: 0, Cancelled 0), Incomplete: 1, Skipped: 0" + childReferences: + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-0 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-1 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-2 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-3 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-4 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-5 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-6 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-7 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-8 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-task-consuming-results + pipelineTaskName: task-consuming-results +`), + }, + { + name: "p-dag-2", + memberOf: "tasks", + p: parse.MustParseV1Pipeline(t, fmt.Sprintf(` +metadata: + name: %s + namespace: foo +spec: + tasks: + - name: matrix-emitting-results + matrix: + params: + - name: platform + value: + - linux + - mac + - windows + - name: browser + value: + - chrome + - safari + - firefox + taskRef: + name: taskwithresults + kind: Task + - name: matrix-consuming-results + taskRef: + name: stringtask + kind: Task + matrix: + params: + - name: result + value: $(tasks.matrix-emitting-results.results.report-url[*]) +`, "p-dag-2")), + expectedTaskRuns: []*v1.TaskRun{ + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-consuming-results-0", "foo", + "pr", "p", "matrix-consuming-results", false), + ` +spec: + params: + - name: result + value: https://api.example/get-report/linux-chrome + serviceAccountName: test-sa + taskRef: + name: stringtask + kind: Task +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-consuming-results-1", "foo", + "pr", "p", "matrix-consuming-results", false), + ` +spec: + params: + - name: result + value: https://api.example/get-report/mac-chrome + serviceAccountName: test-sa + taskRef: + name: stringtask + kind: Task +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-consuming-results-2", "foo", + "pr", "p", "matrix-consuming-results", false), + ` +spec: + params: + - name: result + value: https://api.example/get-report/windows-chrome + serviceAccountName: test-sa + taskRef: + name: stringtask + kind: Task +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-consuming-results-3", "foo", + "pr", "p", "matrix-consuming-results", false), + ` +spec: + params: + - name: result + value: https://api.example/get-report/linux-safari + serviceAccountName: test-sa + taskRef: + name: stringtask + kind: Task +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-consuming-results-4", "foo", + "pr", "p", "matrix-consuming-results", false), + ` +spec: + params: + - name: result + value: https://api.example/get-report/mac-safari + serviceAccountName: test-sa + taskRef: + name: stringtask + kind: Task +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-consuming-results-5", "foo", + "pr", "p", "matrix-consuming-results", false), + ` +spec: + params: + - name: result + value: https://api.example/get-report/windows-safari + serviceAccountName: test-sa + taskRef: + name: stringtask + kind: Task +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-consuming-results-6", "foo", + "pr", "p", "matrix-consuming-results", false), + ` +spec: + params: + - name: result + value: https://api.example/get-report/linux-firefox + serviceAccountName: test-sa + taskRef: + name: stringtask + kind: Task +`), mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-consuming-results-7", "foo", + "pr", "p", "matrix-consuming-results", false), + ` +spec: + params: + - name: result + value: https://api.example/get-report/mac-firefox + serviceAccountName: test-sa + taskRef: + name: stringtask + kind: Task +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-matrix-consuming-results-8", "foo", + "pr", "p", "matrix-consuming-results", false), + ` +spec: + params: + - name: result + value: https://api.example/get-report/windows-firefox + serviceAccountName: test-sa + taskRef: + name: stringtask + kind: Task +`), + }, + expectedPipelineRun: parse.MustParseV1PipelineRun(t, ` +metadata: + name: pr + namespace: foo + annotations: {} + labels: + tekton.dev/pipeline: p-dag-2 +spec: + taskRunTemplate: + serviceAccountName: test-sa + pipelineRef: + name: p-dag-2 +status: + pipelineSpec: + tasks: + - name: matrix-emitting-results + matrix: + params: + - name: platform + value: + - linux + - mac + - windows + - name: browser + value: + - chrome + - safari + - firefox + taskRef: + name: taskwithresults + kind: Task + - name: matrix-consuming-results + taskRef: + name: stringtask + kind: Task + matrix: + params: + - name: result + value: $(tasks.matrix-emitting-results.results.report-url[*]) + conditions: + - type: Succeeded + status: "Unknown" + reason: "Running" + message: "Tasks Completed: 1 (Failed: 0, Cancelled 0), Incomplete: 1, Skipped: 0" + childReferences: + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-0 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-1 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-2 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-3 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-4 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-5 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-6 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-7 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-emitting-results-8 + pipelineTaskName: matrix-emitting-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-consuming-results-0 + pipelineTaskName: matrix-consuming-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-consuming-results-1 + pipelineTaskName: matrix-consuming-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-consuming-results-2 + pipelineTaskName: matrix-consuming-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-consuming-results-3 + pipelineTaskName: matrix-consuming-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-consuming-results-4 + pipelineTaskName: matrix-consuming-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-consuming-results-5 + pipelineTaskName: matrix-consuming-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-consuming-results-6 + pipelineTaskName: matrix-consuming-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-consuming-results-7 + pipelineTaskName: matrix-consuming-results + - apiVersion: tekton.dev/v1 + kind: TaskRun + name: pr-matrix-consuming-results-8 + pipelineTaskName: matrix-consuming-results +`), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pr := parse.MustParseV1PipelineRun(t, fmt.Sprintf(` +metadata: + name: pr + namespace: foo +spec: + taskRunTemplate: + serviceAccountName: test-sa + pipelineRef: + name: %s +`, tt.name)) + d := test.Data{ + PipelineRuns: []*v1.PipelineRun{pr}, + Pipelines: []*v1.Pipeline{tt.p}, + Tasks: []*v1.Task{task1, task2, taskwithresults}, + TaskRuns: trs, + ConfigMaps: cms, + } + prt := newPipelineRunTest(t, d) + defer prt.Cancel() + _, clients := prt.reconcileRun("foo", "pr", []string{}, false) + taskRuns, err := clients.Pipeline.TektonV1().TaskRuns("foo").List(prt.TestAssets.Ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("tekton.dev/pipelineRun=pr,tekton.dev/pipeline=%s", tt.name), + Limit: 1, + }) + if err != nil { + t.Fatalf("Failure to list TaskRun's %s", err) + } + if len(taskRuns.Items) != len(tt.expectedTaskRuns) { + t.Fatalf("Expected %d TaskRuns got %d", len(tt.expectedTaskRuns), len(taskRuns.Items)) + } + for i := range taskRuns.Items { + expectedTaskRun := tt.expectedTaskRuns[i] + expectedTaskRun.Labels["tekton.dev/pipeline"] = tt.name + expectedTaskRun.Labels["tekton.dev/memberOf"] = tt.memberOf + if d := cmp.Diff(expectedTaskRun, &taskRuns.Items[i], ignoreResourceVersion, ignoreTypeMeta); d != "" { + t.Errorf("expected to see TaskRun %v created. Diff %s", tt.expectedTaskRuns[i].Name, diff.PrintWantGot(d)) + } + } + pipelineRun, err := clients.Pipeline.TektonV1().PipelineRuns("foo").Get(prt.TestAssets.Ctx, "pr", metav1.GetOptions{}) + if err != nil { + t.Fatalf("Got an error getting reconciled run out of fake client: %s", err) + } + if d := cmp.Diff(tt.expectedPipelineRun, pipelineRun, ignoreResourceVersion, ignoreTypeMeta, ignoreLastTransitionTime, ignoreStartTime, ignoreFinallyStartTime, ignoreProvenance, cmpopts.EquateEmpty()); d != "" { + t.Errorf("expected PipelineRun was not created. Diff %s", diff.PrintWantGot(d)) + } + }) + } +} func TestReconcile_SetDefaults(t *testing.T) { names.TestingSeed() diff --git a/pkg/reconciler/pipelinerun/resources/apply.go b/pkg/reconciler/pipelinerun/resources/apply.go index b356947bbbd..c4b213f722e 100644 --- a/pkg/reconciler/pipelinerun/resources/apply.go +++ b/pkg/reconciler/pipelinerun/resources/apply.go @@ -150,13 +150,80 @@ func ApplyContexts(spec *v1.PipelineSpec, pipelineName string, pr *v1.PipelineRu return ApplyReplacements(spec, GetContextReplacements(pipelineName, pr), map[string][]string{}, map[string]map[string]string{}) } +// looksLikeMatrixContextVar attempts to check if param contains any matrix context variables +func looksLikeMatrixContextVar(params v1.Params) bool { + for _, param := range params { + if expressions, ok := param.GetVarSubstitutionExpressions(); ok { + for _, expression := range expressions { + subExpressions := strings.Split(expression, ".") + if subExpressions[2] == "matrix" && subExpressions[len(subExpressions)-1] == "length" { + return true + } + } + } + } + return false +} + +// parseParams parses "task name", "result name" from a Matrix Context Variable +// Valid Example 1: +// - Input: tasks.myTask.matrix.length +// - Output: "myTask", "" +// Valid Example 2: +// - Input: tasks.myTask.matrix.ResultName.length +// - Output: "myTask", "ResultName" +func parseParams(params v1.Params) (string, string) { + for _, param := range params { + if expressions, ok := param.GetVarSubstitutionExpressions(); ok { + for _, expression := range expressions { + subExpressions := strings.Split(expression, ".") + pipelineTaskName := subExpressions[1] + if len(subExpressions) == 4 { + return pipelineTaskName, "" + } else if len(subExpressions) == 5 { + return pipelineTaskName, subExpressions[3] + } + } + } + } + return "", "" +} + // ApplyPipelineTaskContexts applies the substitution from $(context.pipelineTask.*) with the specified values. // Uses "0" as a default if a value is not available. -func ApplyPipelineTaskContexts(pt *v1.PipelineTask) *v1.PipelineTask { +func ApplyPipelineTaskContexts(pt *v1.PipelineTask, pipelineRunStatus v1.PipelineRunStatus) *v1.PipelineTask { pt = pt.DeepCopy() + var pipelineTaskName string + var resultName string + var matrixLength int + replacements := map[string]string{ "context.pipelineTask.retries": strconv.Itoa(pt.Retries), } + + if looksLikeMatrixContextVar(pt.Params) { + pipelineTaskName, resultName = parseParams(pt.Params) + // find the referenced pipelineTask to count the matrix combinations + if pipelineTaskName != "" && pipelineRunStatus.PipelineSpec != nil { + for _, task := range pipelineRunStatus.PipelineSpec.Tasks { + if task.Name == pipelineTaskName { + matrixLength = task.Matrix.CountCombinations() + } + } + replacements["tasks."+pipelineTaskName+".matrix.length"] = strconv.Itoa(matrixLength) + } + // the number of child refs will map to the number of results produced by the matrixed PipelineTask + if pipelineTaskName != "" && resultName != "" { + var resultsLength int + for _, ref := range pipelineRunStatus.ChildReferences { + if ref.PipelineTaskName == pipelineTaskName { + resultsLength++ + } + } + replacements["tasks."+pipelineTaskName+".matrix."+resultName+".length"] = strconv.Itoa(resultsLength) + } + } + pt.Params = pt.Params.ReplaceVariables(replacements, map[string][]string{}, map[string]map[string]string{}) if pt.IsMatrixed() { pt.Matrix.Params = pt.Matrix.Params.ReplaceVariables(replacements, map[string][]string{}, map[string]map[string]string{}) diff --git a/pkg/reconciler/pipelinerun/resources/apply_test.go b/pkg/reconciler/pipelinerun/resources/apply_test.go index d03998064aa..f1bee30a90e 100644 --- a/pkg/reconciler/pipelinerun/resources/apply_test.go +++ b/pkg/reconciler/pipelinerun/resources/apply_test.go @@ -28,6 +28,7 @@ import ( resources "github.com/tektoncd/pipeline/pkg/reconciler/pipelinerun/resources" "github.com/tektoncd/pipeline/test/diff" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/selection" ) @@ -3241,6 +3242,7 @@ func TestApplyPipelineTaskContexts(t *testing.T) { for _, tc := range []struct { description string pt v1.PipelineTask + prstatus v1.PipelineRunStatus want v1.PipelineTask }{{ description: "context retries replacement", @@ -3324,9 +3326,111 @@ func TestApplyPipelineTaskContexts(t *testing.T) { }}, }, }, + }, { + description: "matrix length context variable", + pt: v1.PipelineTask{ + Params: v1.Params{{ + Name: "matrixlength", + Value: *v1.NewStructuredValues("$(tasks.matrixed-task-run.matrix.length)"), + }}, + }, + prstatus: v1.PipelineRunStatus{ + PipelineRunStatusFields: v1.PipelineRunStatusFields{ + PipelineSpec: &v1.PipelineSpec{ + Tasks: []v1.PipelineTask{{ + Name: "matrixed-task-run", + Matrix: &v1.Matrix{ + Params: v1.Params{ + {Name: "platform", Value: *v1.NewStructuredValues("linux", "mac", "windows")}, + {Name: "browser", Value: *v1.NewStructuredValues("chrome", "firefox", "safari")}, + }}, + }}, + }, + }}, + want: v1.PipelineTask{ + Params: v1.Params{{ + Name: "matrixlength", + Value: *v1.NewStructuredValues("9"), + }}, + }, + }, { + description: "matrix length and matrix results length context variables in matrix include params ", + pt: v1.PipelineTask{ + Params: v1.Params{{ + Name: "matrixlength", + Value: *v1.NewStructuredValues("$(tasks.matrix-emitting-results.matrix.length)"), + }, { + Name: "matrixresultslength", + Value: *v1.NewStructuredValues("$(tasks.matrix-emitting-results.matrix.length)"), + }}, + }, + prstatus: v1.PipelineRunStatus{ + PipelineRunStatusFields: v1.PipelineRunStatusFields{ + PipelineSpec: &v1.PipelineSpec{ + Tasks: []v1.PipelineTask{{ + Name: "matrix-emitting-results", + Matrix: &v1.Matrix{ + Include: []v1.IncludeParams{{ + Name: "build-1", + Params: v1.Params{{ + Name: "DOCKERFILE", Value: *v1.NewStructuredValues("path/to/Dockerfile1"), + }, { + Name: "IMAGE", Value: *v1.NewStructuredValues("image-1"), + }}, + }, { + Name: "build-2", + Params: v1.Params{{ + Name: "DOCKERFILE", Value: *v1.NewStructuredValues("path/to/Dockerfile2"), + }, { + Name: "IMAGE", Value: *v1.NewStructuredValues("image-2"), + }}, + }, { + Name: "build-3", + Params: v1.Params{{ + Name: "DOCKERFILE", Value: *v1.NewStructuredValues("path/to/Dockerfile3"), + }, { + Name: "IMAGE", Value: *v1.NewStructuredValues("image-3"), + }}, + }}, + }}, + }, + }, + ChildReferences: []v1.ChildStatusReference{{ + TypeMeta: runtime.TypeMeta{ + APIVersion: v1.SchemeGroupVersion.String(), + Kind: "TaskRun", + }, + Name: "pr-matrix-emitting-results-0", + PipelineTaskName: "matrix-emitting-results", + }, { + TypeMeta: runtime.TypeMeta{ + APIVersion: v1.SchemeGroupVersion.String(), + Kind: "TaskRun", + }, + Name: "pr-matrix-emitting-results-1", + PipelineTaskName: "matrix-emitting-results", + }, { + TypeMeta: runtime.TypeMeta{ + APIVersion: v1.SchemeGroupVersion.String(), + Kind: "TaskRun", + }, + Name: "pr-matrix-emitting-results-2", + PipelineTaskName: "matrix-emitting-results", + }}, + }, + }, + want: v1.PipelineTask{ + Params: v1.Params{{ + Name: "matrixlength", + Value: *v1.NewStructuredValues("3"), + }, { + Name: "matrixresultslength", + Value: *v1.NewStructuredValues("3"), + }}, + }, }} { t.Run(tc.description, func(t *testing.T) { - got := resources.ApplyPipelineTaskContexts(&tc.pt) + got := resources.ApplyPipelineTaskContexts(&tc.pt, tc.prstatus) if d := cmp.Diff(&tc.want, got); d != "" { t.Errorf(diff.PrintWantGot(d)) } diff --git a/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go b/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go index 8e47b94a303..f47edc1683d 100644 --- a/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go +++ b/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "sort" "github.com/tektoncd/pipeline/pkg/apis/config" "github.com/tektoncd/pipeline/pkg/apis/pipeline" @@ -64,6 +65,7 @@ type ResolvedPipelineTask struct { CustomRuns []*v1beta1.CustomRun PipelineTask *v1.PipelineTask ResolvedTask *resources.ResolvedTask + ResultsCache map[string][]string } // isDone returns true only if the task is skipped, succeeded or failed @@ -741,3 +743,25 @@ func CheckMissingResultReferences(pipelineRunState PipelineRunState, targets Pip } return nil } + +// createResultsCacheMatrixedTaskRuns creates a cache of results that have been fanned out from a +// referenced matrixed PipelintTask so that you can easily access these results in subsequent Pipeline Tasks +func (t *ResolvedPipelineTask) createResultsCacheMatrixedTaskRuns() { + if len(t.ResultsCache) == 0 { + t.ResultsCache = make(map[string][]string) + } + // Sort the taskRuns by name to ensure the order is deterministic + sort.Slice(t.TaskRuns, func(i, j int) bool { + return t.TaskRuns[i].Name < t.TaskRuns[j].Name + }) + for _, taskRun := range t.TaskRuns { + results := taskRun.Status.Results + for _, result := range results { + if _, ok := t.ResultsCache[result.Name]; ok { + t.ResultsCache[result.Name] = append(t.ResultsCache[result.Name], result.Value.StringVal) + } else { + t.ResultsCache[result.Name] = []string{result.Value.StringVal} + } + } + } +} diff --git a/pkg/reconciler/pipelinerun/resources/pipelinerunstate.go b/pkg/reconciler/pipelinerun/resources/pipelinerunstate.go index 65882f4c987..151d53885f7 100644 --- a/pkg/reconciler/pipelinerun/resources/pipelinerunstate.go +++ b/pkg/reconciler/pipelinerun/resources/pipelinerunstate.go @@ -157,14 +157,34 @@ func (state PipelineRunState) GetTaskRunsResults() map[string][]v1.TaskRunResult if !rpt.isSuccessful() { continue } - // Currently a Matrix cannot produce results so this is for a singular TaskRun - if len(rpt.TaskRuns) == 1 { + if rpt.PipelineTask.IsMatrixed() { + taskRunResults := convertResultsMapToTaskRunResults(rpt.ResultsCache) + if len(taskRunResults) > 0 { + results[rpt.PipelineTask.Name] = taskRunResults + } + } else { results[rpt.PipelineTask.Name] = rpt.TaskRuns[0].Status.Results } } return results } +func convertResultsMapToTaskRunResults(resultsMap map[string][]string) []v1.TaskRunResult { + var taskRunResults []v1.TaskRunResult + for result, val := range resultsMap { + taskRunResult := v1.TaskRunResult{ + Name: result, + Type: v1.ResultsTypeArray, + Value: v1.ParamValue{ + Type: v1.ParamTypeArray, + ArrayVal: val, + }, + } + taskRunResults = append(taskRunResults, taskRunResult) + } + return taskRunResults +} + // GetRunsResults returns a map of all successfully completed Runs in the state, with the pipeline task name as the key // and the results from the corresponding TaskRun as the value. It only includes runs which have completed successfully. func (state PipelineRunState) GetRunsResults() map[string][]v1beta1.CustomRunResult { diff --git a/pkg/reconciler/pipelinerun/resources/resultrefresolution.go b/pkg/reconciler/pipelinerun/resources/resultrefresolution.go index e4f78aa66b2..e8a0faf909c 100644 --- a/pkg/reconciler/pipelinerun/resources/resultrefresolution.go +++ b/pkg/reconciler/pipelinerun/resources/resultrefresolution.go @@ -109,64 +109,73 @@ func removeDup(refs ResolvedResultRefs) ResolvedResultRefs { return deduped } -// convertToResultRefs walks a PipelineTask looking for result references. If any are -// found they are resolved to a value by searching pipelineRunState. The list of resolved -// references are returned. If an error is encountered due to an invalid result reference -// then a nil list and error is returned instead. func convertToResultRefs(pipelineRunState PipelineRunState, target *ResolvedPipelineTask) (ResolvedResultRefs, string, error) { var resolvedResultRefs ResolvedResultRefs - for _, ref := range v1.PipelineTaskResultRefs(target.PipelineTask) { - resolved, pt, err := resolveResultRef(pipelineRunState, ref) - if err != nil { - return nil, pt, err + for _, resultRef := range v1.PipelineTaskResultRefs(target.PipelineTask) { + referencedPipelineTask := pipelineRunState.ToMap()[resultRef.PipelineTask] + if referencedPipelineTask == nil { + return nil, resultRef.PipelineTask, fmt.Errorf("could not find task %q referenced by result", resultRef.PipelineTask) + } + if !referencedPipelineTask.isSuccessful() && !referencedPipelineTask.isFailure() { + return nil, resultRef.PipelineTask, fmt.Errorf("task %q referenced by result was not finished", referencedPipelineTask.PipelineTask.Name) + } + switch { + case referencedPipelineTask.IsCustomTask(): + resolved, err := resolveCustomResultRef(referencedPipelineTask.CustomRuns, resultRef) + if err != nil { + return nil, resultRef.PipelineTask, err + } + resolvedResultRefs = append(resolvedResultRefs, resolved) + // Matrixed referenced Pipeline Task + case len(referencedPipelineTask.TaskRuns) > 1: + arrayValues, err := findResultValuesForMatrix(referencedPipelineTask, resultRef) + if err != nil { + return nil, resultRef.PipelineTask, err + } + for _, taskRun := range referencedPipelineTask.TaskRuns { + resolved := createMatrixedTaskResultForParam(taskRun.Name, arrayValues, resultRef) + resolvedResultRefs = append(resolvedResultRefs, resolved) + } + // Regular PipelineTask + case len(referencedPipelineTask.TaskRuns) == 1: + resolved, err := resolveResultRef(referencedPipelineTask.TaskRuns, resultRef) + if err != nil { + return nil, resultRef.PipelineTask, err + } + resolvedResultRefs = append(resolvedResultRefs, resolved) } - resolvedResultRefs = append(resolvedResultRefs, resolved) } return resolvedResultRefs, "", nil } -func resolveResultRef(pipelineState PipelineRunState, resultRef *v1.ResultRef) (*ResolvedResultRef, string, error) { - referencedPipelineTask := pipelineState.ToMap()[resultRef.PipelineTask] - if referencedPipelineTask == nil { - return nil, resultRef.PipelineTask, fmt.Errorf("could not find task %q referenced by result", resultRef.PipelineTask) - } - if !referencedPipelineTask.isSuccessful() && !referencedPipelineTask.isFailure() { - return nil, resultRef.PipelineTask, fmt.Errorf("task %q referenced by result was not finished", referencedPipelineTask.PipelineTask.Name) +func resolveCustomResultRef(customRuns []*v1beta1.CustomRun, resultRef *v1.ResultRef) (*ResolvedResultRef, error) { + customRun := customRuns[0] + runName := customRun.GetObjectMeta().GetName() + runValue, err := findRunResultForParam(customRun, resultRef) + if err != nil { + return nil, err } + return &ResolvedResultRef{ + Value: *v1.NewStructuredValues(runValue), + FromTaskRun: "", + FromRun: runName, + ResultReference: *resultRef, + }, nil +} - var runName, runValue, taskRunName string - var resultValue v1.ResultValue - var err error - if referencedPipelineTask.IsCustomTask() { - if len(referencedPipelineTask.CustomRuns) != 1 { - return nil, resultRef.PipelineTask, fmt.Errorf("referenced tasks can only have length of 1 since a matrixed task does not support producing results, but was length %d", len(referencedPipelineTask.TaskRuns)) - } - customRun := referencedPipelineTask.CustomRuns[0] - runName = customRun.GetObjectMeta().GetName() - runValue, err = findRunResultForParam(customRun, resultRef) - resultValue = *v1.NewStructuredValues(runValue) - if err != nil { - return nil, resultRef.PipelineTask, err - } - } else { - // Check to make sure the referenced task is not a matrix since a matrix does not support producing results - if len(referencedPipelineTask.TaskRuns) != 1 { - return nil, resultRef.PipelineTask, fmt.Errorf("referenced tasks can only have length of 1 since a matrixed task does not support producing results, but was length %d", len(referencedPipelineTask.TaskRuns)) - } - taskRun := referencedPipelineTask.TaskRuns[0] - taskRunName = taskRun.Name - resultValue, err = findTaskResultForParam(taskRun, resultRef) - if err != nil { - return nil, resultRef.PipelineTask, err - } +func resolveResultRef(taskRuns []*v1.TaskRun, resultRef *v1.ResultRef) (*ResolvedResultRef, error) { + taskRun := taskRuns[0] + taskRunName := taskRun.Name + resultValue, err := findTaskResultForParam(taskRun, resultRef) + if err != nil { + return nil, err } - return &ResolvedResultRef{ Value: resultValue, FromTaskRun: taskRunName, - FromRun: runName, + FromRun: "", ResultReference: *resultRef, - }, "", nil + }, nil } func findRunResultForParam(customRun *v1beta1.CustomRun, reference *v1.ResultRef) (string, error) { @@ -178,7 +187,6 @@ func findRunResultForParam(customRun *v1beta1.CustomRun, reference *v1.ResultRef err := fmt.Errorf("%w: Could not find result with name %s for task %s", ErrInvalidTaskResultReference, reference.Result, reference.PipelineTask) return "", err } - func findTaskResultForParam(taskRun *v1.TaskRun, reference *v1.ResultRef) (v1.ResultValue, error) { results := taskRun.Status.TaskRunStatusFields.Results for _, result := range results { @@ -190,6 +198,29 @@ func findTaskResultForParam(taskRun *v1.TaskRun, reference *v1.ResultRef) (v1.Re return v1.ResultValue{}, err } +func findResultValuesForMatrix(referencedPipelineTask *ResolvedPipelineTask, resultRef *v1.ResultRef) (v1.ParamValue, error) { + if len(referencedPipelineTask.ResultsCache) == 0 { + referencedPipelineTask.createResultsCacheMatrixedTaskRuns() + } + if arrayValues, ok := referencedPipelineTask.ResultsCache[resultRef.Result]; ok { + return v1.ParamValue{ + Type: v1.ParamTypeArray, + ArrayVal: arrayValues, + }, nil + } + err := fmt.Errorf("%w: Could not find result with name %s for task %s", ErrInvalidTaskResultReference, resultRef.Result, resultRef.PipelineTask) + return v1.ParamValue{}, err +} + +func createMatrixedTaskResultForParam(taskRunName string, paramValue v1.ParamValue, resultRef *v1.ResultRef) *ResolvedResultRef { + return &ResolvedResultRef{ + Value: paramValue, + FromTaskRun: taskRunName, + FromRun: "", + ResultReference: *resultRef, + } +} + func (rs ResolvedResultRefs) getStringReplacements() map[string]string { replacements := map[string]string{} for _, r := range rs { diff --git a/pkg/reconciler/pipelinerun/resources/validate_params.go b/pkg/reconciler/pipelinerun/resources/validate_params.go index d954756f1d0..8cc4ea1be48 100644 --- a/pkg/reconciler/pipelinerun/resources/validate_params.go +++ b/pkg/reconciler/pipelinerun/resources/validate_params.go @@ -105,6 +105,13 @@ func ValidateParameterTypesInMatrix(state PipelineRunState) error { if m.HasParams() { for _, param := range m.Params { if param.Value.Type != v1.ParamTypeArray { + // If it's an array type that contains result references because it's consuming results + // from a Matrixed PipelineTask continue + if ps, ok := param.GetVarSubstitutionExpressions(); ok { + if v1.LooksLikeContainsResultRefs(ps) { + continue + } + } return fmt.Errorf("parameters of type array only are allowed, but param \"%s\" has type \"%s\" in pipelineTask \"%s\"", param.Name, string(param.Value.Type), rpt.PipelineTask.Name) } diff --git a/pkg/reconciler/pipelinerun/resources/validate_params_test.go b/pkg/reconciler/pipelinerun/resources/validate_params_test.go index fefce539c7e..4b6e667d8fc 100644 --- a/pkg/reconciler/pipelinerun/resources/validate_params_test.go +++ b/pkg/reconciler/pipelinerun/resources/validate_params_test.go @@ -410,6 +410,17 @@ func TestValidatePipelineParameterTypes(t *testing.T) { }, }}, wantErrs: "parameters of type string only are allowed, but param \"barfoo\" has type \"object\" in pipelineTask \"task\"", + }, { + desc: "parameters in matrix are result references", + state: resources.PipelineRunState{{ + PipelineTask: &v1.PipelineTask{ + Name: "task", + Matrix: &v1.Matrix{ + Params: v1.Params{{ + Name: "url", Value: v1.ParamValue{Type: v1.ParamTypeString, StringVal: `$(tasks.matrix-emitting-results.results.report-url[*])`}, + }}}, + }, + }}, }} { t.Run(tc.desc, func(t *testing.T) { err := resources.ValidateParameterTypesInMatrix(tc.state)