From 89e74aa9530a90020d4afaf556092a54bb495d90 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Fri, 28 May 2021 09:34:07 -0700 Subject: [PATCH] Change branch compilation to avoid adding downstream deps (#264) * Change branch compilation to avoid adding downstream deps Signed-off-by: Haytham Abuelfutuh * Fix deepcopy generated bug Signed-off-by: Haytham Abuelfutuh * Add checks to make sure compiled workflows remain the same Signed-off-by: Haytham Abuelfutuh * Additional unit test to test branch spec generation Signed-off-by: Ketan Umare * Fix compiler unit tests Signed-off-by: Haytham Abuelfutuh * Fixed unit test Signed-off-by: Ketan Umare * Fix one unit test and shorten json tags for connections Signed-off-by: Haytham Abuelfutuh * Use GetConnections() instead of DeprecatedConnections Signed-off-by: Haytham Abuelfutuh * Put back branch node upstream check in CanExecute predicate for backward compatibility Signed-off-by: Haytham Abuelfutuh * lint Signed-off-by: Haytham Abuelfutuh Co-authored-by: Ketan Umare --- .../apis/flyteworkflow/v1alpha1/workflow.go | 41 +- .../flyteworkflow/v1alpha1/workflow_test.go | 12 +- .../v1alpha1/zz_generated.deepcopy.go | 13 + flytepropeller/pkg/compiler/common/builder.go | 2 + .../compiler/common/mocks/workflow_builder.go | 10 + .../pkg/compiler/test/compiler_test.go | 47 ++ ...flows-work-one-python-task-w-f-inputs.yaml | 2 +- .../app-workflows-work-one-python-task-w-f.pb | Bin 560 -> 632 bytes ...pp-workflows-work-one-python-task-w-f.yaml | 6 + .../testdata/branch/compiled/success_1.json | 1 + .../testdata/branch/compiled/success_2.json | 1 + .../testdata/branch/compiled/success_3.json | 1 + .../testdata/branch/compiled/success_4.json | 1 + .../testdata/branch/compiled/success_5.json | 1 + .../testdata/branch/compiled/success_6.json | 1 + .../branch/compiled/success_7_nested.json | 1 + .../test/testdata/branch/k8s/success_1.json | 1 + .../test/testdata/branch/k8s/success_2.json | 1 + .../test/testdata/branch/k8s/success_3.json | 1 + .../test/testdata/branch/k8s/success_4.json | 1 + .../test/testdata/branch/k8s/success_5.json | 1 + .../test/testdata/branch/k8s/success_6.json | 1 + .../testdata/branch/k8s/success_7_nested.json | 1 + .../testdata/branch/success_7_nested.json | 363 ++++++++++++ .../pkg/compiler/transformers/k8s/node.go | 55 +- .../compiler/transformers/k8s/node_test.go | 16 +- .../compiled_closure_branch_nested.json | 538 ++++++++++++++++++ .../pkg/compiler/transformers/k8s/workflow.go | 19 +- .../transformers/k8s/workflow_test.go | 40 ++ .../pkg/compiler/validators/bindings.go | 19 +- .../pkg/compiler/validators/bindings_test.go | 12 +- .../pkg/compiler/validators/branch.go | 49 +- .../pkg/compiler/validators/branch_test.go | 4 + .../pkg/compiler/validators/condition.go | 6 +- .../pkg/compiler/validators/node.go | 32 +- .../pkg/compiler/validators/node_test.go | 2 +- .../pkg/compiler/workflow_compiler.go | 49 +- .../pkg/compiler/workflow_compiler_test.go | 12 +- .../nodes/dynamic/dynamic_workflow_test.go | 18 +- .../pkg/controller/nodes/executor_test.go | 26 +- .../pkg/controller/nodes/predicate.go | 8 +- .../pkg/controller/nodes/predicate_test.go | 110 ---- .../nodes/task/remote_workflow_store_test.go | 31 + .../pkg/controller/workflow/executor_test.go | 2 +- flytepropeller/pkg/visualize/visualize.go | 2 +- 45 files changed, 1304 insertions(+), 256 deletions(-) create mode 100755 flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_1.json create mode 100755 flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_2.json create mode 100755 flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_3.json create mode 100755 flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_4.json create mode 100755 flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_5.json create mode 100755 flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_6.json create mode 100755 flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json create mode 100755 flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_1.json create mode 100755 flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_2.json create mode 100755 flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_3.json create mode 100755 flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_4.json create mode 100755 flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_5.json create mode 100755 flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_6.json create mode 100755 flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json create mode 100644 flytepropeller/pkg/compiler/test/testdata/branch/success_7_nested.json create mode 100644 flytepropeller/pkg/compiler/transformers/k8s/testdata/compiled_closure_branch_nested.json diff --git a/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/workflow.go b/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/workflow.go index 06148b4716..0b4e6f3a31 100644 --- a/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/workflow.go +++ b/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/workflow.go @@ -181,12 +181,13 @@ func (in *Inputs) DeepCopyInto(out *Inputs) { // Once we figure out the autogenerate story we can replace this } -type Connections struct { +// Deprecated: Please use Connections instead +type DeprecatedConnections struct { DownstreamEdges map[NodeID][]NodeID UpstreamEdges map[NodeID][]NodeID } -func (in *Connections) UnmarshalJSON(b []byte) error { +func (in *DeprecatedConnections) UnmarshalJSON(b []byte) error { in.DownstreamEdges = map[NodeID][]NodeID{} err := json.Unmarshal(b, &in.DownstreamEdges) if err != nil { @@ -204,10 +205,23 @@ func (in *Connections) UnmarshalJSON(b []byte) error { return nil } -func (in *Connections) MarshalJSON() ([]byte, error) { +func (in *DeprecatedConnections) MarshalJSON() ([]byte, error) { return json.Marshal(in.DownstreamEdges) } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeprecatedConnections) DeepCopyInto(out *DeprecatedConnections) { + *out = *in + // We do not manipulate the object, so its ok + // Once we figure out the autogenerate story we can replace this +} + +// Connections keep track of downstream and upstream dependencies (including data and execution dependencies). +type Connections struct { + Downstream map[NodeID][]NodeID `json:"downstream"` + Upstream map[NodeID][]NodeID `json:"upstream"` +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Connections) DeepCopyInto(out *Connections) { *out = *in @@ -223,7 +237,13 @@ type WorkflowSpec struct { // Defines the set of connections (both data dependencies and execution dependencies) that the graph is // formed of. The execution engine will respect and follow these connections as it determines which nodes // can and should be executed. - Connections Connections `json:"connections"` + // Deprecated: Please use Connections + DeprecatedConnections DeprecatedConnections `json:"connections"` + + // Defines the set of connections (both data dependencies and execution dependencies) that the graph is + // formed of. The execution engine will respect and follow these connections as it determines which nodes + // can and should be executed. + Connections Connections `json:"edges"` // Defines a single node to execute in case the system determined the Workflow has failed. OnFailure *NodeSpec `json:"onFailure,omitempty"` @@ -257,7 +277,7 @@ func (in *WorkflowSpec) ToNode(name NodeID) ([]NodeID, error) { if _, ok := in.Nodes[name]; !ok { return nil, errors.Errorf("Bad Node [%v], is not defined in the Workflow [%v]", name, in.ID) } - upstreamNodes := in.Connections.UpstreamEdges[name] + upstreamNodes := in.GetConnections().Upstream[name] return upstreamNodes, nil } @@ -266,7 +286,7 @@ func (in *WorkflowSpec) FromNode(name NodeID) ([]NodeID, error) { return nil, errors.Errorf("Bad Node [%v], is not defined in the Workflow [%v]", name, in.ID) } - downstreamNodes := in.Connections.DownstreamEdges[name] + downstreamNodes := in.GetConnections().Downstream[name] return downstreamNodes, nil } @@ -284,6 +304,15 @@ func (in *WorkflowSpec) GetNode(nodeID NodeID) (ExecutableNode, bool) { } func (in *WorkflowSpec) GetConnections() *Connections { + // For backward compatibility, if the new Connections field is not yet populated then copy the connections from the + // deprecated field. This will happen in one of two cases: + // 1. If an old Admin generated the CRD + // 2. If new propeller is deployed and is unmarshalling an old CRD. + if len(in.Connections.Upstream) == 0 && len(in.Connections.Downstream) == 0 { + in.Connections.Upstream = in.DeprecatedConnections.UpstreamEdges + in.Connections.Downstream = in.DeprecatedConnections.DownstreamEdges + } + return &in.Connections } diff --git a/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/workflow_test.go b/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/workflow_test.go index a3b52e317e..fc72a54ef0 100644 --- a/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/workflow_test.go +++ b/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/workflow_test.go @@ -14,7 +14,7 @@ import ( func TestMarshalUnmarshal_Connections(t *testing.T) { r, err := ioutil.ReadFile("testdata/connections.json") assert.NoError(t, err) - o := v1alpha1.Connections{} + o := v1alpha1.DeprecatedConnections{} err = json.Unmarshal(r, &o) assert.NoError(t, err) assert.Equal(t, map[v1alpha1.NodeID][]v1alpha1.NodeID{ @@ -42,10 +42,12 @@ func TestWorkflowSpec(t *testing.T) { assert.NoError(t, err) w := &v1alpha1.FlyteWorkflow{} err = json.Unmarshal(j, w) - assert.NoError(t, err) + if !assert.NoError(t, err) { + t.FailNow() + } + assert.NotNil(t, w.WorkflowSpec) assert.Nil(t, w.GetOnFailureNode()) - assert.Equal(t, 7, len(w.Connections.DownstreamEdges)) - assert.Equal(t, 8, len(w.Connections.UpstreamEdges)) - + assert.Equal(t, 7, len(w.GetConnections().Downstream)) + assert.Equal(t, 8, len(w.GetConnections().Upstream)) } diff --git a/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/zz_generated.deepcopy.go b/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/zz_generated.deepcopy.go index 0438cea079..b4ebb861fe 100644 --- a/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/zz_generated.deepcopy.go +++ b/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/zz_generated.deepcopy.go @@ -110,6 +110,16 @@ func (in *Connections) DeepCopy() *Connections { return out } +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeprecatedConnections. +func (in *DeprecatedConnections) DeepCopy() *DeprecatedConnections { + if in == nil { + return nil + } + out := new(DeprecatedConnections) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DynamicNodeStatus) DeepCopyInto(out *DynamicNodeStatus) { *out = *in @@ -234,6 +244,7 @@ func (in *FlyteWorkflow) DeepCopyInto(out *FlyteWorkflow) { in, out := &in.AcceptedAt, &out.AcceptedAt *out = (*in).DeepCopy() } + out.SecurityContext = in.SecurityContext in.Status.DeepCopyInto(&out.Status) in.RawOutputDataConfig.DeepCopyInto(&out.RawOutputDataConfig) @@ -739,7 +750,9 @@ func (in *WorkflowSpec) DeepCopyInto(out *WorkflowSpec) { (*out)[key] = outVal } } + in.DeprecatedConnections.DeepCopyInto(&out.DeprecatedConnections) in.Connections.DeepCopyInto(&out.Connections) + if in.OnFailure != nil { in, out := &in.OnFailure, &out.OnFailure *out = new(NodeSpec) diff --git a/flytepropeller/pkg/compiler/common/builder.go b/flytepropeller/pkg/compiler/common/builder.go index 5cd68f50c9..be65dd2dec 100644 --- a/flytepropeller/pkg/compiler/common/builder.go +++ b/flytepropeller/pkg/compiler/common/builder.go @@ -18,6 +18,8 @@ type WorkflowBuilder interface { Workflow StoreCompiledSubWorkflow(id WorkflowID, compiledWorkflow *core.CompiledWorkflow) AddExecutionEdge(nodeFrom, nodeTo NodeID) + AddUpstreamEdge(nodeProvider, nodeDependent NodeID) + AddDownstreamEdge(nodeProvider, nodeDependent NodeID) AddNode(n NodeBuilder, errs errors.CompileErrors) (node NodeBuilder, ok bool) ValidateWorkflow(fg *core.CompiledWorkflow, errs errors.CompileErrors) (Workflow, bool) NewNodeBuilder(n *core.Node) NodeBuilder diff --git a/flytepropeller/pkg/compiler/common/mocks/workflow_builder.go b/flytepropeller/pkg/compiler/common/mocks/workflow_builder.go index 40210e8799..d2deaa10e7 100644 --- a/flytepropeller/pkg/compiler/common/mocks/workflow_builder.go +++ b/flytepropeller/pkg/compiler/common/mocks/workflow_builder.go @@ -16,6 +16,11 @@ type WorkflowBuilder struct { mock.Mock } +// AddDownstreamEdge provides a mock function with given fields: nodeProvider, nodeDependent +func (_m *WorkflowBuilder) AddDownstreamEdge(nodeProvider string, nodeDependent string) { + _m.Called(nodeProvider, nodeDependent) +} + // AddExecutionEdge provides a mock function with given fields: nodeFrom, nodeTo func (_m *WorkflowBuilder) AddExecutionEdge(nodeFrom string, nodeTo string) { _m.Called(nodeFrom, nodeTo) @@ -62,6 +67,11 @@ func (_m *WorkflowBuilder) AddNode(n common.NodeBuilder, errs errors.CompileErro return r0, r1 } +// AddUpstreamEdge provides a mock function with given fields: nodeProvider, nodeDependent +func (_m *WorkflowBuilder) AddUpstreamEdge(nodeProvider string, nodeDependent string) { + _m.Called(nodeProvider, nodeDependent) +} + type WorkflowBuilder_GetCompiledSubWorkflow struct { *mock.Call } diff --git a/flytepropeller/pkg/compiler/test/compiler_test.go b/flytepropeller/pkg/compiler/test/compiler_test.go index 8cd60de46c..0806f1ceb4 100644 --- a/flytepropeller/pkg/compiler/test/compiler_test.go +++ b/flytepropeller/pkg/compiler/test/compiler_test.go @@ -10,6 +10,8 @@ import ( "strings" "testing" + "github.com/go-test/deep" + "github.com/ghodss/yaml" "github.com/flyteorg/flyteidl/clients/go/coreutils" @@ -184,6 +186,10 @@ func TestBranches(t *testing.T) { errors.SetConfig(errors.Config{IncludeSource: true}) assert.NoError(t, filepath.Walk("testdata/branch", func(path string, info os.FileInfo, err error) error { if info.IsDir() { + if filepath.Base(info.Name()) != "branch" { + return filepath.SkipDir + } + return nil } @@ -209,6 +215,29 @@ func TestBranches(t *testing.T) { t.FailNow() } + marshaler := jsonpb.Marshaler{} + rawStr, err := marshaler.MarshalToString(compiledWfc) + if !assert.NoError(t, err) { + t.Fail() + } + + compiledFilePath := filepath.Join(filepath.Dir(path), "compiled", filepath.Base(path)) + if *update { + err = ioutil.WriteFile(compiledFilePath, []byte(rawStr), os.ModePerm) + if !assert.NoError(t, err) { + t.Fail() + } + } else { + goldenRaw, err := ioutil.ReadFile(compiledFilePath) + if !assert.NoError(t, err) { + t.Fail() + } + + if diff := deep.Equal(rawStr, string(goldenRaw)); diff != nil { + t.Errorf("Compiled() Diff = %v\r\n got = %v\r\n want = %v", diff, rawStr, string(goldenRaw)) + } + } + inputs := map[string]interface{}{} for varName, v := range compiledWfc.Primary.Template.Interface.Inputs.Variables { inputs[varName] = coreutils.MustMakeDefaultLiteralForType(v.Type) @@ -227,6 +256,24 @@ func TestBranches(t *testing.T) { if assert.NoError(t, err) { assert.NotEmpty(t, raw) } + + k8sObjectFilepath := filepath.Join(filepath.Dir(path), "k8s", filepath.Base(path)) + if *update { + err = ioutil.WriteFile(k8sObjectFilepath, raw, os.ModePerm) + if !assert.NoError(t, err) { + t.Fail() + } + } else { + goldenRaw, err := ioutil.ReadFile(k8sObjectFilepath) + if !assert.NoError(t, err) { + t.Fail() + } + + if diff := deep.Equal(string(raw), string(goldenRaw)); diff != nil { + t.Errorf("K8sObject() Diff = %v\r\n got = %v\r\n want = %v", diff, rawStr, string(goldenRaw)) + } + + } } }) diff --git a/flytepropeller/pkg/compiler/test/testdata/app-workflows-work-one-python-task-w-f-inputs.yaml b/flytepropeller/pkg/compiler/test/testdata/app-workflows-work-one-python-task-w-f-inputs.yaml index 3893cfb77a..0967ef424b 100755 --- a/flytepropeller/pkg/compiler/test/testdata/app-workflows-work-one-python-task-w-f-inputs.yaml +++ b/flytepropeller/pkg/compiler/test/testdata/app-workflows-work-one-python-task-w-f-inputs.yaml @@ -1 +1 @@ -literals: {} +{} diff --git a/flytepropeller/pkg/compiler/test/testdata/app-workflows-work-one-python-task-w-f.pb b/flytepropeller/pkg/compiler/test/testdata/app-workflows-work-one-python-task-w-f.pb index 5f6557e400ff8e248c0ac1ca59de2cc54c8758a4..039e2fadd20e14fed9610bd264934a997330a9f2 100755 GIT binary patch delta 26 icmdnM@`Gi<7N$eY6MHl!-b-NG!8F-{@x$a7j2-}(0}B)Y delta 22 ecmeytvVmp77N%cJ6MHl!-b-Nmz&P1~@dE&IiV3~| diff --git a/flytepropeller/pkg/compiler/test/testdata/app-workflows-work-one-python-task-w-f.yaml b/flytepropeller/pkg/compiler/test/testdata/app-workflows-work-one-python-task-w-f.yaml index a9c3f1f960..3aee111b74 100644 --- a/flytepropeller/pkg/compiler/test/testdata/app-workflows-work-one-python-task-w-f.yaml +++ b/flytepropeller/pkg/compiler/test/testdata/app-workflows-work-one-python-task-w-f.yaml @@ -12,6 +12,12 @@ tasks: command: - pyflyte-execute config: + - key: testKey1 + value: testValue1 + - key: testKey2 + value: testValue2 + - key: testKey3 + value: testValue3 - key: testKey1 value: testValue1 - key: testKey2 diff --git a/flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_1.json b/flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_1.json new file mode 100755 index 0000000000..e9fd78dd47 --- /dev/null +++ b/flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_1.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"project","domain":"domain","name":"test_serialization.my_wf","version":"version"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}]},{"id":"node-0","metadata":{"name":"test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}},{"id":"node-1","metadata":{"name":"test1","retries":{},"interruptible":false},"inputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"upstreamNodeIds":["node-0"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"out_0"},"rightValue":{"primitive":{"integer":"1"}}}},"thenNode":{"id":"node-1-branchnode-0","metadata":{"name":"test_serialization.t2","retries":{},"interruptible":false},"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"}}}},"error":{"failedNodeId":"test1","message":"Unable to choose branch"}}}}],"outputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"node-0":{"ids":["node-1"]},"node-1":{"ids":["end-node"]},"start-node":{"ids":["node-0"]}},"upstream":{"end-node":{"ids":["node-1"]},"node-0":{"ids":["start-node"]},"node-1":{"ids":["node-0"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_2.json b/flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_2.json new file mode 100755 index 0000000000..48820fa416 --- /dev/null +++ b/flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_2.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"project","domain":"domain","name":"test_serialization.my_wf","version":"version"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},{"id":"node-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}},{"id":"node-1","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}},{"id":"node-2","metadata":{"name":"test1","retries":{},"interruptible":false},"inputs":[{"var":"node-0.out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}},{"var":"node-1.out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"upstreamNodeIds":["node-0","node-1"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"node-0.out_0"},"rightValue":{"var":"node-1.out_0"}}},"thenNode":{"id":"node-2-branchnode-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t2","retries":{},"interruptible":false},"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"}}}},"error":{"failedNodeId":"test1","message":"Unable to choose branch"}}}}],"outputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"node-0":{"ids":["node-2"]},"node-1":{"ids":["node-2"]},"node-2":{"ids":["end-node"]},"start-node":{"ids":["node-0","node-1"]}},"upstream":{"end-node":{"ids":["node-2"]},"node-0":{"ids":["start-node"]},"node-1":{"ids":["start-node"]},"node-2":{"ids":["node-0","node-1"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_3.json b/flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_3.json new file mode 100755 index 0000000000..de85ced031 --- /dev/null +++ b/flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_3.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"project","domain":"domain","name":"test_serialization.my_wf","version":"version"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}]},{"id":"node-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}},{"id":"node-1","metadata":{"name":"test1","retries":{},"interruptible":false},"inputs":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-0.out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"upstreamNodeIds":["node-0"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"node-0.out_0"},"rightValue":{"var":".a"}}},"thenNode":{"id":"node-1-branchnode-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"upstreamNodeIds":["node-0"],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}}},"error":{"failedNodeId":"test1","message":"Unable to choose branch"}}}}],"outputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"node-0":{"ids":["node-1"]},"node-1":{"ids":["end-node"]},"start-node":{"ids":["node-0","node-1"]}},"upstream":{"branchnode-0":{"ids":["node-0"]},"end-node":{"ids":["node-1"]},"node-0":{"ids":["start-node"]},"node-1":{"ids":["node-0","start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_4.json b/flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_4.json new file mode 100755 index 0000000000..76e1c6bdc9 --- /dev/null +++ b/flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_4.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"project","domain":"domain","name":"test_serialization.my_wf","version":"version"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},{"id":"node-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}},{"id":"node-1","metadata":{"name":"test1","retries":{},"interruptible":false},"inputs":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-0.out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"upstreamNodeIds":["node-0"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"node-0.out_0"},"rightValue":{"var":".a"}}},"thenNode":{"id":"node-1-branchnode-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"upstreamNodeIds":["node-0"],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}}},"error":{"failedNodeId":"test1","message":"Unable to choose branch"}}}},{"id":"node-2","metadata":{"name":"test2","retries":{},"interruptible":false},"inputs":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-1.out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"upstreamNodeIds":["node-1"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"node-1.out_0"},"rightValue":{"var":".a"}}},"thenNode":{"id":"node-2-branchnode-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"upstreamNodeIds":["node-1"],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}}},"error":{"failedNodeId":"test2","message":"Unable to choose a branch"}}}}],"outputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"node-0":{"ids":["node-1"]},"node-1":{"ids":["node-2"]},"node-2":{"ids":["end-node"]},"start-node":{"ids":["node-0","node-1","node-2"]}},"upstream":{"branchnode-0":{"ids":["node-0","node-1"]},"end-node":{"ids":["node-2"]},"node-0":{"ids":["start-node"]},"node-1":{"ids":["node-0","start-node"]},"node-2":{"ids":["node-1","start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_5.json b/flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_5.json new file mode 100755 index 0000000000..3a5cdd1a1c --- /dev/null +++ b/flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_5.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"project","domain":"domain","name":"test_serialization.my_wf","version":"version"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"},"b":{"type":{"simple":"STRING"},"description":"b"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"},"out_1":{"type":{"simple":"STRING"},"description":"out_1"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}},{"var":"out_1","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},{"id":"node-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t1","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"}}},{"id":"node-1","metadata":{"name":"test1","retries":{},"interruptible":false},"inputs":[{"var":"node-0.t1_int_output","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}}],"upstreamNodeIds":["node-0"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"node-0.t1_int_output"},"rightValue":{"primitive":{"integer":"4"}}}},"thenNode":{"id":"node-1-branchnode-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t2","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"b"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"}}}},"other":[{"condition":{"comparison":{"operator":"GTE","leftValue":{"var":"node-0.t1_int_output"},"rightValue":{"primitive":{"integer":"5"}}}},"thenNode":{"id":"node-1-branchnode-1","metadata":{"name":"flytekit.annotated.task.test_serialization.t2","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"c"}}}],"upstreamNodeIds":["node-0"],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"}}}}],"error":{"failedNodeId":"test1","message":"Unable to choose branch"}}}},{"id":"node-2","metadata":{"name":"test2","retries":{},"interruptible":false},"inputs":[{"var":"node-1.out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"upstreamNodeIds":["node-1"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"node-1.out_0"},"rightValue":{"primitive":{"stringValue":"hello "}}}},"thenNode":{"id":"node-2-branchnode-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t2","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"scalar":{"primitive":{"stringValue":"It is hello"}}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"}}}},"elseNode":{"id":"node-2-branchnode-1","metadata":{"name":"flytekit.annotated.task.test_serialization.t2","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"scalar":{"primitive":{"stringValue":"Not Hello!"}}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"}}}}}}],"outputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}},{"var":"out_1","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"node-0":{"ids":["end-node","node-1"]},"node-1":{"ids":["node-2"]},"node-2":{"ids":["end-node"]},"start-node":{"ids":["node-0"]}},"upstream":{"branchnode-0":{"ids":["start-node"]},"branchnode-1":{"ids":["node-0"]},"end-node":{"ids":["node-0","node-2"]},"node-0":{"ids":["start-node"]},"node-1":{"ids":["node-0"]},"node-2":{"ids":["node-1"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"c":{"type":{"simple":"STRING"},"description":"c"},"t1_int_output":{"type":{"simple":"INTEGER"},"description":"t1_int_output"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"STRING"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_6.json b/flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_6.json new file mode 100755 index 0000000000..fd7e99c8c2 --- /dev/null +++ b/flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_6.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.multiplier","version":"abc"},"metadata":{},"interface":{"inputs":{"variables":{"my_input":{"type":{"simple":"FLOAT"},"description":"my_input"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"node-0","var":"o0"}}}]},{"id":"node-0","metadata":{"name":"fractions","retries":{},"interruptible":false},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GTE","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LTE","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"thenNode":{"id":"node-0-branchn0","metadata":{"name":"flytekit.annotated.python_function_task.recipes.02_intermediate.run_conditions.double","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.double","version":"abc"}}}},"elseNode":{"id":"node-0-branchn1","metadata":{"name":"flytekit.annotated.python_function_task.recipes.02_intermediate.run_conditions.square","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.square","version":"abc"}}}}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"node-0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"node-0":{"ids":["end-node"]},"start-node":{"ids":["node-0"]}},"upstream":{"branchn0":{"ids":["start-node"]},"branchn1":{"ids":["start-node"]},"end-node":{"ids":["node-0"]},"node-0":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","recipes.02_intermediate.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","recipes.02_intermediate.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json b/flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json new file mode 100755 index 0000000000..25844a542b --- /dev/null +++ b/flytepropeller/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.so_nested","version":"abc"},"metadata":{},"interface":{"inputs":{"variables":{"my_input":{"type":{"simple":"FLOAT"},"description":"my_input"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}]},{"id":"branchnode-2","metadata":{"name":"fractions","retries":{},"interruptible":false},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"thenNode":{"id":"branchnode-2-branchbranchnode-1","metadata":{"name":"inner_fractions","retries":{},"interruptible":false},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.5}}}},"thenNode":{"id":"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0","metadata":{"name":"flytekit.core.python_function_task.core.control_flow.run_conditions.double","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"}}}},"error":{"failedNodeId":"inner_fractions","message":"Only \u003c0.5 allowed"}}}}},"other":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":10}}}}}},"thenNode":{"id":"branchnode-2-branchn1","metadata":{"name":"flytekit.core.python_function_task.core.control_flow.run_conditions.square","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"}}}}],"error":{"failedNodeId":"fractions","message":"The input must be between 0 and 10"}}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"branchnode-2":{"ids":["end-node"]},"start-node":{"ids":["branchnode-2"]}},"upstream":{"branchbranchn0":{"ids":["start-node"]},"branchbranchnode-1":{"ids":["start-node"]},"branchbranchnode-1-branchbranchn0":{"ids":["start-node"]},"branchn1":{"ids":["start-node"]},"branchnode-2":{"ids":["start-node"]},"branchnode-2-branchbranchnode-1":{"ids":["start-node"]},"end-node":{"ids":["branchnode-2"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_1.json b/flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_1.json new file mode 100755 index 0000000000..183a121e00 --- /dev/null +++ b/flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_1.json @@ -0,0 +1 @@ +{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"test-serialization-my-wf"}},"spec":{"id":"project:domain:test_serialization.my_wf","nodes":{"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}]},"node-0":{"id":"node-0","name":"test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1":{"id":"node-1","name":"test1","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"out_0"},"rightValue":{"primitive":{"integer":"1"}}}},"then":"node-1-branchnode-0"},"elseFail":{"failedNodeId":"test1","message":"Unable to choose branch"}},"inputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1-branchnode-0":{"id":"node-1-branchnode-0","name":"test_serialization.t2","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ","retry":{"minAttempts":1},"interruptible":false},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"node-0":["node-1"],"node-1":["end-node"],"start-node":["node-0"]},"edges":{"downstream":{"node-0":["node-1"],"node-1":["end-node"],"start-node":["node-0"]},"upstream":{"end-node":["node-1"],"node-0":["start-node"],"node-1":["node-0"]}},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}},"outputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}]},"inputs":{"literals":{"a":{"scalar":{"primitive":{"integer":"0"}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file diff --git a/flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_2.json b/flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_2.json new file mode 100755 index 0000000000..44afd3822f --- /dev/null +++ b/flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_2.json @@ -0,0 +1 @@ +{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"test-serialization-my-wf"}},"spec":{"id":"project:domain:test_serialization.my_wf","nodes":{"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},"node-0":{"id":"node-0","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1":{"id":"node-1","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2":{"id":"node-2","name":"test1","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"node-0.out_0"},"rightValue":{"var":"node-1.out_0"}}},"then":"node-2-branchnode-0"},"elseFail":{"failedNodeId":"test1","message":"Unable to choose branch"}},"inputBindings":[{"var":"node-0.out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}},{"var":"node-1.out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2-branchnode-0":{"id":"node-2-branchnode-0","name":"flytekit.annotated.task.test_serialization.t2","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ","retry":{"minAttempts":1},"interruptible":false},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"node-0":["node-2"],"node-1":["node-2"],"node-2":["end-node"],"start-node":["node-0","node-1"]},"edges":{"downstream":{"node-0":["node-2"],"node-1":["node-2"],"node-2":["end-node"],"start-node":["node-0","node-1"]},"upstream":{"end-node":["node-2"],"node-0":["start-node"],"node-1":["start-node"],"node-2":["node-0","node-1"]}},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}},"outputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},"inputs":{"literals":{"a":{"scalar":{"primitive":{"integer":"0"}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file diff --git a/flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_3.json b/flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_3.json new file mode 100755 index 0000000000..f66f7260b8 --- /dev/null +++ b/flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_3.json @@ -0,0 +1 @@ +{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"test-serialization-my-wf"}},"spec":{"id":"project:domain:test_serialization.my_wf","nodes":{"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}]},"node-0":{"id":"node-0","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1":{"id":"node-1","name":"test1","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"node-0.out_0"},"rightValue":{"var":".a"}}},"then":"node-1-branchnode-0"},"elseFail":{"failedNodeId":"test1","message":"Unable to choose branch"}},"inputBindings":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-0.out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1-branchnode-0":{"id":"node-1-branchnode-0","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"node-0":["node-1"],"node-1":["end-node"],"start-node":["node-0","node-1"]},"edges":{"downstream":{"node-0":["node-1"],"node-1":["end-node"],"start-node":["node-0","node-1"]},"upstream":{"branchnode-0":["node-0"],"end-node":["node-1"],"node-0":["start-node"],"node-1":["node-0","start-node"]}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}},"outputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}]},"inputs":{"literals":{"a":{"scalar":{"primitive":{"integer":"0"}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file diff --git a/flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_4.json b/flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_4.json new file mode 100755 index 0000000000..1be4946c2a --- /dev/null +++ b/flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_4.json @@ -0,0 +1 @@ +{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"test-serialization-my-wf"}},"spec":{"id":"project:domain:test_serialization.my_wf","nodes":{"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},"node-0":{"id":"node-0","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1":{"id":"node-1","name":"test1","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"node-0.out_0"},"rightValue":{"var":".a"}}},"then":"node-1-branchnode-0"},"elseFail":{"failedNodeId":"test1","message":"Unable to choose branch"}},"inputBindings":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-0.out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1-branchnode-0":{"id":"node-1-branchnode-0","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2":{"id":"node-2","name":"test2","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"node-1.out_0"},"rightValue":{"var":".a"}}},"then":"node-2-branchnode-0"},"elseFail":{"failedNodeId":"test2","message":"Unable to choose a branch"}},"inputBindings":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-1.out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2-branchnode-0":{"id":"node-2-branchnode-0","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"node-0":["node-1"],"node-1":["node-2"],"node-2":["end-node"],"start-node":["node-0","node-1","node-2"]},"edges":{"downstream":{"node-0":["node-1"],"node-1":["node-2"],"node-2":["end-node"],"start-node":["node-0","node-1","node-2"]},"upstream":{"branchnode-0":["node-0","node-1"],"end-node":["node-2"],"node-0":["start-node"],"node-1":["node-0","start-node"],"node-2":["node-1","start-node"]}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}},"outputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},"inputs":{"literals":{"a":{"scalar":{"primitive":{"integer":"0"}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file diff --git a/flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_5.json b/flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_5.json new file mode 100755 index 0000000000..281d01a276 --- /dev/null +++ b/flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_5.json @@ -0,0 +1 @@ +{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"test-serialization-my-wf"}},"spec":{"id":"project:domain:test_serialization.my_wf","nodes":{"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}},{"var":"out_1","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},"node-0":{"id":"node-0","name":"flytekit.annotated.task.test_serialization.t1","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1":{"id":"node-1","name":"test1","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"node-0.t1_int_output"},"rightValue":{"primitive":{"integer":"4"}}}},"then":"node-1-branchnode-0"},"elseIf":[{"condition":{"comparison":{"operator":"GTE","leftValue":{"var":"node-0.t1_int_output"},"rightValue":{"primitive":{"integer":"5"}}}},"then":"node-1-branchnode-1"}],"elseFail":{"failedNodeId":"test1","message":"Unable to choose branch"}},"inputBindings":[{"var":"node-0.t1_int_output","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1-branchnode-0":{"id":"node-1-branchnode-0","name":"flytekit.annotated.task.test_serialization.t2","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"b"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1-branchnode-1":{"id":"node-1-branchnode-1","name":"flytekit.annotated.task.test_serialization.t2","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"c"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2":{"id":"node-2","name":"test2","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"node-1.out_0"},"rightValue":{"primitive":{"stringValue":"hello "}}}},"then":"node-2-branchnode-0"},"else":"node-2-branchnode-1"},"inputBindings":[{"var":"node-1.out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2-branchnode-0":{"id":"node-2-branchnode-0","name":"flytekit.annotated.task.test_serialization.t2","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"scalar":{"primitive":{"stringValue":"It is hello"}}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2-branchnode-1":{"id":"node-2-branchnode-1","name":"flytekit.annotated.task.test_serialization.t2","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"scalar":{"primitive":{"stringValue":"Not Hello!"}}}}],"retry":{"minAttempts":1},"interruptible":false},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"node-0":["end-node","node-1"],"node-1":["node-2"],"node-2":["end-node"],"start-node":["node-0"]},"edges":{"downstream":{"node-0":["end-node","node-1"],"node-1":["node-2"],"node-2":["end-node"],"start-node":["node-0"]},"upstream":{"branchnode-0":["start-node"],"branchnode-1":["node-0"],"end-node":["node-0","node-2"],"node-0":["start-node"],"node-1":["node-0"],"node-2":["node-1"]}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"},"out_1":{"type":{"simple":"STRING"},"description":"out_1"}}},"outputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}},{"var":"out_1","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},"inputs":{"literals":{"a":{"scalar":{"primitive":{"integer":"0"}}},"b":{"scalar":{"primitive":{"stringValue":""}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"c":{"type":{"simple":"STRING"},"description":"c"},"t1_int_output":{"type":{"simple":"INTEGER"},"description":"t1_int_output"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"STRING"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file diff --git a/flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_6.json b/flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_6.json new file mode 100755 index 0000000000..961211bcb3 --- /dev/null +++ b/flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_6.json @@ -0,0 +1 @@ +{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"recipes-02-intermediate-run-conditions-multiplier"}},"spec":{"id":"test_proj:test_domain:recipes.02_intermediate.run_conditions.multiplier","nodes":{"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"node-0","var":"o0"}}}]},"node-0":{"id":"node-0","name":"fractions","resources":{},"kind":"branch","branch":{"if":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GTE","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LTE","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"then":"node-0-branchn0"},"else":"node-0-branchn1"},"inputBindings":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-0-branchn0":{"id":"node-0-branchn0","name":"flytekit.annotated.python_function_task.recipes.02_intermediate.run_conditions.double","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"recipes.02_intermediate.run_conditions.double\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-0-branchn1":{"id":"node-0-branchn1","name":"flytekit.annotated.python_function_task.recipes.02_intermediate.run_conditions.square","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"recipes.02_intermediate.run_conditions.square\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"node-0":["end-node"],"start-node":["node-0"]},"edges":{"downstream":{"node-0":["end-node"],"start-node":["node-0"]},"upstream":{"branchn0":["start-node"],"branchn1":["start-node"],"end-node":["node-0"],"node-0":["start-node"]}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}},"outputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"node-0","var":"o0"}}}]},"inputs":{"literals":{"my_input":{"scalar":{"primitive":{"floatValue":0}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"recipes.02_intermediate.run_conditions.double\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","recipes.02_intermediate.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"recipes.02_intermediate.run_conditions.square\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","recipes.02_intermediate.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file diff --git a/flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json b/flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json new file mode 100755 index 0000000000..4dc8500dd9 --- /dev/null +++ b/flytepropeller/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json @@ -0,0 +1 @@ +{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"core-control-flow-run-conditions-so-nested"}},"spec":{"id":"test_proj:test_domain:core.control_flow.run_conditions.so_nested","nodes":{"branchnode-2":{"id":"branchnode-2","name":"fractions","resources":{},"kind":"branch","branch":{"if":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"then":"branchnode-2-branchbranchnode-1"},"elseIf":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":10}}}}}},"then":"branchnode-2-branchn1"}],"elseFail":{"failedNodeId":"fractions","message":"The input must be between 0 and 10"}},"inputBindings":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"branchnode-2-branchbranchnode-1":{"id":"branchnode-2-branchbranchnode-1","name":"inner_fractions","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.5}}}},"then":"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0"},"elseFail":{"failedNodeId":"inner_fractions","message":"Only \u003c0.5 allowed"}},"inputBindings":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0":{"id":"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0","name":"flytekit.core.python_function_task.core.control_flow.run_conditions.double","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"branchnode-2-branchn1":{"id":"branchnode-2-branchn1","name":"flytekit.core.python_function_task.core.control_flow.run_conditions.square","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}]},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"branchnode-2":["end-node"],"start-node":["branchnode-2"]},"edges":{"downstream":{"branchnode-2":["end-node"],"start-node":["branchnode-2"]},"upstream":{"branchbranchn0":["start-node"],"branchbranchnode-1":["start-node"],"branchbranchnode-1-branchbranchn0":["start-node"],"branchn1":["start-node"],"branchnode-2":["start-node"],"branchnode-2-branchbranchnode-1":["start-node"],"end-node":["branchnode-2"]}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}},"outputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}]},"inputs":{"literals":{"my_input":{"scalar":{"primitive":{"floatValue":0}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file diff --git a/flytepropeller/pkg/compiler/test/testdata/branch/success_7_nested.json b/flytepropeller/pkg/compiler/test/testdata/branch/success_7_nested.json new file mode 100644 index 0000000000..ea03a725d1 --- /dev/null +++ b/flytepropeller/pkg/compiler/test/testdata/branch/success_7_nested.json @@ -0,0 +1,363 @@ +{ + "workflow": { + "id": { + "resourceType": "WORKFLOW", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.so_nested", + "version": "abc" + }, + "metadata": { + }, + "interface": { + "inputs": { + "variables": { + "my_input": { + "type": { + "simple": "FLOAT" + }, + "description": "my_input" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "nodes": [ + { + "id": "branchnode-2", + "metadata": { + "name": "fractions", + "retries": { + }, + "interruptible": false + }, + "inputs": [ + { + "var": ".my_input", + "binding": { + "promise": { + "var": "my_input" + } + } + } + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.1 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 1 + } + } + } + } + } + }, + "thenNode": { + "id": "branchbranchnode-1", + "metadata": { + "name": "inner_fractions", + "retries": { + }, + "interruptible": false + }, + "inputs": [ + { + "var": ".my_input", + "binding": { + "promise": { + "var": "my_input" + } + } + } + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.5 + } + } + } + }, + "thenNode": { + "id": "branchbranchn0", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", + "retries": { + }, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.double", + "version": "abc" + } + } + } + }, + "error": { + "failedNodeId": "inner_fractions", + "message": "Only <0.5 allowed" + } + } + } + } + }, + "other": [ + { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 1 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 10 + } + } + } + } + } + }, + "thenNode": { + "id": "branchn1", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.square", + "retries": { + }, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.square", + "version": "abc" + } + } + } + } + ], + "error": { + "failedNodeId": "fractions", + "message": "The input must be between 0 and 10" + } + } + } + } + ], + "outputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "branchnode-2", + "var": "o0" + } + } + } + ], + "metadataDefaults": { + } + }, + "tasks": [ + { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.double", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + } + } + }, + { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.square", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "square", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + } + } + } + ] +} \ No newline at end of file diff --git a/flytepropeller/pkg/compiler/transformers/k8s/node.go b/flytepropeller/pkg/compiler/transformers/k8s/node.go index e2d9aa7faa..3f5ef5312a 100644 --- a/flytepropeller/pkg/compiler/transformers/k8s/node.go +++ b/flytepropeller/pkg/compiler/transformers/k8s/node.go @@ -10,7 +10,7 @@ import ( ) // Gets the compiled subgraph if this node contains an inline-declared coreWorkflow. Otherwise nil. -func buildNodeSpec(n *core.Node, tasks []*core.CompiledTask, errs errors.CompileErrors) (*v1alpha1.NodeSpec, bool) { +func buildNodeSpec(n *core.Node, tasks []*core.CompiledTask, errs errors.CompileErrors) ([]*v1alpha1.NodeSpec, bool) { if n == nil { errs.Collect(errors.NewValueRequiredErr("root", "node")) return nil, !errs.HasErrors() @@ -112,7 +112,12 @@ func buildNodeSpec(n *core.Node, tasks []*core.CompiledTask, errs errors.Compile } case *core.Node_BranchNode: nodeSpec.Kind = v1alpha1.NodeKindBranch - nodeSpec.BranchNode = buildBranchNodeSpec(n.GetBranchNode(), errs.NewScope()) + b, ns := buildBranchNodeSpec(n.GetBranchNode(), tasks, errs.NewScope()) + nodeSpec.BranchNode = b + // The reason why we create a separate list and then append the passed list to it is to maintain the actualNode + // as the first element in the list. That way list[0] will always be the first node + actualNode := []*v1alpha1.NodeSpec{nodeSpec} + return append(actualNode, ns...), !errs.HasErrors() default: if n.GetId() == v1alpha1.StartNodeID { nodeSpec.Kind = v1alpha1.NodeKindStart @@ -121,27 +126,40 @@ func buildNodeSpec(n *core.Node, tasks []*core.CompiledTask, errs errors.Compile } } - return nodeSpec, !errs.HasErrors() + return []*v1alpha1.NodeSpec{nodeSpec}, !errs.HasErrors() } -func buildIfBlockSpec(block *core.IfBlock, _ errors.CompileErrors) *v1alpha1.IfBlock { +func buildIfBlockSpec(block *core.IfBlock, tasks []*core.CompiledTask, errs errors.CompileErrors) (*v1alpha1.IfBlock, []*v1alpha1.NodeSpec) { + nodeSpecs, ok := buildNodeSpec(block.ThenNode, tasks, errs) + if !ok { + return nil, []*v1alpha1.NodeSpec{} + } return &v1alpha1.IfBlock{ Condition: v1alpha1.BooleanExpression{BooleanExpression: block.Condition}, ThenNode: refStr(block.ThenNode.Id), - } + }, nodeSpecs } -func buildBranchNodeSpec(branch *core.BranchNode, errs errors.CompileErrors) *v1alpha1.BranchNodeSpec { +func buildBranchNodeSpec(branch *core.BranchNode, tasks []*core.CompiledTask, errs errors.CompileErrors) (*v1alpha1.BranchNodeSpec, []*v1alpha1.NodeSpec) { if branch == nil { - return nil + return nil, []*v1alpha1.NodeSpec{} } + var childNodes []*v1alpha1.NodeSpec + + branchNode, nodeSpecs := buildIfBlockSpec(branch.IfElse.Case, tasks, errs.NewScope()) res := &v1alpha1.BranchNodeSpec{ - If: *buildIfBlockSpec(branch.IfElse.Case, errs.NewScope()), + If: *branchNode, } + childNodes = append(childNodes, nodeSpecs...) switch branch.IfElse.GetDefault().(type) { case *core.IfElseBlock_ElseNode: + ns, ok := buildNodeSpec(branch.IfElse.GetElseNode(), tasks, errs) + if !ok { + return nil, []*v1alpha1.NodeSpec{} + } + childNodes = append(childNodes, ns...) res.Else = refStr(branch.IfElse.GetElseNode().Id) case *core.IfElseBlock_Error: res.ElseFail = &v1alpha1.Error{Error: branch.IfElse.GetError()} @@ -149,27 +167,32 @@ func buildBranchNodeSpec(branch *core.BranchNode, errs errors.CompileErrors) *v1 other := make([]*v1alpha1.IfBlock, 0, len(branch.IfElse.Other)) for _, block := range branch.IfElse.Other { - other = append(other, buildIfBlockSpec(block, errs.NewScope())) + b, ns := buildIfBlockSpec(block, tasks, errs.NewScope()) + other = append(other, b) + childNodes = append(childNodes, ns...) } res.ElseIf = other - return res + return res, childNodes } func buildNodes(nodes []*core.Node, tasks []*core.CompiledTask, errs errors.CompileErrors) (map[common.NodeID]*v1alpha1.NodeSpec, bool) { res := make(map[common.NodeID]*v1alpha1.NodeSpec, len(nodes)) - for _, nodeBuidler := range nodes { - n, ok := buildNodeSpec(nodeBuidler, tasks, errs.NewScope()) + for _, nodeBuilder := range nodes { + nodeSpecs, ok := buildNodeSpec(nodeBuilder, tasks, errs.NewScope()) if !ok { return nil, ok } - if _, exists := res[n.ID]; exists { - errs.Collect(errors.NewValueCollisionError(nodeBuidler.GetId(), "Id", n.ID)) - } + for _, nref := range nodeSpecs { + n := nref + if _, exists := res[n.ID]; exists { + errs.Collect(errors.NewValueCollisionError(nodeBuilder.GetId(), "Id", n.ID)) + } - res[n.ID] = n + res[n.ID] = n + } } return res, !errs.HasErrors() diff --git a/flytepropeller/pkg/compiler/transformers/k8s/node_test.go b/flytepropeller/pkg/compiler/transformers/k8s/node_test.go index 0f05c899d2..0816d7a020 100644 --- a/flytepropeller/pkg/compiler/transformers/k8s/node_test.go +++ b/flytepropeller/pkg/compiler/transformers/k8s/node_test.go @@ -60,8 +60,10 @@ func TestBuildNodeSpec(t *testing.T) { errors.SetConfig(errors.Config{IncludeSource: true}) errs := errors.NewCompileErrors() - mustBuild := func(n common.Node, errs errors.CompileErrors) *v1alpha1.NodeSpec { - spec, ok := buildNodeSpec(n.GetCoreNode(), tasks, errs) + mustBuild := func(t testing.TB, n common.Node, expectedInnerNodesCount int, errs errors.CompileErrors) *v1alpha1.NodeSpec { + specs, ok := buildNodeSpec(n.GetCoreNode(), tasks, errs) + assert.Len(t, specs, expectedInnerNodesCount) + spec := specs[0] assert.Nil(t, spec.Interruptibe) assert.False(t, errs.HasErrors()) assert.True(t, ok) @@ -83,7 +85,7 @@ func TestBuildNodeSpec(t *testing.T) { }, } - mustBuild(n, errs.NewScope()) + mustBuild(t, n, 1, errs.NewScope()) }) t.Run("Task with resources", func(t *testing.T) { @@ -96,7 +98,7 @@ func TestBuildNodeSpec(t *testing.T) { }, } - spec := mustBuild(n, errs.NewScope()) + spec := mustBuild(t, n, 1, errs.NewScope()) assert.NotNil(t, spec.Resources) assert.NotNil(t, spec.Resources.Requests.Cpu()) assert.Equal(t, expectedCPU.Value(), spec.Resources.Requests.Cpu().Value()) @@ -111,7 +113,7 @@ func TestBuildNodeSpec(t *testing.T) { }, } - mustBuild(n, errs.NewScope()) + mustBuild(t, n, 1, errs.NewScope()) }) t.Run("Workflow", func(t *testing.T) { @@ -124,7 +126,7 @@ func TestBuildNodeSpec(t *testing.T) { }, } - mustBuild(n, errs.NewScope()) + mustBuild(t, n, 1, errs.NewScope()) }) t.Run("Branch", func(t *testing.T) { @@ -177,7 +179,7 @@ func TestBuildNodeSpec(t *testing.T) { }, } - mustBuild(n, errs.NewScope()) + mustBuild(t, n, 2, errs.NewScope()) }) } diff --git a/flytepropeller/pkg/compiler/transformers/k8s/testdata/compiled_closure_branch_nested.json b/flytepropeller/pkg/compiler/transformers/k8s/testdata/compiled_closure_branch_nested.json new file mode 100644 index 0000000000..1140b3f233 --- /dev/null +++ b/flytepropeller/pkg/compiler/transformers/k8s/testdata/compiled_closure_branch_nested.json @@ -0,0 +1,538 @@ +{ + "primary": { + "template": { + "id": { + "resourceType": "WORKFLOW", + "project": "flytesnacks", + "domain": "development", + "name": "core.control_flow.run_conditions.so_nested", + "version": "v1" + }, + "metadata": {}, + "interface": { + "inputs": { + "variables": { + "my_input": { + "type": { + "simple": "FLOAT" + }, + "description": "my_input" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "nodes": [ + { + "id": "start-node" + }, + { + "id": "end-node", + "inputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ] + }, + { + "id": "n0", + "metadata": { + "name": "fractions", + "retries": {}, + "interruptible": false + }, + "inputs": [ + { + "var": ".my_input", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.1 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 1 + } + } + } + } + } + }, + "thenNode": { + "id": "n0-n0", + "metadata": { + "name": "inner_fractions", + "retries": {}, + "interruptible": false + }, + "inputs": [ + { + "var": ".my_input", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.5 + } + } + } + }, + "thenNode": { + "id": "n0-n0-n0-n0", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", + "retries": {}, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "flytesnacks", + "domain": "development", + "name": "core.control_flow.run_conditions.double", + "version": "v1" + } + } + } + }, + "other": [ + { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.5 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.7 + } + } + } + } + } + }, + "thenNode": { + "id": "n0-n0-n0-n1", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.square", + "retries": {}, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "flytesnacks", + "domain": "development", + "name": "core.control_flow.run_conditions.square", + "version": "v1" + } + } + } + } + ], + "error": { + "failedNodeId": "inner_fractions", + "message": "Only \u003c0.7 allowed" + } + } + } + } + }, + "other": [ + { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 1 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 10 + } + } + } + } + } + }, + "thenNode": { + "id": "n0-n1", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.square", + "retries": {}, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "flytesnacks", + "domain": "development", + "name": "core.control_flow.run_conditions.square", + "version": "v1" + } + } + } + } + ], + "elseNode": { + "id": "n0-n2", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", + "retries": {}, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "flytesnacks", + "domain": "development", + "name": "core.control_flow.run_conditions.double", + "version": "v1" + } + } + } + } + } + } + ], + "outputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ], + "metadataDefaults": {} + }, + "connections": { + "downstream": { + "n0": { + "ids": [ + "end-node" + ] + }, + "start-node": { + "ids": [ + "n0" + ] + } + }, + "upstream": { + "end-node": { + "ids": [ + "n0" + ] + }, + "n0": { + "ids": [ + "start-node" + ] + }, + "n0-n0": { + "ids": [ + "start-node" + ] + }, + "n0-n1": { + "ids": [ + "start-node" + ] + }, + "n1": { + "ids": [ + "start-node" + ] + }, + "n2": { + "ids": [ + "start-node" + ] + } + } + } + }, + "tasks": [ + { + "template": { + "id": { + "resourceType": "TASK", + "project": "flytesnacks", + "domain": "development", + "name": "core.control_flow.run_conditions.double", + "version": "v1" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "flytecookbook:core-d5fa3ecfaca02f9b83957c68fd5fe3c9082ccc59", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "core.control_flow.run_conditions", + "task-name", + "double" + ], + "resources": {}, + "env": [ + { + "key": "FLYTE_INTERNAL_CONFIGURATION_PATH", + "value": "/root/sandbox.config" + }, + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "flytecookbook:core-d5fa3ecfaca02f9b83957c68fd5fe3c9082ccc59" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "flytesnacks", + "domain": "development", + "name": "core.control_flow.run_conditions.square", + "version": "v1" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "flytecookbook:core-d5fa3ecfaca02f9b83957c68fd5fe3c9082ccc59", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "core.control_flow.run_conditions", + "task-name", + "square" + ], + "resources": {}, + "env": [ + { + "key": "FLYTE_INTERNAL_CONFIGURATION_PATH", + "value": "/root/sandbox.config" + }, + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "flytecookbook:core-d5fa3ecfaca02f9b83957c68fd5fe3c9082ccc59" + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/flytepropeller/pkg/compiler/transformers/k8s/workflow.go b/flytepropeller/pkg/compiler/transformers/k8s/workflow.go index 580424e131..a6f92aa59c 100644 --- a/flytepropeller/pkg/compiler/transformers/k8s/workflow.go +++ b/flytepropeller/pkg/compiler/transformers/k8s/workflow.go @@ -69,7 +69,11 @@ func buildFlyteWorkflowSpec(wf *core.CompiledWorkflow, tasks []*core.CompiledTas spec *v1alpha1.WorkflowSpec, err error) { var failureN *v1alpha1.NodeSpec if n := wf.Template.GetFailureNode(); n != nil { - failureN, _ = buildNodeSpec(n, tasks, errs.NewScope()) + nodes, ok := buildNodeSpec(n, tasks, errs.NewScope()) + if !ok { + return + } + failureN = nodes[0] } nodes, _ := buildNodes(wf.Template.GetNodes(), tasks, errs.NewScope()) @@ -97,14 +101,19 @@ func buildFlyteWorkflowSpec(wf *core.CompiledWorkflow, tasks []*core.CompiledTas failurePolicy = v1alpha1.WorkflowOnFailurePolicy(wf.Template.Metadata.OnFailure) } + connections := buildConnections(wf) return &v1alpha1.WorkflowSpec{ ID: WorkflowIDAsString(wf.Template.Id), OnFailure: failureN, Nodes: nodes, - Connections: buildConnections(wf), Outputs: outputs, OutputBindings: outputBindings, OnFailurePolicy: failurePolicy, + Connections: connections, + DeprecatedConnections: v1alpha1.DeprecatedConnections{ + DownstreamEdges: connections.Downstream, + UpstreamEdges: connections.Upstream, + }, }, nil } @@ -206,7 +215,7 @@ func BuildFlyteWorkflow(wfClosure *core.CompiledWorkflowClosure, inputs *core.Li } obj.ObjectMeta.Labels[WorkflowNameLabel] = utils.SanitizeLabelValue(WorkflowNameFromID(primarySpec.ID)) - if obj.Nodes == nil || obj.Connections.DownstreamEdges == nil { + if obj.Nodes == nil || obj.Connections.Downstream == nil { // If we come here, we'd better have an error generated earlier. Otherwise, add one to make sure build fails. if !errs.HasErrors() { errs.Collect(errors.NewWorkflowBuildError(fmt.Errorf("failed to build workflow for unknown reason." + @@ -236,7 +245,7 @@ func toMapOfLists(connections map[string]*core.ConnectionSet_IdList) map[string] func buildConnections(w *core.CompiledWorkflow) v1alpha1.Connections { res := v1alpha1.Connections{} - res.DownstreamEdges = toMapOfLists(w.GetConnections().GetDownstream()) - res.UpstreamEdges = toMapOfLists(w.GetConnections().GetUpstream()) + res.Downstream = toMapOfLists(w.GetConnections().GetDownstream()) + res.Upstream = toMapOfLists(w.GetConnections().GetUpstream()) return res } diff --git a/flytepropeller/pkg/compiler/transformers/k8s/workflow_test.go b/flytepropeller/pkg/compiler/transformers/k8s/workflow_test.go index 2316400f30..b52594ff88 100644 --- a/flytepropeller/pkg/compiler/transformers/k8s/workflow_test.go +++ b/flytepropeller/pkg/compiler/transformers/k8s/workflow_test.go @@ -1,12 +1,15 @@ package k8s import ( + "bytes" + "io/ioutil" "testing" "github.com/flyteorg/flyteidl/clients/go/coreutils" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" "github.com/flyteorg/flytepropeller/pkg/compiler/common" "github.com/flyteorg/flytepropeller/pkg/compiler/errors" + "github.com/golang/protobuf/jsonpb" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/util/sets" ) @@ -243,3 +246,40 @@ func TestGenerateName(t *testing.T) { assert.Equal(t, "myexecution", name) }) } + +func TestBuildFlyteWorkflow_withBranch(t *testing.T) { + c, err := ioutil.ReadFile("testdata/compiled_closure_branch_nested.json") + assert.NoError(t, err) + + r := bytes.NewReader(c) + + w := &core.CompiledWorkflowClosure{} + assert.NoError(t, jsonpb.Unmarshal(r, w)) + + assert.Len(t, w.Primary.Connections.Downstream, 2) + ids := w.Primary.Connections.Downstream["start-node"] + assert.Len(t, ids.Ids, 1) + assert.Equal(t, ids.Ids[0], "n0") + + wf, err := BuildFlyteWorkflow( + w, + &core.LiteralMap{ + Literals: map[string]*core.Literal{ + "my_input": coreutils.MustMakePrimitiveLiteral(1.0), + }, + }, &core.WorkflowExecutionIdentifier{ + Project: "p", + Domain: "d", + Name: "n", + }, "ns") + + assert.NoError(t, err) + assert.NotNil(t, wf) + assert.Len(t, wf.Nodes, 8) + + s := sets.NewString("start-node", "end-node", "n0", "n0-n0", "n0-n1", "n0-n2", "n0-n0-n0", "n0-n0-n0-n0", "n0-n0-n0-n1") + + for _, n := range wf.Nodes { + assert.True(t, s.Has(n.ID), "nodeId: %s for node: %s not found", n.ID, n.Name) + } +} diff --git a/flytepropeller/pkg/compiler/validators/bindings.go b/flytepropeller/pkg/compiler/validators/bindings.go index 93d8c43ab8..3016838b93 100644 --- a/flytepropeller/pkg/compiler/validators/bindings.go +++ b/flytepropeller/pkg/compiler/validators/bindings.go @@ -99,8 +99,16 @@ func validateBinding(w c.WorkflowBuilder, nodeID c.NodeID, nodeParam string, bin return nil, nil, !errs.HasErrors() } +type EdgeDirection uint8 + +const ( + EdgeDirectionBidirectional EdgeDirection = iota + EdgeDirectionDownstream + EdgeDirectionUpstream +) + func ValidateBindings(w c.WorkflowBuilder, node c.Node, bindings []*flyte.Binding, params *flyte.VariableMap, - validateParamTypes bool, errs errors.CompileErrors) (resolved *flyte.VariableMap, ok bool) { + validateParamTypes bool, edgeDirection EdgeDirection, errs errors.CompileErrors) (resolved *flyte.VariableMap, ok bool) { resolved = &flyte.VariableMap{ Variables: make(map[string]*flyte.Variable, len(bindings)), @@ -127,7 +135,14 @@ func ValidateBindings(w c.WorkflowBuilder, node c.Node, bindings []*flyte.Bindin param.Type, errs.NewScope()); bindingOk { for _, upNode := range upstreamNodes { // Add implicit Edges - w.AddExecutionEdge(upNode, node.GetId()) + switch edgeDirection { + case EdgeDirectionBidirectional: + w.AddExecutionEdge(upNode, node.GetId()) + case EdgeDirectionDownstream: + w.AddDownstreamEdge(upNode, node.GetId()) + case EdgeDirectionUpstream: + w.AddUpstreamEdge(upNode, node.GetId()) + } } resolved.Variables[binding.GetVar()] = &flyte.Variable{ diff --git a/flytepropeller/pkg/compiler/validators/bindings_test.go b/flytepropeller/pkg/compiler/validators/bindings_test.go index eb9cf5d1db..9c5f2bc742 100644 --- a/flytepropeller/pkg/compiler/validators/bindings_test.go +++ b/flytepropeller/pkg/compiler/validators/bindings_test.go @@ -19,7 +19,7 @@ func TestValidateBindings(t *testing.T) { bindings := []*core.Binding{} vars := &core.VariableMap{} compileErrors := compilerErrors.NewCompileErrors() - resolved, ok := ValidateBindings(wf, n, bindings, vars, true, compileErrors) + resolved, ok := ValidateBindings(wf, n, bindings, vars, true, EdgeDirectionBidirectional, compileErrors) assert.True(t, ok) assert.Empty(t, resolved.Variables) }) @@ -35,7 +35,7 @@ func TestValidateBindings(t *testing.T) { } vars := &core.VariableMap{} compileErrors := compilerErrors.NewCompileErrors() - _, ok := ValidateBindings(wf, n, bindings, vars, true, compileErrors) + _, ok := ValidateBindings(wf, n, bindings, vars, true, EdgeDirectionBidirectional, compileErrors) assert.False(t, ok) if !compileErrors.HasErrors() { assert.Error(t, compileErrors) @@ -67,7 +67,7 @@ func TestValidateBindings(t *testing.T) { } compileErrors := compilerErrors.NewCompileErrors() - _, ok := ValidateBindings(wf, n, bindings, vars, true, compileErrors) + _, ok := ValidateBindings(wf, n, bindings, vars, true, EdgeDirectionBidirectional, compileErrors) assert.False(t, ok) if !compileErrors.HasErrors() { assert.Error(t, compileErrors) @@ -96,7 +96,7 @@ func TestValidateBindings(t *testing.T) { } compileErrors := compilerErrors.NewCompileErrors() - _, ok := ValidateBindings(wf, n, bindings, vars, true, compileErrors) + _, ok := ValidateBindings(wf, n, bindings, vars, true, EdgeDirectionBidirectional, compileErrors) assert.True(t, ok) if compileErrors.HasErrors() { assert.NoError(t, compileErrors) @@ -130,7 +130,7 @@ func TestValidateBindings(t *testing.T) { } compileErrors := compilerErrors.NewCompileErrors() - _, ok := ValidateBindings(wf, n, bindings, vars, true, compileErrors) + _, ok := ValidateBindings(wf, n, bindings, vars, true, EdgeDirectionBidirectional, compileErrors) assert.True(t, ok) if compileErrors.HasErrors() { assert.NoError(t, compileErrors) @@ -192,7 +192,7 @@ func TestValidateBindings(t *testing.T) { } compileErrors := compilerErrors.NewCompileErrors() - _, ok := ValidateBindings(wf, n, bindings, vars, true, compileErrors) + _, ok := ValidateBindings(wf, n, bindings, vars, true, EdgeDirectionBidirectional, compileErrors) assert.True(t, ok) if compileErrors.HasErrors() { assert.NoError(t, compileErrors) diff --git a/flytepropeller/pkg/compiler/validators/branch.go b/flytepropeller/pkg/compiler/validators/branch.go index 536626258f..66084b556b 100644 --- a/flytepropeller/pkg/compiler/validators/branch.go +++ b/flytepropeller/pkg/compiler/validators/branch.go @@ -28,7 +28,6 @@ func validateBranchInterface(w c.WorkflowBuilder, node c.NodeBuilder, errs error return } - finalInputParameterNames := sets.NewString() finalOutputParameterNames := sets.NewString() var outputs map[string]*flyte.Variable @@ -62,34 +61,40 @@ func validateBranchInterface(w c.WorkflowBuilder, node c.NodeBuilder, errs error for _, block := range cases { n := w.NewNodeBuilder(block) + iface2, ok := ValidateUnderlyingInterface(w, n, errs.NewScope()) + + if !ok { + continue + } + + ValidateBindings(w, n, n.GetInputs(), &flyte.VariableMap{Variables: map[string]*flyte.Variable{}}, + false, EdgeDirectionUpstream, errs.NewScope()) + + // Clear out the Inputs. We do not care if the inputs of each of the underlying nodes + // match. We will pull the inputs needed for the underlying branch node at runtime. + iface2 = &flyte.TypedInterface{ + Inputs: &flyte.VariableMap{Variables: map[string]*flyte.Variable{}}, + Outputs: iface2.Outputs, + } + if iface == nil { - // if this is the first node to validate, just assume all other nodes will match the interface - if iface, ok = ValidateUnderlyingInterface(w, n, errs.NewScope()); ok { - // Clear out the Inputs. We do not care if the inputs of each of the underlying nodes - // match. We will pull the inputs needed for the underlying branch node at runtime. - iface = &flyte.TypedInterface{ - Inputs: &flyte.VariableMap{Variables: map[string]*flyte.Variable{}}, - Outputs: iface.Outputs, - } - - outputs, outputsSet = buildVariablesIndex(iface.Outputs) - finalOutputParameterNames = finalOutputParameterNames.Union(outputsSet) - } + iface = iface2 + outputs, outputsSet = buildVariablesIndex(iface.Outputs) + finalOutputParameterNames = finalOutputParameterNames.Union(outputsSet) } else { - if iface2, ok2 := ValidateUnderlyingInterface(w, n, errs.NewScope()); ok2 { - iface2 = &flyte.TypedInterface{ - Inputs: &flyte.VariableMap{Variables: map[string]*flyte.Variable{}}, - Outputs: iface2.Outputs, - } - - validateIfaceMatch(n.GetId(), iface2, errs.NewScope()) - } + validateIfaceMatch(n.GetId(), iface2, errs.NewScope()) } } + // Discover inputs from bindings... these should include all the variables used in the conditional statements. + // When we come to validate the conditions themselves, we will look up these variables and fail if a variable is used + // in a condition but doesn't have a node input binding. + inputVarsFromBindings, _ := ValidateBindings(w, node, node.GetInputs(), &flyte.VariableMap{Variables: map[string]*flyte.Variable{}}, + false, EdgeDirectionUpstream, errs.NewScope()) + if !errs.HasErrors() && iface != nil { iface = &flyte.TypedInterface{ - Inputs: filterVariables(iface.Inputs, finalInputParameterNames), + Inputs: inputVarsFromBindings, Outputs: filterVariables(iface.Outputs, finalOutputParameterNames), } } else { diff --git a/flytepropeller/pkg/compiler/validators/branch_test.go b/flytepropeller/pkg/compiler/validators/branch_test.go index 892a3c9d9b..2c04aa110e 100644 --- a/flytepropeller/pkg/compiler/validators/branch_test.go +++ b/flytepropeller/pkg/compiler/validators/branch_test.go @@ -48,6 +48,7 @@ func Test_validateBranchInterface(t *testing.T) { n2.OnGetCoreNode().Return(coreN2) n2.OnGetTaskNode().Return(taskNode) n2.On("SetInterface", mock.Anything) + n2.OnGetInputs().Return([]*core.Binding{}) task := &mocks.Task{} task.OnGetInterface().Return(&core.TypedInterface{}) @@ -77,6 +78,8 @@ func Test_validateBranchInterface(t *testing.T) { }, }) + n.OnGetInputs().Return([]*core.Binding{}) + _, ok := validateBranchInterface(wf, n, errs) assert.True(t, ok) if errs.HasErrors() { @@ -87,6 +90,7 @@ func Test_validateBranchInterface(t *testing.T) { t.Run("two conditions", func(t *testing.T) { n := &mocks.NodeBuilder{} n.OnGetId().Return("n1") + n.OnGetInputs().Return([]*core.Binding{}) n.OnGetBranchNode().Return(&core.BranchNode{ IfElse: &core.IfElseBlock{ Case: &core.IfBlock{ diff --git a/flytepropeller/pkg/compiler/validators/condition.go b/flytepropeller/pkg/compiler/validators/condition.go index e8692afcb9..a70c5dcb2b 100644 --- a/flytepropeller/pkg/compiler/validators/condition.go +++ b/flytepropeller/pkg/compiler/validators/condition.go @@ -32,7 +32,7 @@ func validateOperand(node c.NodeBuilder, paramName string, operand *flyte.Operan return literalType, !errs.HasErrors() } -func ValidateBooleanExpression(node c.NodeBuilder, expr *flyte.BooleanExpression, requireParamType bool, errs errors.CompileErrors) (ok bool) { +func ValidateBooleanExpression(w c.WorkflowBuilder, node c.NodeBuilder, expr *flyte.BooleanExpression, requireParamType bool, errs errors.CompileErrors) (ok bool) { if expr == nil { errs.Collect(errors.NewBranchNodeHasNoCondition(node.GetId())) } else { @@ -48,8 +48,8 @@ func ValidateBooleanExpression(node c.NodeBuilder, expr *flyte.BooleanExpression } } } else if expr.GetConjunction() != nil { - ValidateBooleanExpression(node, expr.GetConjunction().LeftExpression, requireParamType, errs.NewScope()) - ValidateBooleanExpression(node, expr.GetConjunction().RightExpression, requireParamType, errs.NewScope()) + ValidateBooleanExpression(w, node, expr.GetConjunction().LeftExpression, requireParamType, errs.NewScope()) + ValidateBooleanExpression(w, node, expr.GetConjunction().RightExpression, requireParamType, errs.NewScope()) } else { errs.Collect(errors.NewValueRequiredErr(node.GetId(), "Expr")) } diff --git a/flytepropeller/pkg/compiler/validators/node.go b/flytepropeller/pkg/compiler/validators/node.go index f8d3141818..14d4e7aa54 100644 --- a/flytepropeller/pkg/compiler/validators/node.go +++ b/flytepropeller/pkg/compiler/validators/node.go @@ -53,13 +53,9 @@ func branchNodeIDFormatter(parentNodeID, thenNodeID string) string { return fmt.Sprintf("%v-%v", parentNodeID, thenNodeID) } -type EdgeInfo struct { - from string - to string -} - func ValidateBranchNode(w c.WorkflowBuilder, n c.NodeBuilder, requireParamType bool, errs errors.CompileErrors) ( - discoveredNodes []c.NodeBuilder, additionalEdges []EdgeInfo, ok bool) { + discoveredNodes []c.NodeBuilder, ok bool) { + cases := make([]*flyte.IfBlock, 0, len(n.GetBranchNode().IfElse.Other)+1) if n.GetBranchNode().IfElse.Case == nil { errs.Collect(errors.NewBranchNodeHasNoCondition(n.GetId())) @@ -69,11 +65,10 @@ func ValidateBranchNode(w c.WorkflowBuilder, n c.NodeBuilder, requireParamType b cases = append(cases, n.GetBranchNode().IfElse.Other...) discoveredNodes = make([]c.NodeBuilder, 0, len(cases)) - additionalEdges = make([]EdgeInfo, 0, len(cases)) subNodes := make([]c.NodeBuilder, 0, len(cases)+1) for _, block := range cases { // Validate condition - ValidateBooleanExpression(n, block.Condition, requireParamType, errs.NewScope()) + ValidateBooleanExpression(w, n, block.Condition, requireParamType, errs.NewScope()) if block.GetThenNode() == nil { errs.Collect(errors.NewBranchNodeNotSpecified(n.GetId())) @@ -94,14 +89,10 @@ func ValidateBranchNode(w c.WorkflowBuilder, n c.NodeBuilder, requireParamType b if ValidateNode(w, wrapperNode, requireParamType, errs.NewScope()) { // Add to the global nodes to be able to reference it later discoveredNodes = append(discoveredNodes, wrapperNode) - additionalEdges = append(additionalEdges, EdgeInfo{ - from: n.GetId(), - to: wrapperNode.GetId(), - }) } } - return discoveredNodes, additionalEdges, !errs.HasErrors() + return discoveredNodes, !errs.HasErrors() } func validateNodeID(w c.WorkflowBuilder, nodeID string, errs errors.CompileErrors) (node c.NodeBuilder, ok bool) { @@ -131,26 +122,13 @@ func ValidateNode(w c.WorkflowBuilder, n c.NodeBuilder, validateConditionTypes b // Validate branch node conditions and inner nodes. if n.GetBranchNode() != nil { - if nodes, edges, ok := ValidateBranchNode(w, n, validateConditionTypes, errs.NewScope()); ok { + if nodes, ok := ValidateBranchNode(w, n, validateConditionTypes, errs.NewScope()); ok { renamedNodes := make(map[c.NodeID]c.NodeID, len(nodes)) for _, subNode := range nodes { oldID := subNode.GetId() subNode.SetID(branchNodeIDFormatter(n.GetId(), subNode.GetId())) - w.AddNode(subNode, errs) renamedNodes[oldID] = subNode.GetId() } - - for _, edge := range edges { - if newID, found := renamedNodes[edge.from]; found { - edge.from = newID - } - - if newID, found := renamedNodes[edge.to]; found { - edge.to = newID - } - - w.AddExecutionEdge(edge.from, edge.to) - } } } else if workflowN := n.GetWorkflowNode(); workflowN != nil && workflowN.GetSubWorkflowRef() != nil { workflowID := *workflowN.GetSubWorkflowRef() diff --git a/flytepropeller/pkg/compiler/validators/node_test.go b/flytepropeller/pkg/compiler/validators/node_test.go index cee746e48c..f7e6730a1d 100644 --- a/flytepropeller/pkg/compiler/validators/node_test.go +++ b/flytepropeller/pkg/compiler/validators/node_test.go @@ -22,7 +22,7 @@ func TestValidateBranchNode(t *testing.T) { wf := &mocks.WorkflowBuilder{} errs := errors.NewCompileErrors() - _, _, ok := ValidateBranchNode(wf, n, false, errs) + _, ok := ValidateBranchNode(wf, n, false, errs) assert.False(t, ok) if !errs.HasErrors() { assert.Error(t, errs) diff --git a/flytepropeller/pkg/compiler/workflow_compiler.go b/flytepropeller/pkg/compiler/workflow_compiler.go index e01795c8dd..68c1e98aa9 100755 --- a/flytepropeller/pkg/compiler/workflow_compiler.go +++ b/flytepropeller/pkg/compiler/workflow_compiler.go @@ -108,27 +108,39 @@ func (w workflowBuilder) AddNode(n c.NodeBuilder, errs errors.CompileErrors) (no return } -func (w workflowBuilder) AddExecutionEdge(nodeFrom, nodeTo c.NodeID) { - if nodeFrom == "" { - nodeFrom = c.StartNodeID - } - - if _, found := w.downstreamNodes[nodeFrom]; !found { - w.downstreamNodes[nodeFrom] = sets.String{} - w.CoreWorkflow.Connections.Downstream[nodeFrom] = &core.ConnectionSet_IdList{} +func (w workflowBuilder) AddUpstreamEdge(nodeProvider, nodeDependent c.NodeID) { + if nodeProvider == "" { + nodeProvider = c.StartNodeID } - if _, found := w.upstreamNodes[nodeTo]; !found { - w.upstreamNodes[nodeTo] = sets.String{} - w.CoreWorkflow.Connections.Upstream[nodeTo] = &core.ConnectionSet_IdList{ + if _, found := w.upstreamNodes[nodeDependent]; !found { + w.upstreamNodes[nodeDependent] = sets.String{} + w.CoreWorkflow.Connections.Upstream[nodeDependent] = &core.ConnectionSet_IdList{ Ids: make([]string, 1), } } - w.downstreamNodes[nodeFrom].Insert(nodeTo) - w.upstreamNodes[nodeTo].Insert(nodeFrom) - w.CoreWorkflow.Connections.Downstream[nodeFrom].Ids = w.downstreamNodes[nodeFrom].List() - w.CoreWorkflow.Connections.Upstream[nodeTo].Ids = w.upstreamNodes[nodeTo].List() + w.upstreamNodes[nodeDependent].Insert(nodeProvider) + w.CoreWorkflow.Connections.Upstream[nodeDependent].Ids = w.upstreamNodes[nodeDependent].List() +} + +func (w workflowBuilder) AddDownstreamEdge(nodeProvider, nodeDependent c.NodeID) { + if nodeProvider == "" { + nodeProvider = c.StartNodeID + } + + if _, found := w.downstreamNodes[nodeProvider]; !found { + w.downstreamNodes[nodeProvider] = sets.String{} + w.CoreWorkflow.Connections.Downstream[nodeProvider] = &core.ConnectionSet_IdList{} + } + + w.downstreamNodes[nodeProvider].Insert(nodeDependent) + w.CoreWorkflow.Connections.Downstream[nodeProvider].Ids = w.downstreamNodes[nodeProvider].List() +} + +func (w workflowBuilder) AddExecutionEdge(nodeFrom, nodeTo c.NodeID) { + w.AddDownstreamEdge(nodeFrom, nodeTo) + w.AddUpstreamEdge(nodeFrom, nodeTo) } func (w workflowBuilder) AddEdges(n c.NodeBuilder, errs errors.CompileErrors) (ok bool) { @@ -146,7 +158,7 @@ func (w workflowBuilder) AddEdges(n c.NodeBuilder, errs errors.CompileErrors) (o // Add implicit Edges _, ok = v.ValidateBindings(&w, n, n.GetInputs(), n.GetInterface().GetInputs(), - true /* validateParamTypes */, errs.NewScope()) + true /* validateParamTypes */, v.EdgeDirectionBidirectional, errs.NewScope()) return } @@ -209,7 +221,7 @@ func (w workflowBuilder) ValidateWorkflow(fg *flyteWorkflow, errs errors.Compile for _, n := range wf.Nodes { if n.GetBranchNode() != nil { if inputVars, ok := v.ValidateBindings(&wf, n, n.GetInputs(), n.GetInterface().GetInputs(), - false /* validateParamTypes */, errs.NewScope()); ok { + false /* validateParamTypes */, v.EdgeDirectionUpstream, errs.NewScope()); ok { merge, err := v.UnionDistinctVariableMaps(n.GetInterface().Inputs.Variables, inputVars.Variables) if err != nil { errs.Collect(errors.NewWorkflowBuildError(err)) @@ -261,7 +273,8 @@ func (w workflowBuilder) ValidateWorkflow(fg *flyteWorkflow, errs errors.Compile // Validate workflow outputs are bound if _, wfIfaceOk := v.ValidateInterface(globalOutputNode.GetId(), globalOutputNode.GetInterface(), errs.NewScope()); wfIfaceOk { v.ValidateBindings(&wf, globalOutputNode, globalOutputNode.GetInputs(), - globalOutputNode.GetInterface().GetInputs(), true /* validateParamTypes */, errs.NewScope()) + globalOutputNode.GetInterface().GetInputs(), true, /* validateParamTypes */ + v.EdgeDirectionBidirectional, errs.NewScope()) } // Validate no cycles are detected. diff --git a/flytepropeller/pkg/compiler/workflow_compiler_test.go b/flytepropeller/pkg/compiler/workflow_compiler_test.go index b68d7b41c9..3b4617fc2c 100755 --- a/flytepropeller/pkg/compiler/workflow_compiler_test.go +++ b/flytepropeller/pkg/compiler/workflow_compiler_test.go @@ -5,6 +5,8 @@ import ( "strings" "testing" + "github.com/flyteorg/flytepropeller/pkg/compiler/common/mocks" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" "github.com/flyteorg/flytepropeller/pkg/apis/flyteworkflow/v1alpha1" "github.com/flyteorg/flytepropeller/pkg/compiler/common" @@ -329,8 +331,10 @@ func TestComparisonExpression_MissingLeftRight(t *testing.T) { }, } + w := &mocks.WorkflowBuilder{} + errs := errors.NewCompileErrors() - v.ValidateBooleanExpression(&nodeBuilder{flyteNode: &flyteNode{}}, bExpr, true, errs) + v.ValidateBooleanExpression(w, &nodeBuilder{flyteNode: &flyteNode{}}, bExpr, true, errs) assert.Error(t, errs) assert.Equal(t, 2, errs.ErrorCount()) } @@ -346,8 +350,9 @@ func TestComparisonExpression(t *testing.T) { }, } + w := &mocks.WorkflowBuilder{} errs := errors.NewCompileErrors() - v.ValidateBooleanExpression(&nodeBuilder{flyteNode: &flyteNode{}}, bExpr, true, errs) + v.ValidateBooleanExpression(w, &nodeBuilder{flyteNode: &flyteNode{}}, bExpr, true, errs) assert.True(t, errs.HasErrors()) assert.Equal(t, 1, errs.ErrorCount()) } @@ -370,8 +375,9 @@ func TestBooleanExpression_BranchNodeHasNoCondition(t *testing.T) { }, } + w := &mocks.WorkflowBuilder{} errs := errors.NewCompileErrors() - v.ValidateBooleanExpression(&nodeBuilder{flyteNode: &flyteNode{}}, bExpr, true, errs) + v.ValidateBooleanExpression(w, &nodeBuilder{flyteNode: &flyteNode{}}, bExpr, true, errs) assert.True(t, errs.HasErrors()) assert.Equal(t, 1, errs.ErrorCount()) for e := range *errs.Errors() { diff --git a/flytepropeller/pkg/controller/nodes/dynamic/dynamic_workflow_test.go b/flytepropeller/pkg/controller/nodes/dynamic/dynamic_workflow_test.go index 6f0eaac01b..08184f07af 100644 --- a/flytepropeller/pkg/controller/nodes/dynamic/dynamic_workflow_test.go +++ b/flytepropeller/pkg/controller/nodes/dynamic/dynamic_workflow_test.go @@ -102,8 +102,8 @@ func Test_dynamicNodeHandler_buildContextualDynamicWorkflow_withLaunchPlans(t *t nCtx.OnDataStore().Return(dataStore) endNodeStatus := &mocks2.ExecutableNodeStatus{} - endNodeStatus.OnGetDataDir().Return(storage.DataReference("end-node")) - endNodeStatus.OnGetOutputDir().Return(storage.DataReference("end-node")) + endNodeStatus.OnGetDataDir().Return("end-node") + endNodeStatus.OnGetOutputDir().Return("end-node") subNs := &mocks2.ExecutableNodeStatus{} subNs.On("SetDataDir", mock.Anything).Return() @@ -363,7 +363,7 @@ func Test_dynamicNodeHandler_buildContextualDynamicWorkflow_withLaunchPlans(t *t assert.NoError(t, err) _, err = nCtx.DataStore().ConstructReference(ctx, nCtx.NodeStatus().GetOutputDir(), "futures_compiled.pb") assert.NoError(t, err) - assert.NoError(t, nCtx.DataStore().WriteRaw(context.TODO(), storage.DataReference("/output-dir/futures_compiled.pb"), int64(len(rawDynamicWf)), storage.Options{}, bytes.NewReader(rawDynamicWf))) + assert.NoError(t, nCtx.DataStore().WriteRaw(context.TODO(), "/output-dir/futures_compiled.pb", int64(len(rawDynamicWf)), storage.Options{}, bytes.NewReader(rawDynamicWf))) f, err := nCtx.DataStore().ConstructReference(ctx, nCtx.NodeStatus().GetOutputDir(), "futures.pb") assert.NoError(t, err) @@ -485,19 +485,19 @@ func Test_dynamicNodeHandler_buildContextualDynamicWorkflow_withLaunchPlans(t *t metadata := existsMetadata{} composedPBStore := storageMocks.ComposedProtobufStore{} - composedPBStore.On("Head", mock.MatchedBy(func(ctx context.Context) bool { return true }), storage.DataReference("s3://my-s3-bucket/foo/bar/futures_compiled.pb")). + composedPBStore.OnHeadMatch(mock.MatchedBy(func(ctx context.Context) bool { return true }), storage.DataReference("s3://my-s3-bucket/foo/bar/futures_compiled.pb")). Return(&metadata, nil) djSpec := createDynamicJobSpecWithLaunchPlans() - composedPBStore.On("ReadProtobuf", mock.MatchedBy(func(ctx context.Context) bool { return true }), + composedPBStore.OnReadProtobufMatch(mock.MatchedBy(func(ctx context.Context) bool { return true }), storage.DataReference("s3://my-s3-bucket/foo/bar/futures.pb"), &core.DynamicJobSpec{}).Return(nil).Run(func(args mock.Arguments) { djSpecPtr := args.Get(2).(*core.DynamicJobSpec) *djSpecPtr = *djSpec }) - composedPBStore.On("WriteRaw", + composedPBStore.OnWriteRawMatch( mock.MatchedBy(func(ctx context.Context) bool { return true }), storage.DataReference("s3://my-s3-bucket/foo/bar/futures_compiled.pb"), - int64(1039), + int64(1169), storage.Options{}, mock.MatchedBy(func(rdr *bytes.Reader) bool { return true })).Return(errors.New("foo")) @@ -527,7 +527,9 @@ func Test_dynamicNodeHandler_buildContextualDynamicWorkflow_withLaunchPlans(t *t Domain: "d", } mockLPLauncher := &mocks5.Reader{} - mockLPLauncher.OnGetLaunchPlanMatch(ctx, lpID).Return(&admin.LaunchPlan{ + mockLPLauncher.OnGetLaunchPlanMatch(mock.Anything, mock.MatchedBy(func(id *core.Identifier) bool { + return lpID.Name == id.Name && lpID.Domain == id.Domain && lpID.Project == id.Project && lpID.ResourceType == id.ResourceType + })).Return(&admin.LaunchPlan{ Id: lpID, Closure: &admin.LaunchPlanClosure{ ExpectedInputs: &core.ParameterMap{}, diff --git a/flytepropeller/pkg/controller/nodes/executor_test.go b/flytepropeller/pkg/controller/nodes/executor_test.go index 07b80b700c..b1fa755c70 100644 --- a/flytepropeller/pkg/controller/nodes/executor_test.go +++ b/flytepropeller/pkg/controller/nodes/executor_test.go @@ -187,10 +187,10 @@ func TestNodeExecutor_RecursiveNodeHandler_RecurseStartNodes(t *testing.T) { v1alpha1.StartNodeID: startNode, }, Connections: v1alpha1.Connections{ - UpstreamEdges: map[v1alpha1.NodeID][]v1alpha1.NodeID{ + Upstream: map[v1alpha1.NodeID][]v1alpha1.NodeID{ defaultNodeID: {v1alpha1.StartNodeID}, }, - DownstreamEdges: map[v1alpha1.NodeID][]v1alpha1.NodeID{ + Downstream: map[v1alpha1.NodeID][]v1alpha1.NodeID{ v1alpha1.StartNodeID: {defaultNodeID}, }, }, @@ -293,7 +293,7 @@ func TestNodeExecutor_RecursiveNodeHandler_RecurseEndNode(t *testing.T) { v1alpha1.EndNodeID: n, }, Connections: v1alpha1.Connections{ - UpstreamEdges: map[v1alpha1.NodeID][]v1alpha1.NodeID{ + Upstream: map[v1alpha1.NodeID][]v1alpha1.NodeID{ v1alpha1.EndNodeID: {v1alpha1.StartNodeID}, }, }, @@ -379,10 +379,10 @@ func TestNodeExecutor_RecursiveNodeHandler_RecurseEndNode(t *testing.T) { v1alpha1.EndNodeID: n, }, Connections: v1alpha1.Connections{ - UpstreamEdges: map[v1alpha1.NodeID][]v1alpha1.NodeID{ + Upstream: map[v1alpha1.NodeID][]v1alpha1.NodeID{ v1alpha1.EndNodeID: {v1alpha1.StartNodeID}, }, - DownstreamEdges: map[v1alpha1.NodeID][]v1alpha1.NodeID{ + Downstream: map[v1alpha1.NodeID][]v1alpha1.NodeID{ v1alpha1.StartNodeID: {v1alpha1.EndNodeID}, }, }, @@ -514,10 +514,10 @@ func TestNodeExecutor_RecursiveNodeHandler_Recurse(t *testing.T) { v1alpha1.StartNodeID: startNode, }, Connections: v1alpha1.Connections{ - UpstreamEdges: map[v1alpha1.NodeID][]v1alpha1.NodeID{ + Upstream: map[v1alpha1.NodeID][]v1alpha1.NodeID{ defaultNodeID: {v1alpha1.StartNodeID}, }, - DownstreamEdges: map[v1alpha1.NodeID][]v1alpha1.NodeID{ + Downstream: map[v1alpha1.NodeID][]v1alpha1.NodeID{ v1alpha1.StartNodeID: {defaultNodeID}, }, }, @@ -536,7 +536,7 @@ func TestNodeExecutor_RecursiveNodeHandler_Recurse(t *testing.T) { nodeN2 := "n2" ctx := context.Background() connections := &v1alpha1.Connections{ - UpstreamEdges: map[v1alpha1.NodeID][]v1alpha1.NodeID{ + Upstream: map[v1alpha1.NodeID][]v1alpha1.NodeID{ nodeN2: {nodeN0}, }, } @@ -1020,10 +1020,10 @@ func TestNodeExecutor_RecursiveNodeHandler_NoDownstream(t *testing.T) { v1alpha1.StartNodeID: startNode, }, Connections: v1alpha1.Connections{ - UpstreamEdges: map[v1alpha1.NodeID][]v1alpha1.NodeID{ + Upstream: map[v1alpha1.NodeID][]v1alpha1.NodeID{ defaultNodeID: {v1alpha1.StartNodeID}, }, - DownstreamEdges: map[v1alpha1.NodeID][]v1alpha1.NodeID{ + Downstream: map[v1alpha1.NodeID][]v1alpha1.NodeID{ v1alpha1.StartNodeID: {defaultNodeID}, }, }, @@ -1124,7 +1124,7 @@ func TestNodeExecutor_RecursiveNodeHandler_UpstreamNotReady(t *testing.T) { defaultNodeID: n, }, Connections: v1alpha1.Connections{ - UpstreamEdges: map[v1alpha1.NodeID][]v1alpha1.NodeID{ + Upstream: map[v1alpha1.NodeID][]v1alpha1.NodeID{ defaultNodeID: {v1alpha1.StartNodeID}, }, }, @@ -1754,10 +1754,10 @@ func TestNodeExecutor_RecursiveNodeHandler_ParallelismLimit(t *testing.T) { v1alpha1.StartNodeID: startNode, }, Connections: v1alpha1.Connections{ - UpstreamEdges: map[v1alpha1.NodeID][]v1alpha1.NodeID{ + Upstream: map[v1alpha1.NodeID][]v1alpha1.NodeID{ defaultNodeID: {v1alpha1.StartNodeID}, }, - DownstreamEdges: map[v1alpha1.NodeID][]v1alpha1.NodeID{ + Downstream: map[v1alpha1.NodeID][]v1alpha1.NodeID{ v1alpha1.StartNodeID: {defaultNodeID}, }, }, diff --git a/flytepropeller/pkg/controller/nodes/predicate.go b/flytepropeller/pkg/controller/nodes/predicate.go index f006e3181e..0d5aea8a35 100644 --- a/flytepropeller/pkg/controller/nodes/predicate.go +++ b/flytepropeller/pkg/controller/nodes/predicate.go @@ -38,7 +38,9 @@ func (p PredicatePhase) String() string { return "undefined" } -func CanExecute(ctx context.Context, dag executors.DAGStructure, nl executors.NodeLookup, node v1alpha1.BaseNode) (PredicatePhase, error) { +func CanExecute(ctx context.Context, dag executors.DAGStructure, nl executors.NodeLookup, node v1alpha1.BaseNode) ( + PredicatePhase, error) { + nodeID := node.GetID() if nodeID == v1alpha1.StartNodeID { logger.Debugf(ctx, "Start Node id is assumed to be ready.") @@ -66,6 +68,8 @@ func CanExecute(ctx context.Context, dag executors.DAGStructure, nl executors.No return PredicatePhaseUndefined, errors.Errorf(errors.BadSpecificationError, nodeID, "Upstream node [%v] of node [%v] not defined", upstreamNodeID, nodeID) } + // Deprecated: This if block will be removed in a future version. It's harmless (will be no-op) for newly + // compiled Workflows as sub-branch-nodes won't have an execution or code dependency on branch nodes. // This only happens if current node is the child node of a branch node if upstreamNode.GetBranchNode() == nil || upstreamNodeStatus.GetBranchStatus().GetPhase() != v1alpha1.BranchNodeSuccess { logger.Debugf(ctx, "Branch sub node is expected to have parent branch node in succeeded state") @@ -115,6 +119,8 @@ func GetParentNodeMaxEndTime(ctx context.Context, dag executors.DAGStructure, nl return zeroTime, errors.Errorf(errors.BadSpecificationError, nodeID, "Upstream node [%v] of node [%v] not defined", upstreamNodeID, nodeID) } + // Deprecated: This if block will be removed in a future version. It's harmless (will be no-op) for newly + // compiled Workflows as sub-branch-nodes won't have an execution or code dependency on branch nodes. // This only happens if current node is the child node of a branch node if upstreamNode.GetBranchNode() == nil || upstreamNodeStatus.GetBranchStatus().GetPhase() != v1alpha1.BranchNodeSuccess { logger.Debugf(ctx, "Branch sub node is expected to have parent branch node in succeeded state") diff --git a/flytepropeller/pkg/controller/nodes/predicate_test.go b/flytepropeller/pkg/controller/nodes/predicate_test.go index 3f967c4977..92afb9664b 100644 --- a/flytepropeller/pkg/controller/nodes/predicate_test.go +++ b/flytepropeller/pkg/controller/nodes/predicate_test.go @@ -319,116 +319,6 @@ func TestCanExecute(t *testing.T) { assert.Equal(t, PredicatePhaseUndefined, p) }) - // ParentNode has no branch node - t.Run("upstreamConnectionsParentHasNoBranch", func(t *testing.T) { - // Setup - mockN2Status := &mocks.ExecutableNodeStatus{} - // No parent node - mockN2Status.On("GetParentNodeID").Return(&nodeN0) - mockN2Status.On("IsDirty").Return(false) - - mockNode := &mocks.BaseNode{} - mockNode.On("GetID").Return(nodeN2) - - mockN0Node := &mocks.ExecutableNode{} - mockN0Node.On("GetBranchNode").Return(nil) - mockN0Status := &mocks.ExecutableNodeStatus{} - mockN0Status.On("GetPhase").Return(v1alpha1.NodePhaseSucceeded) - mockN0Status.On("IsDirty").Return(false) - - mockN1Status := &mocks.ExecutableNodeStatus{} - mockN1Status.On("GetPhase").Return(v1alpha1.NodePhaseSucceeded) - mockN1Status.On("IsDirty").Return(false) - - mockWf := &mocks.ExecutableWorkflow{} - mockWf.OnGetNodeExecutionStatus(ctx, nodeN0).Return(mockN0Status) - mockWf.OnGetNodeExecutionStatus(ctx, nodeN1).Return(mockN1Status) - mockWf.OnGetNodeExecutionStatus(ctx, nodeN2).Return(mockN2Status) - mockWf.OnToNode(nodeN2).Return(upstreamN2, nil) - mockWf.On("GetNode", nodeN0).Return(mockN0Node, true) - mockWf.On("GetID").Return("w1") - - p, err := CanExecute(ctx, mockWf, mockWf, mockNode) - assert.Error(t, err) - assert.Equal(t, PredicatePhaseUndefined, p) - }) - - // ParentNode branch not ready - t.Run("upstreamConnectionsBranchNodeNotReady", func(t *testing.T) { - // Setup - mockN2Status := &mocks.ExecutableNodeStatus{} - // No parent node - mockN2Status.On("GetParentNodeID").Return(&nodeN0) - mockN2Status.On("IsDirty").Return(false) - - mockNode := &mocks.BaseNode{} - mockNode.On("GetID").Return(nodeN2) - - mockN0BranchStatus := &mocks.MutableBranchNodeStatus{} - mockN0BranchStatus.On("GetPhase").Return(v1alpha1.BranchNodeNotYetEvaluated) - mockN0BranchNode := &mocks.ExecutableBranchNode{} - mockN0Node := &mocks.ExecutableNode{} - mockN0Node.On("GetBranchNode").Return(mockN0BranchNode) - mockN0Status := &mocks.ExecutableNodeStatus{} - mockN0Status.On("GetPhase").Return(v1alpha1.NodePhaseSucceeded) - mockN0Status.On("GetBranchStatus").Return(mockN0BranchStatus) - mockN0Status.On("IsDirty").Return(false) - - mockN1Status := &mocks.ExecutableNodeStatus{} - mockN1Status.On("GetPhase").Return(v1alpha1.NodePhaseSucceeded) - mockN1Status.On("IsDirty").Return(false) - - mockWf := &mocks.ExecutableWorkflow{} - mockWf.OnGetNodeExecutionStatus(ctx, nodeN0).Return(mockN0Status) - mockWf.OnGetNodeExecutionStatus(ctx, nodeN1).Return(mockN1Status) - mockWf.OnGetNodeExecutionStatus(ctx, nodeN2).Return(mockN2Status) - mockWf.OnToNode(nodeN2).Return(upstreamN2, nil) - mockWf.On("GetNode", nodeN0).Return(mockN0Node, true) - mockWf.On("GetID").Return("w1") - - p, err := CanExecute(ctx, mockWf, mockWf, mockNode) - assert.Error(t, err) - assert.Equal(t, PredicatePhaseUndefined, p) - }) - - // ParentNode branch is errored - t.Run("upstreamConnectionsBranchNodeError", func(t *testing.T) { - // Setup - mockN2Status := &mocks.ExecutableNodeStatus{} - // No parent node - mockN2Status.On("GetParentNodeID").Return(&nodeN0) - mockN2Status.On("IsDirty").Return(false) - - mockNode := &mocks.BaseNode{} - mockNode.On("GetID").Return(nodeN2) - - mockN0BranchStatus := &mocks.MutableBranchNodeStatus{} - mockN0BranchStatus.On("GetPhase").Return(v1alpha1.BranchNodeError) - mockN0BranchNode := &mocks.ExecutableBranchNode{} - mockN0Node := &mocks.ExecutableNode{} - mockN0Node.On("GetBranchNode").Return(mockN0BranchNode) - mockN0Status := &mocks.ExecutableNodeStatus{} - mockN0Status.On("GetPhase").Return(v1alpha1.NodePhaseSucceeded) - mockN0Status.On("GetBranchStatus").Return(mockN0BranchStatus) - mockN0Status.On("IsDirty").Return(false) - - mockN1Status := &mocks.ExecutableNodeStatus{} - mockN1Status.On("GetPhase").Return(v1alpha1.NodePhaseSucceeded) - mockN1Status.On("IsDirty").Return(false) - - mockWf := &mocks.ExecutableWorkflow{} - mockWf.OnGetNodeExecutionStatus(ctx, nodeN0).Return(mockN0Status) - mockWf.OnGetNodeExecutionStatus(ctx, nodeN1).Return(mockN1Status) - mockWf.OnGetNodeExecutionStatus(ctx, nodeN2).Return(mockN2Status) - mockWf.OnToNode(nodeN2).Return(upstreamN2, nil) - mockWf.On("GetNode", nodeN0).Return(mockN0Node, true) - mockWf.On("GetID").Return("w1") - - p, err := CanExecute(ctx, mockWf, mockWf, mockNode) - assert.Error(t, err) - assert.Equal(t, PredicatePhaseUndefined, p) - }) - // ParentNode branch ready t.Run("upstreamConnectionsBranchSuccessOtherSuccess", func(t *testing.T) { // Setup diff --git a/flytepropeller/pkg/controller/nodes/task/remote_workflow_store_test.go b/flytepropeller/pkg/controller/nodes/task/remote_workflow_store_test.go index a5874348bb..9ab5b25c6d 100644 --- a/flytepropeller/pkg/controller/nodes/task/remote_workflow_store_test.go +++ b/flytepropeller/pkg/controller/nodes/task/remote_workflow_store_test.go @@ -4,6 +4,8 @@ import ( "context" "testing" + "github.com/go-test/deep" + "github.com/golang/protobuf/proto" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" @@ -36,6 +38,34 @@ func Test_cacheFlyteWorkflow(t *testing.T) { WorkflowSpec: &v1alpha1.WorkflowSpec{ ID: "abc", Connections: v1alpha1.Connections{ + Downstream: map[v1alpha1.NodeID][]v1alpha1.NodeID{}, + Upstream: map[v1alpha1.NodeID][]v1alpha1.NodeID{}, + }, + DeprecatedConnections: v1alpha1.DeprecatedConnections{ + DownstreamEdges: map[v1alpha1.NodeID][]v1alpha1.NodeID{}, + UpstreamEdges: map[v1alpha1.NodeID][]v1alpha1.NodeID{}, + }, + }, + } + + ctx := context.TODO() + location := storage.DataReference("somekey/file.json") + r := RemoteFileWorkflowStore{store: store} + assert.NoError(t, r.PutFlyteWorkflowCRD(ctx, expected, location)) + actual, err := r.GetWorkflowCRD(ctx, location) + assert.NoError(t, err) + if diff := deep.Equal(expected, actual); len(diff) > 0 { + t.Errorf("Cached() Diff = %v\r\n got = %v\r\n want = %v", diff, actual, expected) + } + }) + + t.Run("cache CRD with deprecatedConnections", func(t *testing.T) { + expected := &v1alpha1.FlyteWorkflow{ + TypeMeta: v1.TypeMeta{}, + ObjectMeta: v1.ObjectMeta{}, + WorkflowSpec: &v1alpha1.WorkflowSpec{ + ID: "abc", + DeprecatedConnections: v1alpha1.DeprecatedConnections{ DownstreamEdges: map[v1alpha1.NodeID][]v1alpha1.NodeID{}, UpstreamEdges: map[v1alpha1.NodeID][]v1alpha1.NodeID{}, }, @@ -50,6 +80,7 @@ func Test_cacheFlyteWorkflow(t *testing.T) { assert.NoError(t, err) assert.Equal(t, expected, actual) }) + t.Run("cache compiled workflow", func(t *testing.T) { expected := &core.CompiledWorkflowClosure{ Primary: &core.CompiledWorkflow{ diff --git a/flytepropeller/pkg/controller/workflow/executor_test.go b/flytepropeller/pkg/controller/workflow/executor_test.go index 4539b4e606..6bf7c5e188 100644 --- a/flytepropeller/pkg/controller/workflow/executor_test.go +++ b/flytepropeller/pkg/controller/workflow/executor_test.go @@ -277,7 +277,7 @@ func walkAndPrint(conns v1alpha1.Connections, ns map[v1alpha1.NodeID]*v1alpha1.N for len(ds) > 0 { sub := sets.NewString() for _, x := range ds { - sub.Insert(conns.DownstreamEdges[x]...) + sub.Insert(conns.Downstream[x]...) if !visited[x] { s, ok := ns[x] if ok { diff --git a/flytepropeller/pkg/visualize/visualize.go b/flytepropeller/pkg/visualize/visualize.go index 2725514640..2ab8d69f9c 100644 --- a/flytepropeller/pkg/visualize/visualize.go +++ b/flytepropeller/pkg/visualize/visualize.go @@ -97,7 +97,7 @@ func WorkflowToGraphViz(g *v1alpha1.FlyteWorkflow) string { for nodesToVisit := NewNodeNameQ(start.ID); nodesToVisit.HasNext(); { node := nodesToVisit.Deque() - nodes, found := g.GetConnections().DownstreamEdges[node] + nodes, found := g.GetConnections().Downstream[node] if found { nodesToVisit.Enqueue(nodes...)