Skip to content
This repository has been archived by the owner on May 31, 2024. It is now read-only.

OverwriteCache flag for creating/relaunching executions #358

Merged
merged 5 commits into from
Nov 16, 2022
Merged

OverwriteCache flag for creating/relaunching executions #358

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
34e2426
Added direnv .envrc to gitignore
Oct 5, 2022
3333ee0
Implemented skipCache flag for creating executions
Oct 5, 2022
43ffe5a
Renamed skipCache flag to overwriteCache
Nov 3, 2022
2adfbfd
Updated to latest released versions of flyteidl and flytestdlib
Nov 15, 2022
5108851
Removed unneeded overwriteCache parameter in favor of global executio…
Nov 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,5 @@ _test
docs/build/*
cmd/upgrade/flyte.ext
.vscode
# direnv
.envrc
11 changes: 5 additions & 6 deletions cmd/create/execution.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,9 +138,7 @@ The generated file would look similar to this. Here, empty values have been dump

::

flytectl create execution --execFile execution_spec.yaml -p flytesnacks -d development --clusterPool my-gpu-cluster

Usage
flytectl create execution --execFile execution_spec.yaml -p flytesnacks -d development --clusterPool my-gpu-cluster
`
)

Expand All @@ -149,16 +147,17 @@ Usage
// ExecutionConfig hold configuration for create execution flags and configuration of the actual task or workflow to be launched.
type ExecutionConfig struct {
// pflag section
ExecFile string `json:"execFile,omitempty" pflag:",file for the execution params.If not specified defaults to <<workflow/task>_name>.execution_spec.yaml"`
TargetDomain string `json:"targetDomain" pflag:",project where execution needs to be created.If not specified configured domain would be used."`
TargetProject string `json:"targetProject" pflag:",project where execution needs to be created.If not specified configured project would be used."`
ExecFile string `json:"execFile,omitempty" pflag:",file for the execution params. If not specified defaults to <<workflow/task>_name>.execution_spec.yaml"`
TargetDomain string `json:"targetDomain" pflag:",project where execution needs to be created. If not specified configured domain would be used."`
TargetProject string `json:"targetProject" pflag:",project where execution needs to be created. If not specified configured project would be used."`
KubeServiceAcct string `json:"kubeServiceAcct" pflag:",kubernetes service account AuthRole for launching execution."`
IamRoleARN string `json:"iamRoleARN" pflag:",iam role ARN AuthRole for launching execution."`
Relaunch string `json:"relaunch" pflag:",execution id to be relaunched."`
Recover string `json:"recover" pflag:",execution id to be recreated from the last known failure point."`
DryRun bool `json:"dryRun" pflag:",execute command without making any modifications."`
Version string `json:"version" pflag:",specify version of execution workflow/task."`
ClusterPool string `json:"clusterPool" pflag:",specify which cluster pool to assign execution to."`
OverwriteCache bool `json:"overwriteCache" pflag:",skip cached results when performing execution,causing all outputs to be re-calculated and stored data to be overwritten. Does not work for recovered executions."`
// Non plfag section is read from the execution config generated by get task/launch plan
Workflow string `json:"workflow,omitempty"`
Task string `json:"task,omitempty"`
Expand Down
4 changes: 3 additions & 1 deletion cmd/create/execution_util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,8 @@ func relaunchExecution(ctx context.Context, executionName string, project string
Project: project,
Domain: domain,
},
Name: targetExecutionName,
Name: targetExecutionName,
OverwriteCache: executionConfig.OverwriteCache,
})
if err != nil {
return err
Expand Down Expand Up @@ -167,6 +168,7 @@ func createExecutionRequest(ID *core.Identifier, inputs *core.LiteralMap, securi
AuthRole: authRole,
SecurityContext: securityContext,
ClusterAssignment: clusterAssignment,
OverwriteCache: executionConfig.OverwriteCache,
},
Inputs: inputs,
}
Expand Down
10 changes: 10 additions & 0 deletions cmd/create/execution_util_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,3 +212,13 @@ func Test_resolveOverrides(t *testing.T) {
assert.Equal(t, "v1", cfg.Version)
assert.Equal(t, "gpu", cfg.ClusterPool)
}

func TestCreateExecutionForRelaunchOverwritingCache(t *testing.T) {
s := setup()
createExecutionUtilSetup()
executionConfig.OverwriteCache = true
relaunchRequest.OverwriteCache = true // ensure request has overwriteCache param set
s.MockAdminClient.OnRelaunchExecutionMatch(s.Ctx, relaunchRequest).Return(executionCreateResponse, nil)
err := relaunchExecution(s.Ctx, "execName", config.GetConfig().Project, config.GetConfig().Domain, s.CmdCtx, executionConfig, "")
assert.Nil(t, err)
}
7 changes: 4 additions & 3 deletions cmd/create/executionconfig_flags.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 14 additions & 0 deletions cmd/create/executionconfig_flags_test.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

107 changes: 53 additions & 54 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,31 +9,30 @@ require (
github.com/docker/docker v20.10.7+incompatible
github.com/docker/go-connections v0.4.0
github.com/enescakir/emoji v1.0.0
github.com/flyteorg/flyteidl v1.1.19
github.com/flyteorg/flytestdlib v1.0.0
github.com/flyteorg/flyteidl v1.2.5
github.com/flyteorg/flytestdlib v1.0.13
github.com/ghodss/yaml v1.0.0
github.com/go-ozzo/ozzo-validation/v4 v4.3.0
github.com/golang/protobuf v1.5.0
github.com/golang/protobuf v1.5.2
github.com/google/go-github/v42 v42.0.0
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/uuid v1.2.0
github.com/hashicorp/go-version v1.3.0
github.com/kataras/tablewriter v0.0.0-20180708051242-e063d29b7c23
github.com/landoop/tableprinter v0.0.0-20180806200924-8bd8c2576d27
github.com/mitchellh/mapstructure v1.4.1
github.com/mitchellh/mapstructure v1.4.3
github.com/mouuff/go-rocket-update v1.5.1
github.com/opencontainers/image-spec v1.0.2
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4
github.com/pkg/errors v0.9.1
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.1.3
github.com/spf13/cobra v1.4.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.7.0
github.com/stretchr/testify v1.7.1
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0
github.com/zalando/go-keyring v0.1.1
golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93
google.golang.org/grpc v1.36.0
google.golang.org/protobuf v1.27.1
golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5
google.golang.org/grpc v1.46.0
google.golang.org/protobuf v1.28.0
gopkg.in/yaml.v2 v2.4.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
gotest.tools v2.2.0+incompatible
Expand All @@ -49,47 +48,51 @@ require (
)

require (
cloud.google.com/go v0.78.0 // indirect
cloud.google.com/go/storage v1.12.0 // indirect
github.com/Azure/azure-sdk-for-go v62.3.0+incompatible // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 // indirect
cloud.google.com/go v0.101.0 // indirect
cloud.google.com/go/compute v1.6.1 // indirect
cloud.google.com/go/iam v0.3.0 // indirect
cloud.google.com/go/storage v1.22.0 // indirect
github.com/Azure/azure-sdk-for-go v63.4.0+incompatible // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.23.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v0.9.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.0 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/Azure/go-autorest/autorest v0.11.17 // indirect
github.com/Azure/go-autorest/autorest/adal v0.9.10 // indirect
github.com/Azure/go-autorest/autorest v0.11.27 // indirect
github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
github.com/Azure/go-autorest/logger v0.2.0 // indirect
github.com/Azure/go-autorest/logger v0.2.1 // indirect
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/Microsoft/go-winio v0.5.0 // indirect
github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 // indirect
github.com/aws/aws-sdk-go v1.37.3 // indirect
github.com/aws/aws-sdk-go v1.44.2 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash v1.1.0 // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/containerd/containerd v1.5.10 // indirect
github.com/coocood/freecache v1.1.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
github.com/danieljoos/wincred v1.1.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/distribution v2.8.0+incompatible // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/evanphx/json-patch v4.9.0+incompatible // indirect
github.com/fatih/color v1.10.0 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/flyteorg/flyteplugins v1.0.0 // indirect
github.com/flyteorg/stow v0.3.3 // indirect
github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/flyteorg/stow v0.3.6 // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/go-logr/logr v0.4.0 // indirect
github.com/godbus/dbus/v5 v5.0.4 // indirect
github.com/gofrs/uuid v4.2.0+incompatible // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/google/go-cmp v0.5.6 // indirect
github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/go-cmp v0.5.8 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/googleapis/gax-go/v2 v2.0.5 // indirect
github.com/googleapis/gax-go/v2 v2.3.0 // indirect
github.com/googleapis/gnostic v0.5.1 // indirect
github.com/googleapis/go-type-adapters v1.0.0 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
Expand All @@ -100,11 +103,10 @@ require (
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/jstemmer/go-junit-report v0.9.1 // indirect
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
github.com/magiconair/properties v1.8.4 // indirect
github.com/mattn/go-colorable v0.1.8 // indirect
github.com/mattn/go-isatty v0.0.13 // indirect
github.com/magiconair/properties v1.8.6 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect
Expand All @@ -113,37 +115,34 @@ require (
github.com/morikuni/aec v1.0.0 // indirect
github.com/ncw/swift v1.0.53 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/pelletier/go-toml v1.8.1 // indirect
github.com/pelletier/go-toml v1.9.4 // indirect
github.com/pelletier/go-toml/v2 v2.0.0-beta.8 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.10.0 // indirect
github.com/prometheus/client_golang v1.12.1 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.18.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/russross/blackfriday/v2 v2.0.1 // indirect
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
github.com/spf13/afero v1.5.1 // indirect
github.com/spf13/cast v1.3.1 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/spf13/afero v1.8.2 // indirect
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/viper v1.7.1 // indirect
github.com/spf13/viper v1.11.0 // indirect
github.com/stretchr/objx v0.3.0 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
go.opencensus.io v0.22.6 // indirect
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
go.opencensus.io v0.23.0 // indirect
golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect
golang.org/x/exp v0.0.0-20220428152302-39d4317da171 // indirect
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 // indirect
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
golang.org/x/tools v0.1.10 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/api v0.40.0 // indirect
golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
google.golang.org/api v0.76.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c // indirect
google.golang.org/genproto v0.0.0-20220426171045-31bebdecfb46 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.62.0 // indirect
gopkg.in/ini.v1 v1.66.4 // indirect
k8s.io/klog/v2 v2.8.0 // indirect
k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 // indirect
k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 // indirect
Expand Down
Loading