Skip to content
This repository has been archived by the owner on Oct 9, 2023. It is now read-only.

Add GetOutputMetadata to the output reader #289

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion go/tasks/pluginmachinery/core/allocationstatus_enumer.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion go/tasks/pluginmachinery/core/phase_enumer.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion go/tasks/pluginmachinery/core/transitiontype_enumer.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion go/tasks/pluginmachinery/internal/webapi/phase_enumer.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions go/tasks/pluginmachinery/io/iface.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ type OutputReader interface {
Read(ctx context.Context) (*core.LiteralMap, *ExecutionError, error)
// DeckExists checks if the deck file has been generated.
DeckExists(ctx context.Context) (bool, error)
// GetOutputMetadata get the metadata from the output of tasks, such as deck URI.
GetOutputMetadata(ctx context.Context) map[string]string
}

// CheckpointPaths provides the paths / keys to input Checkpoints directory and an output checkpoints directory.
Expand Down
34 changes: 34 additions & 0 deletions go/tasks/pluginmachinery/io/mocks/output_reader.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 7 additions & 0 deletions go/tasks/pluginmachinery/ioutils/in_memory_output_reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,13 @@ func (r InMemoryOutputReader) DeckExists(_ context.Context) (bool, error) {
return r.DeckPath != nil, nil
}

func (r InMemoryOutputReader) GetOutputMetadata(_ context.Context) map[string]string {
if r.DeckPath == nil {
return map[string]string{}
}
return map[string]string{deckURIKey: r.DeckPath.String()}
}

func NewInMemoryOutputReader(literals *core.LiteralMap, DeckPath *storage.DataReference, err *io.ExecutionError) InMemoryOutputReader {
return InMemoryOutputReader{
literals: literals,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ func TestInMemoryOutputReader(t *testing.T) {
or := NewInMemoryOutputReader(&flyteIdlCore.LiteralMap{Literals: lt}, &deckPath, nil)

assert.Equal(t, &deckPath, or.DeckPath)
assert.Equal(t, deckPath.String(), or.GetOutputMetadata(context.Background())[deckURIKey])
ctx := context.TODO()

ok, err := or.IsError(ctx)
Expand Down
2 changes: 2 additions & 0 deletions go/tasks/pluginmachinery/ioutils/paths.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ const (
// CheckpointPrefix specifies the common prefix that can be used as a directory where all the checkpoint information
// will be stored. This directory is under the raw output-prefix path
CheckpointPrefix = "_flytecheckpoints"

deckURIKey = "deck-uri"
)

// ConstructCheckpointPath returns a checkpoint path under the given `base` / rawOutputPrefix, following the conventions of
Expand Down
4 changes: 4 additions & 0 deletions go/tasks/pluginmachinery/ioutils/remote_file_output_reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,10 @@ func (r RemoteFileOutputReader) DeckExists(ctx context.Context) (bool, error) {
return md.Exists(), nil
}

func (r RemoteFileOutputReader) GetOutputMetadata(_ context.Context) map[string]string {
return map[string]string{deckURIKey: r.outPath.GetDeckPath().String()}
}

func NewRemoteFileOutputReader(_ context.Context, store storage.ComposedProtobufStore, outPaths io.OutputFilePaths, maxDatasetSize int64) RemoteFileOutputReader {
return RemoteFileOutputReader{
outPath: outPaths,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ func TestReadOrigin(t *testing.T) {
exists, err := r.DeckExists(ctx)
assert.NoError(t, err)
assert.True(t, exists)
assert.Equal(t, "deck.html", r.GetOutputMetadata(ctx)[deckURIKey])
})

t.Run("system", func(t *testing.T) {
Expand Down
1 change: 0 additions & 1 deletion go/tasks/pluginmachinery/workqueue/workstatus_enumer.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion go/tasks/plugins/array/core/phase_enumer.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion go/tasks/plugins/presto/client/prestostatus_enumer.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.