Skip to content

Commit

Permalink
Bump flyteorg/stow to 0.3.8 (#4312)
Browse files Browse the repository at this point in the history 
* Bump flyteorg/stow to 0.3.8

Signed-off-by: Eduardo Apolinario <[email protected]>

* Update configmap and helper to support azure storage (#4328)

* go mod tidy

Signed-off-by: Haytham Abuelfutuh <[email protected]>

* Update stow in flytestdlib

Signed-off-by: Eduardo Apolinario <[email protected]>

---------

Signed-off-by: Eduardo Apolinario <[email protected]>
Signed-off-by: Haytham Abuelfutuh <[email protected]>
Co-authored-by: Eduardo Apolinario <[email protected]>
Co-authored-by: cgrass <[email protected]>
Co-authored-by: Haytham Abuelfutuh <[email protected]>
  • Loading branch information
@eapolinario @cgrass @EngHabu
4 people authored Nov 28, 2023
1 parent 4ce0bf0 commit 5da5619
Show file tree
Hide file tree
Showing 28 changed files with 399 additions and 552 deletions.
11 changes: 11 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,3 +100,14 @@ build_native_flyte:
docker build \
--build-arg FLYTECONSOLE_VERSION=$(FLYTECONSOLE_VERSION) \
--tag flyte-binary:native .

.PHONY: go-tidy
go-tidy:
go mod tidy
make -C datacatalog go-tidy
make -C flyteadmin go-tidy
make -C flyteidl go-tidy
make -C flytepropeller go-tidy
make -C flyteplugins go-tidy
make -C flytestdlib go-tidy
make -C flytecopilot go-tidy
4 changes: 4 additions & 0 deletions charts/flyte-binary/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,10 @@ Chart for basic single Flyte executable deployment
| configuration.logging.plugins.stackdriver.templateUri | string | `""` | |
| configuration.storage.metadataContainer | string | `"my-organization-flyte-container"` | |
| configuration.storage.provider | string | `"s3"` | |
| configuration.storage.providerConfig.azure.account | string | `"storage-account-name"` | |
| configuration.storage.providerConfig.azure.configDomainSuffix | string | `""` | |
| configuration.storage.providerConfig.azure.configUploadConcurrency | int | `4` | |
| configuration.storage.providerConfig.azure.key | string | `""` | |
| configuration.storage.providerConfig.gcs.project | string | `"my-organization-gcp-project"` | |
| configuration.storage.providerConfig.s3.accessKey | string | `""` | |
| configuration.storage.providerConfig.s3.authType | string | `"iam"` | |
Expand Down
2 changes: 2 additions & 0 deletions charts/flyte-binary/templates/_helpers.tpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ Get the Flyte user data prefix.
{{- printf "s3://%s/data" $userDataContainer -}}
{{- else if eq "gcs" .Values.configuration.storage.provider -}}
{{- printf "gs://%s/data" $userDataContainer -}}
{{- else if eq "azure" .Values.configuration.storage.provider -}}
{{- printf "abfs://%s/data" $userDataContainer -}}
{{- end -}}
{{- end -}}

Expand Down
15 changes: 15 additions & 0 deletions charts/flyte-binary/templates/configmap.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,21 @@ data:
{{- printf "Invalid value for S3 storage provider authentication type. Expected one of (iam, accesskey), but got: %s" .authType | fail }}
{{- end }}
{{- end }}
{{- else if eq "azure" .provider }}
{{- with .providerConfig.azure }}
kind: azure
config:
account: {{ .account }}
{{- if .key }}
key: {{ .key }}
{{- end }}
{{- if .configDomainSuffix }}
configDomainSuffix: {{ .configDomainSuffix }}
{{- end }}
{{- if .configUploadConcurrency }}
configUploadConcurrency: {{ .configUploadConcurrency }}
{{- end }}
{{- end }}
{{- else if eq "gcs" .provider }}
kind: google
config:
Expand Down
10 changes: 10 additions & 0 deletions charts/flyte-binary/values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,16 @@ configuration:
gcs:
# project Google Cloud project in which bucket resides
project: "my-organization-gcp-project"
# azure Provider configuration for Azure object store
azure:
# configDomainSuffix Domain name suffix
configDomainSuffix: ""
# configUploadConcurrency Upload Concurrency (default 4)
configUploadConcurrency: 4
# account Storage Account name
account: "storage-account-name"
# key Storage Account key if used
key: ""
# logging Specify configuration for logs emitted by Flyte
logging:
# level Set the log level
Expand Down
4 changes: 4 additions & 0 deletions datacatalog/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,7 @@ linux_compile:
.PHONY: generate
generate:
@go generate ./...

.PHONY: go-tidy
go-tidy:
go mod tidy
31 changes: 14 additions & 17 deletions datacatalog/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,16 +29,11 @@ require (
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v0.13.0 // indirect
cloud.google.com/go/storage v1.28.1 // indirect
github.com/Azure/azure-sdk-for-go v63.4.0+incompatible // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.23.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v0.9.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.0 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/Azure/go-autorest/autorest v0.11.27 // indirect
github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
github.com/Azure/go-autorest/logger v0.2.1 // indirect
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect
github.com/aws/aws-sdk-go v1.44.2 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash v1.1.0 // indirect
Expand All @@ -48,7 +43,7 @@ require (
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/flyteorg/stow v0.3.7 // indirect
github.com/flyteorg/stow v0.3.8 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-logr/logr v1.2.4 // indirect
Expand All @@ -57,12 +52,12 @@ require (
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/google/uuid v1.3.1 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
github.com/googleapis/gax-go/v2 v2.7.1 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
Expand All @@ -79,6 +74,7 @@ require (
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/magiconair/properties v1.8.6 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
Expand All @@ -91,6 +87,7 @@ require (
github.com/ncw/swift v1.0.53 // indirect
github.com/pelletier/go-toml v1.9.4 // indirect
github.com/pelletier/go-toml/v2 v2.0.0-beta.8 // indirect
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.16.0 // indirect
Expand All @@ -110,13 +107,13 @@ require (
go.opentelemetry.io/otel/metric v1.19.0 // indirect
go.opentelemetry.io/otel/sdk v1.19.0 // indirect
go.opentelemetry.io/otel/trace v1.19.0 // indirect
golang.org/x/crypto v0.11.0 // indirect
golang.org/x/crypto v0.13.0 // indirect
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
golang.org/x/net v0.13.0 // indirect
golang.org/x/net v0.15.0 // indirect
golang.org/x/oauth2 v0.8.0 // indirect
golang.org/x/sys v0.12.0 // indirect
golang.org/x/term v0.10.0 // indirect
golang.org/x/text v0.11.0 // indirect
golang.org/x/term v0.12.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/api v0.114.0 // indirect
Expand Down
Loading

0 comments on commit 5da5619

Please sign in to comment.