forked from kubeflow/kubeflow
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
KfConfig - Internal data structure for holding config info (kubeflow#…
…4210) * KfctlConfig init * add more fields * add status and cache * link to gcp * add GetPluginSpec * add condition access funcs * add condition helper * add plugin access funcs * add plugin spec writer * remove functiosn from kfdef v1beta1 * remove kfloader * add sourceVersion * LoadConfigFromURI init * WriteConfigToFile init * copy applications * copy secrets * copy repos * copy plugins * add runtime fields * copy status * beta transformer * temp commit * updates * serialize plugins * serialize secrets * serialize repos * serialize cache * add interface * finish v1alpha1 * change naming * v1beta1 converter * test init * v1alpha1 test * updates * beta test init * finish test * fix * remove temp files * fix * KfConfig converter * appDir * write file * rename * fix test * remove temp codes * remove unused field * fix * add fields * fill in new fields in v1alpha1 * fix tests * fill fields in v1beta1
- Loading branch information
1 parent
4a1c06b
commit db672b8
Showing
21 changed files
with
2,622 additions
and
1,124 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,165 @@ | ||
| package configconverters | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "github.com/ghodss/yaml" | ||
| gogetter "github.com/hashicorp/go-getter" | ||
| kfapis "github.com/kubeflow/kubeflow/bootstrap/v3/pkg/apis" | ||
| kfconfig "github.com/kubeflow/kubeflow/bootstrap/v3/pkg/apis/apps/kfconfig" | ||
| log "github.com/sirupsen/logrus" | ||
| "io/ioutil" | ||
| netUrl "net/url" | ||
| "path" | ||
| "strings" | ||
| ) | ||
|
|
||
| type Converter interface { | ||
| ToKfConfig(appdir string, kfdefBytes []byte) (*kfconfig.KfConfig, error) | ||
| ToKfDefSerialized(config kfconfig.KfConfig) ([]byte, error) | ||
| } | ||
|
|
||
| const ( | ||
| KfConfigFile = "app.yaml" | ||
| Api = "kfdef.apps.kubeflow.org" | ||
| ) | ||
|
|
||
| func isValidUrl(toTest string) bool { | ||
| _, err := netUrl.ParseRequestURI(toTest) | ||
| if err != nil { | ||
| return false | ||
| } else { | ||
| return true | ||
| } | ||
| } | ||
|
|
||
| func LoadConfigFromURI(configFile string) (*kfconfig.KfConfig, error) { | ||
| if configFile == "" { | ||
| return nil, fmt.Errorf("config file must be the URI of a KfDef spec") | ||
| } | ||
|
|
||
| // TODO(jlewi): We should check if configFile doesn't specify a protocol or the protocol | ||
| // is file:// then we can just read it rather than fetching with go-getter. | ||
| appDir, err := ioutil.TempDir("", "") | ||
| if err != nil { | ||
| return nil, fmt.Errorf("Create a temporary directory to copy the file to.") | ||
| } | ||
| // Open config file | ||
| // | ||
| // TODO(jlewi): Should we use hashicorp go-getter.GetAny here? We use that to download | ||
| // the tarballs for the repos. Maybe we should use that here as well to be consistent. | ||
| appFile := path.Join(appDir, KfConfigFile) | ||
|
|
||
| log.Infof("Downloading %v to %v", configFile, appFile) | ||
| configFileUri, err := netUrl.Parse(configFile) | ||
| if err != nil { | ||
| log.Errorf("could not parse configFile url") | ||
| } | ||
| if isValidUrl(configFile) { | ||
| errGet := gogetter.GetFile(appFile, configFile) | ||
| if errGet != nil { | ||
| return nil, &kfapis.KfError{ | ||
| Code: int(kfapis.INVALID_ARGUMENT), | ||
| Message: fmt.Sprintf("could not fetch specified config %s: %v", configFile, err), | ||
| } | ||
| } | ||
| } else { | ||
| g := new(gogetter.FileGetter) | ||
| g.Copy = true | ||
| errGet := g.GetFile(appFile, configFileUri) | ||
| if errGet != nil { | ||
| return nil, &kfapis.KfError{ | ||
| Code: int(kfapis.INVALID_ARGUMENT), | ||
| Message: fmt.Sprintf("could not fetch specified config %s: %v", configFile, err), | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // Read contents | ||
| configFileBytes, err := ioutil.ReadFile(appFile) | ||
| if err != nil { | ||
| return nil, &kfapis.KfError{ | ||
| Code: int(kfapis.INTERNAL_ERROR), | ||
| Message: fmt.Sprintf("could not read from config file %s: %v", configFile, err), | ||
| } | ||
| } | ||
|
|
||
| // Check API version. | ||
| var obj map[string]interface{} | ||
| if err = yaml.Unmarshal(configFileBytes, &obj); err != nil { | ||
| return nil, &kfapis.KfError{ | ||
| Code: int(kfapis.INVALID_ARGUMENT), | ||
| Message: fmt.Sprintf("invalid config file format: %v", err), | ||
| } | ||
| } | ||
| apiVersion, ok := obj["apiVersion"] | ||
| if !ok { | ||
| return nil, &kfapis.KfError{ | ||
| Code: int(kfapis.INVALID_ARGUMENT), | ||
| Message: "invalid config: apiVersion is not found.", | ||
| } | ||
| } | ||
| apiVersionSeparated := strings.Split(apiVersion.(string), "/") | ||
| if len(apiVersionSeparated) < 2 || apiVersionSeparated[0] != Api { | ||
| return nil, &kfapis.KfError{ | ||
| Code: int(kfapis.INVALID_ARGUMENT), | ||
| Message: fmt.Sprintf("invalid config: apiVersion must be in the format of %v/<version>, got %v", Api, apiVersion), | ||
| } | ||
| } | ||
|
|
||
| converters := map[string]Converter{ | ||
| "v1alpha1": V1alpha1{}, | ||
| "v1beta1": V1beta1{}, | ||
| } | ||
|
|
||
| converter, ok := converters[apiVersionSeparated[1]] | ||
| if !ok { | ||
| versions := []string{} | ||
| for key := range converters { | ||
| versions = append(versions, key) | ||
| } | ||
| return nil, &kfapis.KfError{ | ||
| Code: int(kfapis.INVALID_ARGUMENT), | ||
| Message: fmt.Sprintf("invalid config: version not supported; supported versions: %v, got %v", | ||
| strings.Join(versions, ", "), apiVersionSeparated[1]), | ||
| } | ||
| } | ||
| return converter.ToKfConfig(appDir, configFileBytes) | ||
| } | ||
|
|
||
| func WriteConfigToFile(config kfconfig.KfConfig, filename string) error { | ||
| converters := map[string]Converter{ | ||
| "v1alpha1": V1alpha1{}, | ||
| "v1beta1": V1beta1{}, | ||
| } | ||
| apiVersionSeparated := strings.Split(config.APIVersion, "/") | ||
| if len(apiVersionSeparated) < 2 || apiVersionSeparated[0] != Api { | ||
| return &kfapis.KfError{ | ||
| Code: int(kfapis.INVALID_ARGUMENT), | ||
| Message: fmt.Sprintf("invalid config: apiVersion must be in the format of %v/<version>, got %v", Api, config.APIVersion), | ||
| } | ||
| } | ||
|
|
||
| converter, ok := converters[apiVersionSeparated[1]] | ||
| if !ok { | ||
| return &kfapis.KfError{ | ||
| Code: int(kfapis.INVALID_ARGUMENT), | ||
| Message: fmt.Sprintf("invalid config: unable to find converter for version %v", config.APIVersion), | ||
| } | ||
| } | ||
|
|
||
| kfdefBytes, err := converter.ToKfDefSerialized(config) | ||
| if err != nil { | ||
| return &kfapis.KfError{ | ||
| Code: int(kfapis.INVALID_ARGUMENT), | ||
| Message: fmt.Sprintf("error when marshaling KfDef: %v", err), | ||
| } | ||
| } | ||
| err = ioutil.WriteFile(filename, kfdefBytes, 0644) | ||
| if err != nil { | ||
| return &kfapis.KfError{ | ||
| Code: int(kfapis.INTERNAL_ERROR), | ||
| Message: fmt.Sprintf("error when writing KfDef: %v", err), | ||
| } | ||
| } | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| package testdata |
Oops, something went wrong.