KfConfig changes (kubeflow#4295)

* port changes from 4250 * cleanup
ricardov1 · Oct 15, 2019 · 27da9c8 · 27da9c8
1 parent 82a6ab7
commit 27da9c8
Show file tree

Hide file tree

Showing 9 changed files with 415 additions and 27 deletions.
diff --git a/bootstrap/pkg/apis/apps/configconverters/converters.go b/bootstrap/pkg/apis/apps/configconverters/converters.go
@@ -2,6 +2,8 @@ package configconverters
 
 import (
 	"fmt"
+	"os"
+
 	"github.com/ghodss/yaml"
 	gogetter "github.com/hashicorp/go-getter"
 	kfapis "github.com/kubeflow/kubeflow/bootstrap/v3/pkg/apis"
@@ -123,7 +125,15 @@ func LoadConfigFromURI(configFile string) (*kfconfig.KfConfig, error) {
 				strings.Join(versions, ", "), apiVersionSeparated[1]),
 		}
 	}
-	return converter.ToKfConfig(appDir, configFileBytes)
+	cwd, err := os.Getwd()
+	if err != nil {
+		return nil, &kfapis.KfError{
+			Code:    int(kfapis.INTERNAL_ERROR),
+			Message: fmt.Sprintf("could not get current directory for KfDef %v", err),
+		}
+	}
+
+	return converter.ToKfConfig(cwd, configFileBytes)
 }
 
 func WriteConfigToFile(config kfconfig.KfConfig, filename string) error {

diff --git a/bootstrap/pkg/apis/apps/configconverters/testdata/kfconfig_v1alpha1.yaml b/bootstrap/pkg/apis/apps/configconverters/testdata/kfconfig_v1alpha1.yaml
@@ -6,6 +6,7 @@ metadata:
   namespace: kubeflow
 spec:
   appDir: /tmp/myapp2
+  platform: gcp
   project: foo-project
   email: [email protected]
   applications:
@@ -331,4 +332,6 @@ spec:
   repos:
   - name: manifests
     uri: https://github.com/kubeflow/manifests/archive/master.tar.gz
+  skipInitProject: true
+  useIstio: true
 status: {}
diff --git a/bootstrap/pkg/apis/apps/configconverters/testdata/kfconfig_v1beta1.yaml b/bootstrap/pkg/apis/apps/configconverters/testdata/kfconfig_v1beta1.yaml
@@ -330,4 +330,5 @@ spec:
   repos:
   - name: manifests
     uri: https://github.com/kubeflow/manifests/archive/master.tar.gz
+  SkipInitProject: true
 status: {}
diff --git a/bootstrap/pkg/apis/apps/configconverters/v1alpha1.go b/bootstrap/pkg/apis/apps/configconverters/v1alpha1.go
@@ -9,6 +9,7 @@ import (
 	kfconfig "github.com/kubeflow/kubeflow/bootstrap/v3/pkg/apis/apps/kfconfig"
 	kfdeftypes "github.com/kubeflow/kubeflow/bootstrap/v3/pkg/apis/apps/kfdef/v1alpha1"
 	kfgcp "github.com/kubeflow/kubeflow/bootstrap/v3/pkg/kfapp/gcp"
+	log "github.com/sirupsen/logrus"
 )
 
 // Empty struct - used to implement Converter interface.
@@ -52,6 +53,7 @@ func copyGcpPluginSpec(from *kfdeftypes.KfDef, to *kfconfig.KfConfig) error {
 }
 
 func (v V1alpha1) ToKfConfig(appdir string, kfdefBytes []byte) (*kfconfig.KfConfig, error) {
+	log.Infof("converting to kfconfig, appdir=%v", appdir)
 	kfdef := &kfdeftypes.KfDef{}
 	if err := yaml.Unmarshal(kfdefBytes, kfdef); err != nil {
 		return nil, &kfapis.KfError{
@@ -62,13 +64,16 @@ func (v V1alpha1) ToKfConfig(appdir string, kfdefBytes []byte) (*kfconfig.KfConf
 
 	config := &kfconfig.KfConfig{
 		Spec: kfconfig.KfConfigSpec{
-			AppDir:       kfdef.Spec.AppDir,
-			UseBasicAuth: kfdef.Spec.UseBasicAuth,
-			Project:      kfdef.Spec.Project,
-			Email:        kfdef.Spec.Email,
-			IpName:       kfdef.Spec.IpName,
-			Hostname:     kfdef.Spec.Hostname,
-			Zone:         kfdef.Spec.Zone,
+			AppDir:          kfdef.Spec.AppDir,
+			UseBasicAuth:    kfdef.Spec.UseBasicAuth,
+			Project:         kfdef.Spec.Project,
+			Email:           kfdef.Spec.Email,
+			IpName:          kfdef.Spec.IpName,
+			Hostname:        kfdef.Spec.Hostname,
+			SkipInitProject: kfdef.Spec.SkipInitProject,
+			Zone:            kfdef.Spec.Zone,
+			Platform:        kfdef.Spec.Platform,
+			UseIstio:        true,
 		},
 	}
 	if config.Spec.AppDir == "" {
@@ -144,6 +149,10 @@ func (v V1alpha1) ToKfConfig(appdir string, kfdefBytes []byte) (*kfconfig.KfConf
 			src.EnvSource = &kfconfig.EnvSource{
 				Name: secret.SecretSource.EnvSource.Name,
 			}
+		} else if secret.SecretSource.HashedSource != nil {
+			src.HashedSource = &kfconfig.HashedSource{
+				HashedValue: secret.SecretSource.HashedSource.HashedValue,
+			}
 		}
 		s.SecretSource = src
 		config.Spec.Secrets = append(config.Spec.Secrets, s)
@@ -245,12 +254,13 @@ func (v V1alpha1) ToKfDefSerialized(config kfconfig.KfConfig) ([]byte, error) {
 			platform = kftypesv3.GCP
 		}
 	}
-	if platform == "" {
-		return []byte(""), &kfapis.KfError{
-			Code:    int(kfapis.INVALID_ARGUMENT),
-			Message: "Not able to find platform in plugins",
-		}
-	}
+	// TODO: Platform is not always needed?
+	// if platform == "" {
+	// 	return []byte(""), &kfapis.KfError{
+	// 		Code:    int(kfapis.INVALID_ARGUMENT),
+	// 		Message: "Not able to find platform in plugins",
+	// 	}
+	// }
 	kfdef.Spec.Platform = platform
 
 	for _, secret := range config.Spec.Secrets {
@@ -269,6 +279,11 @@ func (v V1alpha1) ToKfDefSerialized(config kfconfig.KfConfig) ([]byte, error) {
 					Name: secret.SecretSource.EnvSource.Name,
 				}
 			}
+			if secret.SecretSource.HashedSource != nil {
+				s.SecretSource.HashedSource = &kfdeftypes.HashedSource{
+					HashedValue: secret.SecretSource.HashedSource.HashedValue,
+				}
+			}
 		}
 		kfdef.Spec.Secrets = append(kfdef.Spec.Secrets, s)
 	}
@@ -281,6 +296,7 @@ func (v V1alpha1) ToKfDefSerialized(config kfconfig.KfConfig) ([]byte, error) {
 		kfdef.Spec.Repos = append(kfdef.Spec.Repos, r)
 	}
 
+	kfdef.Status = kfdeftypes.KfDefStatus{}
 	for _, cond := range config.Status.Conditions {
 		c := kfdeftypes.KfDefCondition{
 			Type:               kfdeftypes.KfDefConditionType(cond.Type),
@@ -293,6 +309,7 @@ func (v V1alpha1) ToKfDefSerialized(config kfconfig.KfConfig) ([]byte, error) {
 		kfdef.Status.Conditions = append(kfdef.Status.Conditions, c)
 	}
 
+	kfdef.Status.ReposCache = make(map[string]kfdeftypes.RepoCache)
 	for _, cache := range config.Status.Caches {
 		kfdef.Status.ReposCache[cache.Name] = kfdeftypes.RepoCache{
 			LocalPath: cache.LocalPath,

diff --git a/bootstrap/pkg/apis/apps/configconverters/v1alpha1_test.go b/bootstrap/pkg/apis/apps/configconverters/v1alpha1_test.go
@@ -2,6 +2,7 @@ package configconverters
 
 import (
 	"github.com/ghodss/yaml"
+	"github.com/google/go-cmp/cmp"
 	kftypes "github.com/kubeflow/kubeflow/bootstrap/v3/pkg/apis/apps"
 	kfconfig "github.com/kubeflow/kubeflow/bootstrap/v3/pkg/apis/apps/kfconfig"
 	kfdeftypes "github.com/kubeflow/kubeflow/bootstrap/v3/pkg/apis/apps/kfdef/v1alpha1"
@@ -56,7 +57,7 @@ func TestV1alpha1_ConvertToKfConfigs(t *testing.T) {
 		if !reflect.DeepEqual(config, expectedConfig) {
 			pGot := kfutils.PrettyPrint(config)
 			pWant := kfutils.PrettyPrint(expectedConfig)
-			t.Errorf("Loaded KfConfig doesn't match;\nexpected\n%v\ngot\n%v\n", pWant, pGot)
+			t.Errorf("Loaded KfConfig doesn't match: %v", cmp.Diff(pGot, pWant))
 		}
 	}
 }
@@ -126,7 +127,7 @@ func TestV1alpha1_ConvertToKfDef(t *testing.T) {
 		if !reflect.DeepEqual(got, want) {
 			pGot := kfutils.PrettyPrint(got)
 			pWant := kfutils.PrettyPrint(want)
-			t.Errorf("Loaded KfConfig doesn't match;\nexpected\n%v\ngot\n%v\n", pWant, pGot)
+			t.Errorf("Loaded KfConfig doesn't match: %v", cmp.Diff(pGot, pWant))
 		}
 	}
 }
diff --git a/bootstrap/pkg/apis/apps/configconverters/v1beta1.go b/bootstrap/pkg/apis/apps/configconverters/v1beta1.go
@@ -101,6 +101,9 @@ func (v V1beta1) ToKfConfig(appdir string, kfdefBytes []byte) (*kfconfig.KfConfi
 			if h, ok := s["hostname"]; ok {
 				config.Spec.Hostname = h.(string)
 			}
+			if h, ok := s["skipInitProject"]; ok {
+				config.Spec.SkipInitProject = h.(bool)
+			}
 			if z, ok := s["zone"]; ok {
 				config.Spec.Zone = z.(string)
 			}

diff --git a/bootstrap/pkg/apis/apps/configconverters/v1beta1_test.go b/bootstrap/pkg/apis/apps/configconverters/v1beta1_test.go
@@ -2,6 +2,7 @@ package configconverters
 
 import (
 	"github.com/ghodss/yaml"
+	"github.com/google/go-cmp/cmp"
 	kfconfig "github.com/kubeflow/kubeflow/bootstrap/v3/pkg/apis/apps/kfconfig"
 	kfutils "github.com/kubeflow/kubeflow/bootstrap/v3/pkg/utils"
 	"io/ioutil"
@@ -53,7 +54,7 @@ func TestV1beta1_expectedConfig(t *testing.T) {
 		if !reflect.DeepEqual(config, expectedConfig) {
 			pGot := kfutils.PrettyPrint(config)
 			pWant := kfutils.PrettyPrint(expectedConfig)
-			t.Errorf("Loaded KfConfig doesn't match;\nexpected\n%v\ngot\n%v\n", pWant, pGot)
+			t.Errorf("Loaded KfConfig doesn't match %v", cmp.Diff(pGot, pWant))
 		}
 	}