From f2b67e694a6c7d7ffa46b6cf31d6e0e537d7f636 Mon Sep 17 00:00:00 2001 From: Junxing Zhu Date: Wed, 6 Sep 2023 11:15:19 +0800 Subject: [PATCH] feat: generate kcl data from yaml (#150) --- go.mod | 5 +++ go.sum | 14 +++++++ pkg/tools/gen/genkcl.go | 6 +++ pkg/tools/gen/genkcl_json.go | 44 ++++----------------- pkg/tools/gen/genkcl_test.go | 14 +++++++ pkg/tools/gen/genkcl_yaml.go | 55 ++++++++++++++++++++++++++ pkg/tools/gen/testdata/json/expect.k | 4 +- pkg/tools/gen/testdata/yaml/expect.k | 41 +++++++++++++++++++ pkg/tools/gen/testdata/yaml/input.yaml | 21 ++++++++++ 9 files changed, 166 insertions(+), 38 deletions(-) create mode 100644 pkg/tools/gen/genkcl_yaml.go create mode 100644 pkg/tools/gen/testdata/yaml/expect.k create mode 100644 pkg/tools/gen/testdata/yaml/input.yaml diff --git a/go.mod b/go.mod index da3b1495..6dbe3db1 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.19 require ( github.com/chai2010/jsonv v1.1.3 github.com/chai2010/protorpc v1.1.4 + github.com/goccy/go-yaml v1.11.0 github.com/gofrs/flock v0.8.1 github.com/golang/protobuf v1.5.3 github.com/google/go-cmp v0.5.9 @@ -49,6 +50,7 @@ require ( github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect + github.com/fatih/color v1.10.0 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.4.1 // indirect github.com/go-git/go-git/v5 v5.6.1 // indirect @@ -63,6 +65,8 @@ require ( github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.16.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-colorable v0.1.8 // indirect + github.com/mattn/go-isatty v0.0.12 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect @@ -93,6 +97,7 @@ require ( golang.org/x/sys v0.7.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/tools v0.7.0 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect oras.land/oras-go v1.2.3 // indirect diff --git a/go.sum b/go.sum index 97053a26..2228aea6 100644 --- a/go.sum +++ b/go.sum @@ -126,6 +126,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= @@ -154,7 +156,12 @@ github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/goccy/go-yaml v1.11.0 h1:n7Z+zx8S9f9KgzG6KtQKf+kwqXZlLNR2F6018Dgau54= +github.com/goccy/go-yaml v1.11.0/go.mod h1:H+mJrWtjPTJAHvRbV09MCK9xYwODM+wRTVFFTWckfng= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -265,10 +272,15 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -523,6 +535,7 @@ golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -627,6 +640,7 @@ golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= diff --git a/pkg/tools/gen/genkcl.go b/pkg/tools/gen/genkcl.go index d059b7dc..01d0cc4f 100644 --- a/pkg/tools/gen/genkcl.go +++ b/pkg/tools/gen/genkcl.go @@ -9,6 +9,7 @@ import ( "strconv" "strings" + "github.com/goccy/go-yaml" "kcl-lang.io/kcl-go/pkg/logger" ) @@ -26,6 +27,7 @@ const ( ModeJsonSchema ModeTerraformSchema ModeJson + ModeYaml ) type kclGenerator struct { @@ -66,6 +68,8 @@ func (k *kclGenerator) GenSchema(w io.Writer, filename string, src interface{}) default: k.opts.Mode = ModeJson } + case yaml.Unmarshal(code, &i) == nil: + k.opts.Mode = ModeYaml default: return errors.New("failed to detect mode") } @@ -80,6 +84,8 @@ func (k *kclGenerator) GenSchema(w io.Writer, filename string, src interface{}) return k.genSchemaFromTerraformSchema(w, filename, src) case ModeJson: return k.genKclFromJsonData(w, filename, src) + case ModeYaml: + return k.genKclFromYaml(w, filename, src) default: return errors.New("unknown mode") } diff --git a/pkg/tools/gen/genkcl_json.go b/pkg/tools/gen/genkcl_json.go index ca5a3104..63ca195a 100644 --- a/pkg/tools/gen/genkcl_json.go +++ b/pkg/tools/gen/genkcl_json.go @@ -1,9 +1,8 @@ package gen import ( - "encoding/json" + "github.com/goccy/go-yaml" "io" - "sort" ) func (k *kclGenerator) genKclFromJsonData(w io.Writer, filename string, src interface{}) error { @@ -11,44 +10,17 @@ func (k *kclGenerator) genKclFromJsonData(w io.Writer, filename string, src inte if err != nil { return err } - jsonData := &map[string]interface{}{} - if err = json.Unmarshal(code, jsonData); err != nil { + + // as yaml can be viewed as a superset of json, + // we can handle json data like yaml. + yamlData := &yaml.MapSlice{} + if err = yaml.UnmarshalWithOptions(code, yamlData, yaml.UseOrderedMap(), yaml.UseJSONUnmarshaler()); err != nil { return err } - // convert json data to kcl - result := convertKclFromJson(jsonData) + // convert to kcl + result := convertKclFromYaml(yamlData) // generate kcl code return k.genKcl(w, kclFile{Data: result}) } - -func convertKclFromJson(jsonData *map[string]interface{}) []data { - var result []data - for key, value := range *jsonData { - switch value := value.(type) { - case map[string]interface{}: - result = append(result, data{ - Key: key, - Value: convertKclFromJson(&value), - }) - case []interface{}: - var vals []interface{} - for _, v := range value { - switch v := v.(type) { - case map[string]interface{}: - vals = append(vals, convertKclFromJson(&v)) - default: - vals = append(vals, v) - } - } - result = append(result, data{Key: key, Value: vals}) - default: - result = append(result, data{Key: key, Value: value}) - } - } - sort.Slice(result, func(i, j int) bool { - return result[i].Key < result[j].Key - }) - return result -} diff --git a/pkg/tools/gen/genkcl_test.go b/pkg/tools/gen/genkcl_test.go index 73a44f57..59132f32 100644 --- a/pkg/tools/gen/genkcl_test.go +++ b/pkg/tools/gen/genkcl_test.go @@ -180,6 +180,20 @@ func TestGenKclFromJson(t *testing.T) { assert2.Equal(t, expect, string(bytes.ReplaceAll(result, []byte("\r\n"), []byte("\n")))) } +func TestGenKclFromYaml(t *testing.T) { + input := filepath.Join("testdata", "yaml", "input.yaml") + expectFilepath := filepath.Join("testdata", "yaml", "expect.k") + expect := readFileString(t, expectFilepath) + + var buf bytes.Buffer + err := GenKcl(&buf, input, nil, &GenKclOptions{}) + if err != nil { + t.Fatal(err) + } + result := buf.Bytes() + assert2.Equal(t, expect, string(bytes.ReplaceAll(result, []byte("\r\n"), []byte("\n")))) +} + func readFileString(t testing.TB, p string) (content string) { data, err := os.ReadFile(p) if err != nil { diff --git a/pkg/tools/gen/genkcl_yaml.go b/pkg/tools/gen/genkcl_yaml.go new file mode 100644 index 00000000..60ecdaa4 --- /dev/null +++ b/pkg/tools/gen/genkcl_yaml.go @@ -0,0 +1,55 @@ +package gen + +import ( + "github.com/goccy/go-yaml" + "io" +) + +func (k *kclGenerator) genKclFromYaml(w io.Writer, filename string, src interface{}) error { + code, err := readSource(filename, src) + if err != nil { + return err + } + + yamlData := &yaml.MapSlice{} + if err = yaml.UnmarshalWithOptions(code, yamlData, yaml.UseOrderedMap()); err != nil { + return err + } + + // convert yaml data to kcl + result := convertKclFromYaml(yamlData) + + // generate kcl code + return k.genKcl(w, kclFile{Data: result}) +} + +func convertKclFromYaml(yamlData *yaml.MapSlice) []data { + var result []data + for _, item := range *yamlData { + key, ok := item.Key.(string) + if !ok { + continue + } + switch value := item.Value.(type) { + case yaml.MapSlice: + result = append(result, data{ + Key: key, + Value: convertKclFromYaml(&value), + }) + case []interface{}: + var vals []interface{} + for _, v := range value { + switch v := v.(type) { + case yaml.MapSlice: + vals = append(vals, convertKclFromYaml(&v)) + default: + vals = append(vals, v) + } + } + result = append(result, data{Key: key, Value: vals}) + default: + result = append(result, data{Key: key, Value: value}) + } + } + return result +} diff --git a/pkg/tools/gen/testdata/json/expect.k b/pkg/tools/gen/testdata/json/expect.k index 090b16b8..394d7dbf 100644 --- a/pkg/tools/gen/testdata/json/expect.k +++ b/pkg/tools/gen/testdata/json/expect.k @@ -6,10 +6,10 @@ Editing this file might prove futile when you re-run the KCL auto-gen generate c apiVersion = "apps/v1" kind = "Deployment" metadata = { + name = "nginx-deployment" labels = { app = "nginx" } - name = "nginx-deployment" } spec = { replicas = 3 @@ -27,8 +27,8 @@ spec = { spec = { containers = [ { - image = "nginx:latest" name = "nginx-container" + image = "nginx:latest" ports = [ { containerPort = 80 diff --git a/pkg/tools/gen/testdata/yaml/expect.k b/pkg/tools/gen/testdata/yaml/expect.k new file mode 100644 index 00000000..f7b2eb0a --- /dev/null +++ b/pkg/tools/gen/testdata/yaml/expect.k @@ -0,0 +1,41 @@ +""" +This file was generated by the KCL auto-gen tool. DO NOT EDIT. +Editing this file might prove futile when you re-run the KCL auto-gen generate command. +""" + +apiVersion = "apps/v1" +kind = "Deployment" +metadata = { + name = "nginx-deployment" + labels = { + app = "nginx" + } +} +spec = { + replicas = 3 + selector = { + matchLabels = { + app = "nginx" + } + } + template = { + metadata = { + labels = { + app = "nginx" + } + } + spec = { + containers = [ + { + name = "nginx" + image = "nginx:latest" + ports = [ + { + containerPort = 80 + } + ] + } + ] + } + } +} diff --git a/pkg/tools/gen/testdata/yaml/input.yaml b/pkg/tools/gen/testdata/yaml/input.yaml new file mode 100644 index 00000000..9f946805 --- /dev/null +++ b/pkg/tools/gen/testdata/yaml/input.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + labels: + app: nginx +spec: + replicas: 3 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: 'nginx:latest' + ports: + - containerPort: 80