Skip to content

Commit

Permalink
feat: generate kcl data from yaml (#150)
Browse files Browse the repository at this point in the history
  • Loading branch information
jakezhu9 authored Sep 6, 2023
1 parent f7b33b7 commit f2b67e6
Show file tree
Hide file tree
Showing 9 changed files with 166 additions and 38 deletions.
5 changes: 5 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ go 1.19
require (
github.com/chai2010/jsonv v1.1.3
github.com/chai2010/protorpc v1.1.4
github.com/goccy/go-yaml v1.11.0
github.com/gofrs/flock v0.8.1
github.com/golang/protobuf v1.5.3
github.com/google/go-cmp v0.5.9
Expand Down Expand Up @@ -49,6 +50,7 @@ require (
github.com/docker/go-metrics v0.0.1 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/fatih/color v1.10.0 // indirect
github.com/go-git/gcfg v1.5.0 // indirect
github.com/go-git/go-billy/v5 v5.4.1 // indirect
github.com/go-git/go-git/v5 v5.6.1 // indirect
Expand All @@ -63,6 +65,8 @@ require (
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/klauspost/compress v1.16.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.8 // indirect
github.com/mattn/go-isatty v0.0.12 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/moby/locker v1.0.1 // indirect
github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
Expand Down Expand Up @@ -93,6 +97,7 @@ require (
golang.org/x/sys v0.7.0 // indirect
golang.org/x/text v0.9.0 // indirect
golang.org/x/tools v0.7.0 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
oras.land/oras-go v1.2.3 // indirect
Expand Down
14 changes: 14 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
Expand Down Expand Up @@ -154,7 +156,12 @@ github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/goccy/go-yaml v1.11.0 h1:n7Z+zx8S9f9KgzG6KtQKf+kwqXZlLNR2F6018Dgau54=
github.com/goccy/go-yaml v1.11.0/go.mod h1:H+mJrWtjPTJAHvRbV09MCK9xYwODM+wRTVFFTWckfng=
github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
Expand Down Expand Up @@ -265,10 +272,15 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
Expand Down Expand Up @@ -523,6 +535,7 @@ golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand Down Expand Up @@ -627,6 +640,7 @@ golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
Expand Down
6 changes: 6 additions & 0 deletions pkg/tools/gen/genkcl.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"strconv"
"strings"

"github.com/goccy/go-yaml"
"kcl-lang.io/kcl-go/pkg/logger"
)

Expand All @@ -26,6 +27,7 @@ const (
ModeJsonSchema
ModeTerraformSchema
ModeJson
ModeYaml
)

type kclGenerator struct {
Expand Down Expand Up @@ -66,6 +68,8 @@ func (k *kclGenerator) GenSchema(w io.Writer, filename string, src interface{})
default:
k.opts.Mode = ModeJson
}
case yaml.Unmarshal(code, &i) == nil:
k.opts.Mode = ModeYaml
default:
return errors.New("failed to detect mode")
}
Expand All @@ -80,6 +84,8 @@ func (k *kclGenerator) GenSchema(w io.Writer, filename string, src interface{})
return k.genSchemaFromTerraformSchema(w, filename, src)
case ModeJson:
return k.genKclFromJsonData(w, filename, src)
case ModeYaml:
return k.genKclFromYaml(w, filename, src)
default:
return errors.New("unknown mode")
}
Expand Down
44 changes: 8 additions & 36 deletions pkg/tools/gen/genkcl_json.go
Original file line number Diff line number Diff line change
@@ -1,54 +1,26 @@
package gen

import (
"encoding/json"
"github.com/goccy/go-yaml"
"io"
"sort"
)

func (k *kclGenerator) genKclFromJsonData(w io.Writer, filename string, src interface{}) error {
code, err := readSource(filename, src)
if err != nil {
return err
}
jsonData := &map[string]interface{}{}
if err = json.Unmarshal(code, jsonData); err != nil {

// as yaml can be viewed as a superset of json,
// we can handle json data like yaml.
yamlData := &yaml.MapSlice{}
if err = yaml.UnmarshalWithOptions(code, yamlData, yaml.UseOrderedMap(), yaml.UseJSONUnmarshaler()); err != nil {
return err
}

// convert json data to kcl
result := convertKclFromJson(jsonData)
// convert to kcl
result := convertKclFromYaml(yamlData)

// generate kcl code
return k.genKcl(w, kclFile{Data: result})
}

func convertKclFromJson(jsonData *map[string]interface{}) []data {
var result []data
for key, value := range *jsonData {
switch value := value.(type) {
case map[string]interface{}:
result = append(result, data{
Key: key,
Value: convertKclFromJson(&value),
})
case []interface{}:
var vals []interface{}
for _, v := range value {
switch v := v.(type) {
case map[string]interface{}:
vals = append(vals, convertKclFromJson(&v))
default:
vals = append(vals, v)
}
}
result = append(result, data{Key: key, Value: vals})
default:
result = append(result, data{Key: key, Value: value})
}
}
sort.Slice(result, func(i, j int) bool {
return result[i].Key < result[j].Key
})
return result
}
14 changes: 14 additions & 0 deletions pkg/tools/gen/genkcl_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,20 @@ func TestGenKclFromJson(t *testing.T) {
assert2.Equal(t, expect, string(bytes.ReplaceAll(result, []byte("\r\n"), []byte("\n"))))
}

func TestGenKclFromYaml(t *testing.T) {
input := filepath.Join("testdata", "yaml", "input.yaml")
expectFilepath := filepath.Join("testdata", "yaml", "expect.k")
expect := readFileString(t, expectFilepath)

var buf bytes.Buffer
err := GenKcl(&buf, input, nil, &GenKclOptions{})
if err != nil {
t.Fatal(err)
}
result := buf.Bytes()
assert2.Equal(t, expect, string(bytes.ReplaceAll(result, []byte("\r\n"), []byte("\n"))))
}

func readFileString(t testing.TB, p string) (content string) {
data, err := os.ReadFile(p)
if err != nil {
Expand Down
55 changes: 55 additions & 0 deletions pkg/tools/gen/genkcl_yaml.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package gen

import (
"github.com/goccy/go-yaml"
"io"
)

func (k *kclGenerator) genKclFromYaml(w io.Writer, filename string, src interface{}) error {
code, err := readSource(filename, src)
if err != nil {
return err
}

yamlData := &yaml.MapSlice{}
if err = yaml.UnmarshalWithOptions(code, yamlData, yaml.UseOrderedMap()); err != nil {
return err
}

// convert yaml data to kcl
result := convertKclFromYaml(yamlData)

// generate kcl code
return k.genKcl(w, kclFile{Data: result})
}

func convertKclFromYaml(yamlData *yaml.MapSlice) []data {
var result []data
for _, item := range *yamlData {
key, ok := item.Key.(string)
if !ok {
continue
}
switch value := item.Value.(type) {
case yaml.MapSlice:
result = append(result, data{
Key: key,
Value: convertKclFromYaml(&value),
})
case []interface{}:
var vals []interface{}
for _, v := range value {
switch v := v.(type) {
case yaml.MapSlice:
vals = append(vals, convertKclFromYaml(&v))
default:
vals = append(vals, v)
}
}
result = append(result, data{Key: key, Value: vals})
default:
result = append(result, data{Key: key, Value: value})
}
}
return result
}
4 changes: 2 additions & 2 deletions pkg/tools/gen/testdata/json/expect.k
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ Editing this file might prove futile when you re-run the KCL auto-gen generate c
apiVersion = "apps/v1"
kind = "Deployment"
metadata = {
name = "nginx-deployment"
labels = {
app = "nginx"
}
name = "nginx-deployment"
}
spec = {
replicas = 3
Expand All @@ -27,8 +27,8 @@ spec = {
spec = {
containers = [
{
image = "nginx:latest"
name = "nginx-container"
image = "nginx:latest"
ports = [
{
containerPort = 80
Expand Down
41 changes: 41 additions & 0 deletions pkg/tools/gen/testdata/yaml/expect.k
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
"""
This file was generated by the KCL auto-gen tool. DO NOT EDIT.
Editing this file might prove futile when you re-run the KCL auto-gen generate command.
"""

apiVersion = "apps/v1"
kind = "Deployment"
metadata = {
name = "nginx-deployment"
labels = {
app = "nginx"
}
}
spec = {
replicas = 3
selector = {
matchLabels = {
app = "nginx"
}
}
template = {
metadata = {
labels = {
app = "nginx"
}
}
spec = {
containers = [
{
name = "nginx"
image = "nginx:latest"
ports = [
{
containerPort = 80
}
]
}
]
}
}
}
21 changes: 21 additions & 0 deletions pkg/tools/gen/testdata/yaml/input.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-deployment
labels:
app: nginx
spec:
replicas: 3
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: 'nginx:latest'
ports:
- containerPort: 80

0 comments on commit f2b67e6

Please sign in to comment.