From a0473763c6212762ef7dc114d8f6cea183aec26a Mon Sep 17 00:00:00 2001 From: peefy Date: Mon, 19 Feb 2024 11:18:18 +0800 Subject: [PATCH] feat: add go plugin APIs and examples Signed-off-by: peefy --- README.md | 31 ++++- examples/kubernetes/main.go | 5 +- examples/plugin/main.go | 22 ++++ go.mod | 1 + go.sum | 2 + kclvm.go | 6 +- pkg/kcl/api.go | 29 ++--- pkg/kcl_plugin/hello_plugin/api.go | 102 ----------------- pkg/kcl_plugin/hello_plugin/api_test.go | 80 ------------- pkg/kcl_plugin/hello_plugin/hello.go | 74 ------------ pkg/kcl_plugin/plugin_nocgo.go | 12 -- pkg/kcl_plugin/plugin_py.go | 9 -- pkg/native/api.go | 35 ++++++ pkg/native/api_test.go | 20 ++++ pkg/native/cgo.go | 88 ++++++++++++++ pkg/native/client.go | 108 ++++++++++++++++++ pkg/native/client_test.go | 32 ++++++ pkg/native/loader.go | 40 +++++++ pkg/{kcl_plugin => plugin}/api.go | 2 +- pkg/{kcl_plugin => plugin}/api_test.go | 2 +- pkg/{kcl_plugin => plugin}/error.go | 24 ++-- pkg/plugin/hello_plugin/api.go | 21 ++++ pkg/plugin/hello_plugin/api_test.go | 16 +++ pkg/{kcl_plugin => plugin}/plugin.go | 19 ++- .../plugin_spec.go => plugin/spec.go} | 2 +- pkg/{kcl_plugin => plugin}/utils_c_string.go | 2 +- .../builtin_service_client.go | 2 +- pkg/{kclvm_runtime => runtime}/init.go | 2 +- pkg/{kclvm_runtime => runtime}/kclvm.go | 2 +- pkg/{kclvm_runtime => runtime}/proc.go | 16 +-- pkg/{kclvm_runtime => runtime}/runtime.go | 2 +- pkg/service/client_builtin_service.go | 6 +- pkg/service/client_kclvm_service.go | 10 +- pkg/service/testmain_test.go | 4 +- 34 files changed, 482 insertions(+), 346 deletions(-) create mode 100644 examples/plugin/main.go delete mode 100644 pkg/kcl_plugin/hello_plugin/api.go delete mode 100644 pkg/kcl_plugin/hello_plugin/api_test.go delete mode 100644 pkg/kcl_plugin/hello_plugin/hello.go delete mode 100644 pkg/kcl_plugin/plugin_nocgo.go delete mode 100644 pkg/kcl_plugin/plugin_py.go create mode 100644 pkg/native/api.go create mode 100644 pkg/native/api_test.go create mode 100644 pkg/native/cgo.go create mode 100644 pkg/native/client.go create mode 100644 pkg/native/client_test.go create mode 100644 pkg/native/loader.go rename pkg/{kcl_plugin => plugin}/api.go (98%) rename pkg/{kcl_plugin => plugin}/api_test.go (98%) rename pkg/{kcl_plugin => plugin}/error.go (68%) create mode 100644 pkg/plugin/hello_plugin/api.go create mode 100644 pkg/plugin/hello_plugin/api_test.go rename pkg/{kcl_plugin => plugin}/plugin.go (80%) rename pkg/{kcl_plugin/plugin_spec.go => plugin/spec.go} (99%) rename pkg/{kcl_plugin => plugin}/utils_c_string.go (96%) rename pkg/{kclvm_runtime => runtime}/builtin_service_client.go (97%) rename pkg/{kclvm_runtime => runtime}/init.go (98%) rename pkg/{kclvm_runtime => runtime}/kclvm.go (99%) rename pkg/{kclvm_runtime => runtime}/proc.go (90%) rename pkg/{kclvm_runtime => runtime}/runtime.go (99%) diff --git a/README.md b/README.md index e90573c8..89dbd71a 100644 --- a/README.md +++ b/README.md @@ -38,11 +38,11 @@ import ( ) func main() { - yaml := kcl.MustRun("kubernetes.k", kcl.WithCode(k_code)).GetRawYamlResult() + yaml := kcl.MustRun("kubernetes.k", kcl.WithCode(code)).GetRawYamlResult() fmt.Println(yaml) } -const k_code = ` +const code = ` apiVersion = "apps/v1" kind = "Deployment" metadata = { @@ -96,6 +96,33 @@ spec: - containerPort: 80 ``` +## Run KCL Code with Go Plugin + +```go +package main + +import ( + "fmt" + + "kcl-lang.io/kcl-go/pkg/kcl" + "kcl-lang.io/kcl-go/pkg/native" // Import the native API + _ "kcl-lang.io/kcl-go/pkg/plugin/hello_plugin" // Import the hello plugin +) + +func main() { + // Note we use `native.MustRun` here instead of `kcl.MustRun`, because it needs the cgo feature. + yaml := native.MustRun("main.k", kcl.WithCode(code)).GetRawYamlResult() + fmt.Println(yaml) +} + +const code = ` +import kcl_plugin.hello + +name = "kcl" +three = hello.add(1,2) # hello.add is written by Go +` +``` + ## Documents See the [KCL website](https://kcl-lang.io) diff --git a/examples/kubernetes/main.go b/examples/kubernetes/main.go index 329e24f6..874876b9 100644 --- a/examples/kubernetes/main.go +++ b/examples/kubernetes/main.go @@ -4,15 +4,14 @@ import ( "fmt" kcl "kcl-lang.io/kcl-go" - _ "kcl-lang.io/kcl-go/pkg/kcl_plugin/hello_plugin" ) func main() { - yaml := kcl.MustRun("kubernetes.k", kcl.WithCode(k_code)).GetRawYamlResult() + yaml := kcl.MustRun("kubernetes.k", kcl.WithCode(code)).GetRawYamlResult() fmt.Println(yaml) } -const k_code = ` +const code = ` apiVersion = "apps/v1" kind = "Deployment" metadata = { diff --git a/examples/plugin/main.go b/examples/plugin/main.go new file mode 100644 index 00000000..174035d4 --- /dev/null +++ b/examples/plugin/main.go @@ -0,0 +1,22 @@ +package main + +import ( + "fmt" + + "kcl-lang.io/kcl-go/pkg/kcl" + "kcl-lang.io/kcl-go/pkg/native" // Import the native API + _ "kcl-lang.io/kcl-go/pkg/plugin/hello_plugin" // Import the hello plugin +) + +func main() { + // Note we use `native.MustRun` here instead of `kcl.MustRun`, because it needs the cgo feature. + yaml := native.MustRun("main.k", kcl.WithCode(code)).GetRawYamlResult() + fmt.Println(yaml) +} + +const code = ` +import kcl_plugin.hello + +name = "kcl" +three = hello.add(1,2) +` diff --git a/go.mod b/go.mod index c209002a..ad4142d6 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21 require ( github.com/chai2010/jsonv v1.1.3 github.com/chai2010/protorpc v1.1.4 + github.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb github.com/getkin/kin-openapi v0.123.0 github.com/goccy/go-yaml v1.11.3 github.com/gofrs/flock v0.8.1 diff --git a/go.sum b/go.sum index 9f4841f9..195eb3c9 100644 --- a/go.sum +++ b/go.sum @@ -99,6 +99,8 @@ github.com/containerd/containerd v1.7.0 h1:G/ZQr3gMZs6ZT0qPUZ15znx5QSdQdASW11nXT github.com/containerd/containerd v1.7.0/go.mod h1:QfR7Efgb/6X2BDpTPJRvPTYDE9rsF0FsXX9J8sIs/sc= github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= +github.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb h1:GIzvVQ9UkUlOhSDlqmrQAAAUd6R3E+caIisNEyWXvNE= +github.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= diff --git a/kclvm.go b/kclvm.go index fece111c..96209191 100644 --- a/kclvm.go +++ b/kclvm.go @@ -34,7 +34,7 @@ import ( "io" "kcl-lang.io/kcl-go/pkg/kcl" - "kcl-lang.io/kcl-go/pkg/kclvm_runtime" + "kcl-lang.io/kcl-go/pkg/runtime" "kcl-lang.io/kcl-go/pkg/tools/format" "kcl-lang.io/kcl-go/pkg/tools/lint" "kcl-lang.io/kcl-go/pkg/tools/list" @@ -59,12 +59,12 @@ type ( // InitKclvmPath init kclvm path. func InitKclvmPath(kclvmRoot string) { - kclvm_runtime.InitKclvmRoot(kclvmRoot) + runtime.InitKclvmRoot(kclvmRoot) } // InitKclvmRuntime init kclvm process. func InitKclvmRuntime(n int) { - kclvm_runtime.InitRuntime(n) + runtime.InitRuntime(n) } // MustRun is like Run but panics if return any error. diff --git a/pkg/kcl/api.go b/pkg/kcl/api.go index e36a52d0..56aa7eeb 100644 --- a/pkg/kcl/api.go +++ b/pkg/kcl/api.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "reflect" "strings" @@ -324,19 +325,7 @@ func GetSchemaTypeMapping(file, code, schemaName string) (map[string]*gpyrpc.Kcl return resp.SchemaTypeMapping, nil } -func run(pathList []string, opts ...Option) (*KCLResultList, error) { - args, err := ParseArgs(pathList, opts...) - if err != nil { - return nil, err - } - - client := service.NewKclvmServiceClient() - resp, err := client.ExecProgram(args.ExecProgram_Args) - if err != nil { - return nil, err - } - // Output log message - logger := args.GetLogger() +func ExecResultToKCLResult(resp *gpyrpc.ExecProgram_Result, logger io.Writer) (*KCLResultList, error) { if logger != nil && resp.LogMessage != "" { _, err := logger.Write([]byte(resp.LogMessage)) if err != nil { @@ -374,3 +363,17 @@ func run(pathList []string, opts ...Option) (*KCLResultList, error) { result.raw_yaml_result = resp.YamlResult return &result, nil } + +func run(pathList []string, opts ...Option) (*KCLResultList, error) { + args, err := ParseArgs(pathList, opts...) + if err != nil { + return nil, err + } + + client := service.NewKclvmServiceClient() + resp, err := client.ExecProgram(args.ExecProgram_Args) + if err != nil { + return nil, err + } + return ExecResultToKCLResult(resp, args.GetLogger()) +} diff --git a/pkg/kcl_plugin/hello_plugin/api.go b/pkg/kcl_plugin/hello_plugin/api.go deleted file mode 100644 index 773f2400..00000000 --- a/pkg/kcl_plugin/hello_plugin/api.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2023 The KCL Authors. All rights reserved. - -package hello_plugin - -import ( - "fmt" - - "kcl-lang.io/kcl-go/pkg/kcl_plugin" -) - -func init() { - kcl_plugin.RegisterPlugin(kcl_plugin.Plugin{ - Name: "hello", - ResetFunc: _reset_plugin, - MethodMap: _MethodMap, - }) -} - -var _MethodMap = map[string]kcl_plugin.MethodSpec{ - // func set_global_int(v int64) - "set_global_int": { - Type: &kcl_plugin.MethodType{}, - Body: func(args *kcl_plugin.MethodArgs) (*kcl_plugin.MethodResult, error) { - set_global_int(args.IntArg(0)) - return nil, nil - }, - }, - - // func get_global_int() int64 - "get_global_int": { - Type: &kcl_plugin.MethodType{}, - Body: func(args *kcl_plugin.MethodArgs) (*kcl_plugin.MethodResult, error) { - v := get_global_int() - return &kcl_plugin.MethodResult{V: float64(v)}, nil - }, - }, - - // func say_hello(msg string) - "say_hello": { - Type: &kcl_plugin.MethodType{}, - Body: func(args *kcl_plugin.MethodArgs) (*kcl_plugin.MethodResult, error) { - msg := fmt.Sprint(args.StrArg(0)) - say_hello(msg) - - return nil, nil - }, - }, - - // func add(a, b int64) int64 - "add": { - Type: &kcl_plugin.MethodType{}, - Body: func(args *kcl_plugin.MethodArgs) (*kcl_plugin.MethodResult, error) { - v := add(args.IntArg(0), args.IntArg(1)) - return &kcl_plugin.MethodResult{V: float64(v)}, nil - }, - }, - - // func tolower(s string) string - "tolower": { - Type: &kcl_plugin.MethodType{}, - Body: func(args *kcl_plugin.MethodArgs) (*kcl_plugin.MethodResult, error) { - v := tolower(args.StrArg(0)) - return &kcl_plugin.MethodResult{V: v}, nil - }, - }, - - // func update_dict(d map[string]interface{}, key, value string) map[string]interface{} - "update_dict": { - Type: &kcl_plugin.MethodType{}, - Body: func(args *kcl_plugin.MethodArgs) (*kcl_plugin.MethodResult, error) { - d := args.Arg(0).(map[string]interface{}) - v := update_dict(d, args.StrArg(1), args.StrArg(2)) - return &kcl_plugin.MethodResult{V: v}, nil - }, - }, - - // func list_append(list []interface{}, values ...interface{}) []interface{} - "list_append": { - Type: &kcl_plugin.MethodType{}, - Body: func(args *kcl_plugin.MethodArgs) (*kcl_plugin.MethodResult, error) { - d := args.Arg(0).([]interface{}) - v := list_append(d, args.Args[1:]...) - return &kcl_plugin.MethodResult{V: v}, nil - }, - }, - - // func foo(a, b interface{}, x []interface{}, values map[string]interface{}) interface{} { - "foo": { - Type: &kcl_plugin.MethodType{}, - Body: func(args *kcl_plugin.MethodArgs) (*kcl_plugin.MethodResult, error) { - values := args.KwArgs - - a := args.Arg(0) - b := args.Arg(1) - x := args.Args[2:] - - v := foo(a, b, nil, x, values) - - return &kcl_plugin.MethodResult{V: v}, nil - }, - }, -} diff --git a/pkg/kcl_plugin/hello_plugin/api_test.go b/pkg/kcl_plugin/hello_plugin/api_test.go deleted file mode 100644 index 68aa40c0..00000000 --- a/pkg/kcl_plugin/hello_plugin/api_test.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2023 The KCL Authors. All rights reserved. - -package hello_plugin - -import ( - "testing" - - "kcl-lang.io/kcl-go/pkg/kcl_plugin" -) - -func TestPlugin_global_int(t *testing.T) { - if !kcl_plugin.CgoEnabled { - t.Skip("cgo disabled") - } - - kcl_plugin.Invoke("kcl_plugin.hello.set_global_int", []interface{}{123}, nil) - result_json := kcl_plugin.Invoke("kcl_plugin.hello.get_global_int", nil, nil) - if result_json != "123" { - t.Fatal(result_json) - } - - kcl_plugin.ResetPlugin() - - result_json = kcl_plugin.Invoke("kcl_plugin.hello.get_global_int", nil, nil) - if result_json != "0" { - t.Fatal(result_json) - } - - kcl_plugin.Invoke("kcl_plugin.hello.set_global_int", []interface{}{1024}, nil) - result_json = kcl_plugin.Invoke("kcl_plugin.hello.get_global_int", nil, nil) - if result_json != "1024" { - t.Fatal(result_json) - } -} - -func TestPlugin_add(t *testing.T) { - if !kcl_plugin.CgoEnabled { - t.Skip("cgo disabled") - } - - result_json := kcl_plugin.Invoke("kcl_plugin.hello.add", []interface{}{111, 22}, nil) - if result_json != "133" { - t.Fatal(result_json) - } -} - -func TestPlugin_tolower(t *testing.T) { - if !kcl_plugin.CgoEnabled { - t.Skip("cgo disabled") - } - result_json := kcl_plugin.Invoke("kcl_plugin.hello.tolower", []interface{}{"KCL"}, nil) - if result_json != `"kcl"` { - t.Fatal(result_json) - } -} - -func TestPlugin_update_dict(t *testing.T) { - if !kcl_plugin.CgoEnabled { - t.Skip("cgo disabled") - } - dict := map[string]interface{}{ - "name": 123, - } - - result_json := kcl_plugin.Invoke("kcl_plugin.hello.update_dict", []interface{}{dict, "name", "kcl-lang"}, nil) - if result_json != `{"name":"kcl-lang"}` { - t.Fatal(result_json) - } -} - -func TestPlugin_list_append(t *testing.T) { - if !kcl_plugin.CgoEnabled { - t.Skip("cgo disabled") - } - list := []interface{}{"abc"} - result_json := kcl_plugin.Invoke("kcl_plugin.hello.list_append", []interface{}{list, "name", 123}, nil) - if result_json != `["abc","name",123]` { - t.Fatal(result_json) - } -} diff --git a/pkg/kcl_plugin/hello_plugin/hello.go b/pkg/kcl_plugin/hello_plugin/hello.go deleted file mode 100644 index 9713a700..00000000 --- a/pkg/kcl_plugin/hello_plugin/hello.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2023 The KCL Authors. All rights reserved. - -package hello_plugin - -import ( - "fmt" - "strings" - "sync/atomic" - - "kcl-lang.io/kcl-go/pkg/kcl_plugin" -) - -var global_int int64 - -func _reset_plugin() { - atomic.SwapInt64(&global_int, 0) -} - -func set_global_int(v int64) { - if kcl_plugin.DebugMode { - fmt.Printf("plugin-go://hello_plugin.set_global_int(%d)\n", v) - } - atomic.SwapInt64(&global_int, v) -} - -func get_global_int() int64 { - if kcl_plugin.DebugMode { - fmt.Printf("plugin-go://hello_plugin.get_global_int(): %d\n", global_int) - } - return atomic.LoadInt64(&global_int) -} - -func say_hello(msg string) { - fmt.Println("hello.say_hello:", msg) -} - -func add(a, b int64) int64 { - if kcl_plugin.DebugMode { - fmt.Printf("plugin-go://hello_plugin.add(%d, %d)\n", a, b) - } - return a + b -} - -func tolower(s string) string { - return strings.ToLower(s) -} - -func update_dict(d map[string]interface{}, key, value string) map[string]interface{} { - d[key] = value - return d -} - -func list_append(list []interface{}, values ...interface{}) []interface{} { - return append(list, values...) -} - -func foo( - a, b interface{}, - __sep__ *struct{}, - x []interface{}, - values map[string]interface{}, -) interface{} { - m := make(map[string]interface{}) - - m["a"] = a - m["b"] = b - m["x"] = x - - for k, v := range values { - m[k] = v - } - - return m -} diff --git a/pkg/kcl_plugin/plugin_nocgo.go b/pkg/kcl_plugin/plugin_nocgo.go deleted file mode 100644 index 3548fc6b..00000000 --- a/pkg/kcl_plugin/plugin_nocgo.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2023 The KCL Authors. All rights reserved. - -//go:build !cgo -// +build !cgo - -package kcl_plugin - -const CgoEnabled = false - -func Invoke(method string, args []interface{}, kwargs map[string]interface{}) (result_json string) { - panic("unsupport") -} diff --git a/pkg/kcl_plugin/plugin_py.go b/pkg/kcl_plugin/plugin_py.go deleted file mode 100644 index 21f0301f..00000000 --- a/pkg/kcl_plugin/plugin_py.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright 2023 The KCL Authors. All rights reserved. - -package kcl_plugin - -import "fmt" - -func py_callPluginMethod(method, args_json, kwargs_json string) (result_json string) { - return JSONError(fmt.Errorf("invalid method py: %s, not found", method)) -} diff --git a/pkg/native/api.go b/pkg/native/api.go new file mode 100644 index 00000000..cbb9209d --- /dev/null +++ b/pkg/native/api.go @@ -0,0 +1,35 @@ +//go:build cgo +// +build cgo + +package native + +import ( + "kcl-lang.io/kcl-go/pkg/kcl" +) + +func MustRun(path string, opts ...kcl.Option) *kcl.KCLResultList { + v, err := Run(path, opts...) + if err != nil { + panic(err) + } + + return v +} + +func Run(path string, opts ...kcl.Option) (*kcl.KCLResultList, error) { + return run([]string{path}, opts...) +} + +func run(pathList []string, opts ...kcl.Option) (*kcl.KCLResultList, error) { + args, err := kcl.ParseArgs(pathList, opts...) + if err != nil { + return nil, err + } + + client := NewNativeServiceClient() + resp, err := client.ExecProgram(args.ExecProgram_Args) + if err != nil { + return nil, err + } + return kcl.ExecResultToKCLResult(resp, args.GetLogger()) +} diff --git a/pkg/native/api_test.go b/pkg/native/api_test.go new file mode 100644 index 00000000..28e074f0 --- /dev/null +++ b/pkg/native/api_test.go @@ -0,0 +1,20 @@ +//go:build cgo +// +build cgo + +package native + +import ( + "fmt" + "testing" + + "kcl-lang.io/kcl-go/pkg/kcl" + _ "kcl-lang.io/kcl-go/pkg/plugin/hello_plugin" +) + +func TestNativeRun(t *testing.T) { + yaml := MustRun("main.k", kcl.WithCode(code)).GetRawYamlResult() + fmt.Println(yaml) + + // name: kcl + // three: 3 +} diff --git a/pkg/native/cgo.go b/pkg/native/cgo.go new file mode 100644 index 00000000..1a6c3e96 --- /dev/null +++ b/pkg/native/cgo.go @@ -0,0 +1,88 @@ +//go:build cgo +// +build cgo + +package native + +/* +#include +#include +#include +typedef struct kclvm_service kclvm_service; +kclvm_service * kclvm_service_new(void *f,uint64_t plugin_agent){ + kclvm_service * (*new_service)(uint64_t); + new_service = (kclvm_service * (*)(uint64_t))f; + return new_service(plugin_agent); +} +void kclvm_service_delete(void *f,kclvm_service * c){ + void (*delete_service)(kclvm_service *); + delete_service = (void (*)(kclvm_service *))f; + return delete_service(c); +} +void kclvm_service_free_string(void *f,const char * res) { + void (*free_string)(const char *); + free_string = (void (*)(const char *))f; + return free_string(res); +} +const char* kclvm_service_call(void *f,kclvm_service* c,const char * method,const char * args){ + const char* (*service_call)(kclvm_service*,const char *,const char *); + service_call = (const char* (*)(kclvm_service*,const char *,const char *))f; + return service_call(c,method,args); +} +*/ +import "C" + +// NewKclvmService takes a pluginAgent and returns a pointer of capi kclvm_service. +// pluginAgent is the address of C function pointer with type :const char * (*)(const char *method,const char *args,const char *kwargs), +// in which "method" is the fully qualified name of plugin method, +// and "args" ,"kwargs" and return value of pluginAgent are JSON string +func NewKclvmService(pluginAgent C.uint64_t) *C.kclvm_service { + const fnName = "kclvm_service_new" + + newServ, err := lib.GetSymbolPointer(fnName) + + if err != nil { + panic(err) + } + + return C.kclvm_service_new(newServ, pluginAgent) +} + +// NewKclvmService releases the memory of kclvm_service +func DeleteKclvmService(serv *C.kclvm_service) { + const fnName = "kclvm_service_delete" + + deleteServ, err := lib.GetSymbolPointer(fnName) + + if err != nil { + panic(err) + } + + C.kclvm_service_delete(deleteServ, serv) +} + +// KclvmServiceFreeString releases the memory of the return value of KclvmServiceCall +func KclvmServiceFreeString(str *C.char) { + const fnName = "kclvm_service_free_string" + + freeString, err := lib.GetSymbolPointer(fnName) + + if err != nil { + panic(err) + } + + C.kclvm_service_free_string(freeString, str) +} + +// KclvmServiceCall calls kclvm service by c api +// args should be serialized as protobuf byte stream +func KclvmServiceCall(serv *C.kclvm_service, method *C.char, args *C.char) *C.char { + const fnName = "kclvm_service_call" + + serviceCall, err := lib.GetSymbolPointer(fnName) + + if err != nil { + panic(err) + } + + return C.kclvm_service_call(serviceCall, serv, method, args) +} diff --git a/pkg/native/client.go b/pkg/native/client.go new file mode 100644 index 00000000..a1becabf --- /dev/null +++ b/pkg/native/client.go @@ -0,0 +1,108 @@ +//go:build cgo +// +build cgo + +package native + +/* +#include +typedef struct kclvm_service kclvm_service; +*/ +import "C" +import ( + "errors" + "runtime" + "strings" + "sync" + "unsafe" + + "github.com/coreos/pkg/dlopen" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" + "kcl-lang.io/kcl-go/pkg/plugin" + "kcl-lang.io/kcl-go/pkg/spec/gpyrpc" +) + +var libInit sync.Once + +var lib *dlopen.LibHandle + +type NativeServiceClient struct { + client *C.kclvm_service +} + +func NewNativeServiceClient() *NativeServiceClient { + libInit.Do(func() { + lib = loadServiceNativeLib() + }) + c := new(NativeServiceClient) + c.client = NewKclvmService(C.uint64_t(plugin.GetInvokeJsonProxyPtr())) + runtime.SetFinalizer(c, func(x *NativeServiceClient) { + DeleteKclvmService(x.client) + x.client = nil + }) + return c +} + +func (c *NativeServiceClient) cApiCall(callName string, in proto.Message, out protoreflect.ProtoMessage) error { + type Validator interface { + Validate() error + } + + if callName == "" { + return errors.New("kclvm service c api call: empty method name") + } + + if in == nil { + return errors.New("kclvm service c api call: unknown proto input type") + } + + if out == nil { + return errors.New("kclvm service c api call: unknown proto output type") + } + + if x, ok := in.(Validator); ok { + if err := x.Validate(); err != nil { + return err + } + } + inBytes, err := proto.Marshal(in) + if err != nil { + return err + } + + cCallName := C.CString(callName) + + defer C.free(unsafe.Pointer(cCallName)) + + cIn := C.CString(string(inBytes)) + + defer C.free(unsafe.Pointer(cIn)) + + cOut := KclvmServiceCall(c.client, cCallName, cIn) + + defer KclvmServiceFreeString(cOut) + + msg := C.GoString(cOut) + + if strings.HasPrefix(msg, "ERROR:") { + return errors.New(strings.TrimPrefix(msg, "ERROR:")) + } + + err = proto.Unmarshal([]byte(msg), out) + if err != nil { + return errors.New(msg) + } + + return nil +} + +func (c *NativeServiceClient) ExecProgram(in *gpyrpc.ExecProgram_Args) (*gpyrpc.ExecProgram_Result, error) { + if in == nil { + in = new(gpyrpc.ExecProgram_Args) + } + + out := new(gpyrpc.ExecProgram_Result) + err := c.cApiCall("KclvmService.ExecProgram", in, out) + + return out, err +} diff --git a/pkg/native/client_test.go b/pkg/native/client_test.go new file mode 100644 index 00000000..18e422f6 --- /dev/null +++ b/pkg/native/client_test.go @@ -0,0 +1,32 @@ +//go:build cgo +// +build cgo + +package native + +import ( + "testing" + + _ "kcl-lang.io/kcl-go/pkg/plugin/hello_plugin" + "kcl-lang.io/kcl-go/pkg/spec/gpyrpc" +) + +const code = ` +import kcl_plugin.hello + +name = "kcl" +three = hello.add(1,2) +` + +func TestExecProgramWithPlugin(t *testing.T) { + client := NewNativeServiceClient() + result, err := client.ExecProgram(&gpyrpc.ExecProgram_Args{ + KFilenameList: []string{"main.k"}, + KCodeList: []string{code}, + }) + if err != nil { + t.Fatal(err) + } + if result.ErrMessage != "" { + t.Fatal("error message must be empty") + } +} diff --git a/pkg/native/loader.go b/pkg/native/loader.go new file mode 100644 index 00000000..34f5533c --- /dev/null +++ b/pkg/native/loader.go @@ -0,0 +1,40 @@ +package native + +import ( + "fmt" + "path/filepath" + "runtime" + + "github.com/coreos/pkg/dlopen" + kcl_runtime "kcl-lang.io/kcl-go/pkg/runtime" +) + +const libName = "kclvm_cli_cdylib" + +func loadServiceNativeLib() *dlopen.LibHandle { + root := kcl_runtime.MustGetKclvmRoot() + libPaths := []string{} + sysType := runtime.GOOS + fullLibName := "lib" + libName + ".so" + + if sysType == "darwin" { + fullLibName = "lib" + libName + ".dylib" + } else if sysType == "windows" { + fullLibName = libName + ".dll" + } + + libPath := filepath.Join(root, "bin", fullLibName) + libPaths = append(libPaths, libPath) + + h, err := dlopen.GetHandle(libPaths) + + if err != nil { + panic(fmt.Errorf(`couldn't get a handle to kcl native library: %v`, err)) + } + + runtime.SetFinalizer(h, func(x *dlopen.LibHandle) { + x.Close() + }) + + return h +} diff --git a/pkg/kcl_plugin/api.go b/pkg/plugin/api.go similarity index 98% rename from pkg/kcl_plugin/api.go rename to pkg/plugin/api.go index afd8e169..e401420d 100644 --- a/pkg/kcl_plugin/api.go +++ b/pkg/plugin/api.go @@ -1,6 +1,6 @@ // Copyright 2023 The KCL Authors. All rights reserved. -package kcl_plugin +package plugin import ( "os" diff --git a/pkg/kcl_plugin/api_test.go b/pkg/plugin/api_test.go similarity index 98% rename from pkg/kcl_plugin/api_test.go rename to pkg/plugin/api_test.go index 1e9506b5..5ae97171 100644 --- a/pkg/kcl_plugin/api_test.go +++ b/pkg/plugin/api_test.go @@ -1,6 +1,6 @@ // Copyright 2023 The KCL Authors. All rights reserved. -package kcl_plugin +package plugin import ( "strings" diff --git a/pkg/kcl_plugin/error.go b/pkg/plugin/error.go similarity index 68% rename from pkg/kcl_plugin/error.go rename to pkg/plugin/error.go index 64ed92c4..35eebab2 100644 --- a/pkg/kcl_plugin/error.go +++ b/pkg/plugin/error.go @@ -1,17 +1,18 @@ // Copyright 2023 The KCL Authors. All rights reserved. -package kcl_plugin +package plugin import ( "encoding/json" ) type PanicInfo struct { - X__kcl_PanicInfo__ bool `json:"__kcl_PanicInfo__"` + IsPanic bool `json:"__kcl_PanicInfo__"` - RustFile string `json:"rust_file,omitempty"` - RustLine int `json:"rust_line,omitempty"` - RustCol int `json:"rust_col,omitempty"` + BackTrace []BacktraceFrame `json:"backtrace,omitempty"` + RustFile string `json:"rust_file,omitempty"` + RustLine int `json:"rust_line,omitempty"` + RustCol int `json:"rust_col,omitempty"` KclPkgPath string `json:"kcl_pkgpath,omitempty"` KclFile string `json:"kcl_file,omitempty"` @@ -30,14 +31,21 @@ type PanicInfo struct { IsWarning string `json:"is_warning,omitempty"` } +type BacktraceFrame struct { + File string `json:"file,omitempty"` + Func string `json:"func,omitempty"` + Line int `json:"line,omitempty"` + Col int `json:"col,omitempty"` +} + func JSONError(err error) string { if x, ok := err.(*PanicInfo); ok { return x.JSONError() } if err != nil { x := &PanicInfo{ - X__kcl_PanicInfo__: true, - Message: err.Error(), + IsPanic: true, + Message: err.Error(), } return x.JSONError() } @@ -45,7 +53,7 @@ func JSONError(err error) string { } func (p *PanicInfo) JSONError() string { - p.X__kcl_PanicInfo__ = true + p.IsPanic = true d, _ := json.Marshal(p) return string(d) } diff --git a/pkg/plugin/hello_plugin/api.go b/pkg/plugin/hello_plugin/api.go new file mode 100644 index 00000000..293703f4 --- /dev/null +++ b/pkg/plugin/hello_plugin/api.go @@ -0,0 +1,21 @@ +// Copyright 2023 The KCL Authors. All rights reserved. + +package hello_plugin + +import ( + "kcl-lang.io/kcl-go/pkg/plugin" +) + +func init() { + plugin.RegisterPlugin(plugin.Plugin{ + Name: "hello", + MethodMap: map[string]plugin.MethodSpec{ + "add": { + Body: func(args *plugin.MethodArgs) (*plugin.MethodResult, error) { + v := args.IntArg(0) + args.IntArg(1) + return &plugin.MethodResult{V: v}, nil + }, + }, + }, + }) +} diff --git a/pkg/plugin/hello_plugin/api_test.go b/pkg/plugin/hello_plugin/api_test.go new file mode 100644 index 00000000..7ac470c1 --- /dev/null +++ b/pkg/plugin/hello_plugin/api_test.go @@ -0,0 +1,16 @@ +// Copyright 2023 The KCL Authors. All rights reserved. + +package hello_plugin + +import ( + "testing" + + "kcl-lang.io/kcl-go/pkg/plugin" +) + +func TestPlugin_add(t *testing.T) { + result_json := plugin.Invoke("kcl_plugin.hello.add", []interface{}{111, 22}, nil) + if result_json != "133" { + t.Fatal(result_json) + } +} diff --git a/pkg/kcl_plugin/plugin.go b/pkg/plugin/plugin.go similarity index 80% rename from pkg/kcl_plugin/plugin.go rename to pkg/plugin/plugin.go index 47011c66..ee87b528 100644 --- a/pkg/kcl_plugin/plugin.go +++ b/pkg/plugin/plugin.go @@ -3,20 +3,20 @@ //go:build cgo // +build cgo -package kcl_plugin +package plugin /* #include #include -extern char* kclvm_go_capi_InvokeJsonProxy( +extern char* kcl_go_capi_InvokeJsonProxy( char* method, char* args_json, char* kwargs_json ); -static uint64_t kclvm_go_capi_getInvokeJsonProxyPtr() { - return (uint64_t)(kclvm_go_capi_InvokeJsonProxy); +static uint64_t kcl_go_capi_getInvokeJsonProxyPtr() { + return (uint64_t)(kcl_go_capi_InvokeJsonProxy); } */ import "C" @@ -28,8 +28,8 @@ import ( const CgoEnabled = true -//export kclvm_go_capi_InvokeJsonProxy -func kclvm_go_capi_InvokeJsonProxy(_method, _args_json, _kwargs_json *C.char) (result_json *C.char) { +//export kcl_go_capi_InvokeJsonProxy +func kcl_go_capi_InvokeJsonProxy(_method, _args_json, _kwargs_json *C.char) (result_json *C.char) { var method, args_json, kwargs_json string if _method != nil { @@ -46,8 +46,8 @@ func kclvm_go_capi_InvokeJsonProxy(_method, _args_json, _kwargs_json *C.char) (r return c_String_new(result) } -func GetInvokeJsonProxyPtr() uintptr { - ptr := uintptr(C.kclvm_go_capi_getInvokeJsonProxyPtr()) +func GetInvokeJsonProxyPtr() uint64 { + ptr := uint64(C.kcl_go_capi_getInvokeJsonProxyPtr()) return ptr } @@ -101,8 +101,7 @@ func _Invoke(method, args_json, kwargs_json string) (result_json string) { // get method methodSpec, found := GetMethodSpec(method) if !found { - // try python plugin - return py_callPluginMethod(method, args_json, kwargs_json) + return JSONError(fmt.Errorf("invalid method: %s, not found", method)) } // call plugin method diff --git a/pkg/kcl_plugin/plugin_spec.go b/pkg/plugin/spec.go similarity index 99% rename from pkg/kcl_plugin/plugin_spec.go rename to pkg/plugin/spec.go index c45c97f9..2af3dc92 100644 --- a/pkg/kcl_plugin/plugin_spec.go +++ b/pkg/plugin/spec.go @@ -1,6 +1,6 @@ // Copyright 2023 The KCL Authors. All rights reserved. -package kcl_plugin +package plugin import ( "encoding/json" diff --git a/pkg/kcl_plugin/utils_c_string.go b/pkg/plugin/utils_c_string.go similarity index 96% rename from pkg/kcl_plugin/utils_c_string.go rename to pkg/plugin/utils_c_string.go index 1748099d..c07975bd 100644 --- a/pkg/kcl_plugin/utils_c_string.go +++ b/pkg/plugin/utils_c_string.go @@ -1,6 +1,6 @@ // Copyright 2023 The KCL Authors. All rights reserved. -package kcl_plugin +package plugin // #include import "C" diff --git a/pkg/kclvm_runtime/builtin_service_client.go b/pkg/runtime/builtin_service_client.go similarity index 97% rename from pkg/kclvm_runtime/builtin_service_client.go rename to pkg/runtime/builtin_service_client.go index eabdec50..c8c088ba 100644 --- a/pkg/kclvm_runtime/builtin_service_client.go +++ b/pkg/runtime/builtin_service_client.go @@ -1,6 +1,6 @@ // Copyright The KCL Authors. All rights reserved. -package kclvm_runtime +package runtime import ( "fmt" diff --git a/pkg/kclvm_runtime/init.go b/pkg/runtime/init.go similarity index 98% rename from pkg/kclvm_runtime/init.go rename to pkg/runtime/init.go index b9fb83fe..0d9d6648 100644 --- a/pkg/kclvm_runtime/init.go +++ b/pkg/runtime/init.go @@ -1,6 +1,6 @@ // Copyright The KCL Authors. All rights reserved. -package kclvm_runtime +package runtime import ( "fmt" diff --git a/pkg/kclvm_runtime/kclvm.go b/pkg/runtime/kclvm.go similarity index 99% rename from pkg/kclvm_runtime/kclvm.go rename to pkg/runtime/kclvm.go index ebf700f0..6b6d0179 100644 --- a/pkg/kclvm_runtime/kclvm.go +++ b/pkg/runtime/kclvm.go @@ -1,6 +1,6 @@ // Copyright The KCL Authors. All rights reserved. -package kclvm_runtime +package runtime import ( "context" diff --git a/pkg/kclvm_runtime/proc.go b/pkg/runtime/proc.go similarity index 90% rename from pkg/kclvm_runtime/proc.go rename to pkg/runtime/proc.go index f96cbe20..4cdde43a 100644 --- a/pkg/kclvm_runtime/proc.go +++ b/pkg/runtime/proc.go @@ -1,6 +1,6 @@ // Copyright The KCL Authors. All rights reserved. -package kclvm_runtime +package runtime import ( "bytes" @@ -24,12 +24,9 @@ type _Process struct { done chan error } -// 创建新的进程, 可能失败 func createProcess(exe string, arg ...string) (p *_Process, err error) { p = new(_Process) - p.cmd = exec.Command(exe, arg...) - p.stdin, err = p.cmd.StdinPipe() if err != nil { return nil, err @@ -38,25 +35,20 @@ func createProcess(exe string, arg ...string) (p *_Process, err error) { if err != nil { return nil, err } - p.stderr = newLimitBuffer(10 * 1024) p.cmd.Stderr = p.stderr - - // 启动进程 + // Start the process if err := p.cmd.Start(); err != nil { return nil, err } - - // 等待退出结果(2个缓存, 对应 Wait 和 Kill 返回值) + // Wait for the exit result (2 buffers, corresponding to Wait and Kill return values) p.done = make(chan error, 2) go func() { p.done <- p.cmd.Wait() }() - - // NewXxxServiceClient 会独占 信道(只能选择1个), 多个客户端需要手工构建 client + // NewXxxServiceClient will occupy the channel (only one can be selected), multiple clients need to manually build the client conn := &procReadWriteCloser{proc: p, r: p.stdout, w: p.stdin} p.c = rpc.NewClientWithCodec(jsonrpc2.NewClientCodec(conn)) - return p, nil } diff --git a/pkg/kclvm_runtime/runtime.go b/pkg/runtime/runtime.go similarity index 99% rename from pkg/kclvm_runtime/runtime.go rename to pkg/runtime/runtime.go index d0b40a7d..1ba2bc2a 100644 --- a/pkg/kclvm_runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -1,6 +1,6 @@ // Copyright The KCL Authors. All rights reserved. -package kclvm_runtime +package runtime import ( "fmt" diff --git a/pkg/service/client_builtin_service.go b/pkg/service/client_builtin_service.go index 77652a94..fe3742d7 100644 --- a/pkg/service/client_builtin_service.go +++ b/pkg/service/client_builtin_service.go @@ -3,13 +3,13 @@ package service import ( - "kcl-lang.io/kcl-go/pkg/kclvm_runtime" + "kcl-lang.io/kcl-go/pkg/runtime" ) -type BuiltinServiceClient = kclvm_runtime.BuiltinServiceClient +type BuiltinServiceClient = runtime.BuiltinServiceClient func NewBuiltinServiceClient() *BuiltinServiceClient { return &BuiltinServiceClient{ - Runtime: kclvm_runtime.GetRuntime(), + Runtime: runtime.GetRuntime(), } } diff --git a/pkg/service/client_kclvm_service.go b/pkg/service/client_kclvm_service.go index f2d7f589..adb9bb23 100644 --- a/pkg/service/client_kclvm_service.go +++ b/pkg/service/client_kclvm_service.go @@ -7,19 +7,19 @@ import ( "io" "net/rpc" - "kcl-lang.io/kcl-go/pkg/kclvm_runtime" + "kcl-lang.io/kcl-go/pkg/runtime" "kcl-lang.io/kcl-go/pkg/spec/gpyrpc" ) type KclvmServiceClient struct { - Runtime *kclvm_runtime.Runtime - pyRuntime *kclvm_runtime.Runtime + Runtime *runtime.Runtime + pyRuntime *runtime.Runtime } func NewKclvmServiceClient() *KclvmServiceClient { c := &KclvmServiceClient{ - Runtime: kclvm_runtime.GetRuntime(), - pyRuntime: kclvm_runtime.GetPyRuntime(), + Runtime: runtime.GetRuntime(), + pyRuntime: runtime.GetPyRuntime(), } return c } diff --git a/pkg/service/testmain_test.go b/pkg/service/testmain_test.go index 02255b0f..9b8c61aa 100644 --- a/pkg/service/testmain_test.go +++ b/pkg/service/testmain_test.go @@ -11,7 +11,7 @@ import ( "testing" "time" - "kcl-lang.io/kcl-go/pkg/kclvm_runtime" + "kcl-lang.io/kcl-go/pkg/runtime" ) const tEnvNumCpu = "KCLVM_GO_API_TEST_NUM_CPU" @@ -21,7 +21,7 @@ func TestMain(m *testing.M) { if s := os.Getenv(tEnvNumCpu); s != "" { if x, err := strconv.Atoi(s); err == nil { fmt.Println("TestMain: nWorker =", x) - kclvm_runtime.InitRuntime(x) + runtime.InitRuntime(x) } }