From ee7a0f1139b5a27b31a947d7fdf688a0321112d3 Mon Sep 17 00:00:00 2001 From: Alexander Bezzubov Date: Sat, 1 Jun 2019 19:47:13 +0200 Subject: [PATCH 1/4] python: initial impl of bindings using cFFI A PoC that exposes single function `enry.language_by_extension()` and a small number of helpers to deal with string coversion between Go<->C<->Python. Signed-off-by: Alexander Bezzubov --- Makefile | 7 +++++++ java/Makefile | 2 ++ python/README.md | 29 +++++++++++++++++++++++++++++ python/enry.py | 40 ++++++++++++++++++++++++++++++++++++++++ python/enry_build.py | 33 +++++++++++++++++++++++++++++++++ 5 files changed, 111 insertions(+) create mode 100644 python/README.md create mode 100644 python/enry.py create mode 100644 python/enry_build.py diff --git a/Makefile b/Makefile index fb978eb8..34c47b9f 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,7 @@ LINUX_DIR=$(RESOURCES_DIR)/linux-x86-64 LINUX_SHARED_LIB=$(LINUX_DIR)/libenry.so DARWIN_DIR=$(RESOURCES_DIR)/darwin DARWIN_SHARED_LIB=$(DARWIN_DIR)/libenry.dylib +STATIC_LIB=$(RESOURCES_DIR)/libenry.a HEADER_FILE=libenry.h NATIVE_LIB=./shared/enry.go @@ -79,4 +80,10 @@ $(LINUX_SHARED_LIB): CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -buildmode=c-shared -o $(LINUX_SHARED_LIB) $(NATIVE_LIB) && \ mv $(LINUX_DIR)/$(HEADER_FILE) $(RESOURCES_DIR)/$(HEADER_FILE) + +static: $(STATIC_LIB) + +$(STATIC_LIB): + CGO_ENABLED=1 go build -buildmode=c-archive -o $(STATIC_LIB) $(NATIVE_LIB) + .PHONY: benchmarks benchmarks-samples benchmarks-slow diff --git a/java/Makefile b/java/Makefile index 0fadca6d..08ed5eb1 100644 --- a/java/Makefile +++ b/java/Makefile @@ -59,4 +59,6 @@ package: clean: rm -rf $(JAR) rm -rf $(RESOURCES_DIR) + rm -rf $(JNAERATOR_JAR) +.PHONY: test package clean linux-shared darwin-shared os-shared-lib diff --git a/python/README.md b/python/README.md new file mode 100644 index 00000000..eab4f15d --- /dev/null +++ b/python/README.md @@ -0,0 +1,29 @@ +# Python bindings for enry + +Python bingings thoug cFFI (API, out-of-line) for calling enr Go functions though CGo wrapper. + +## Build + +``` +$ make static +$ python enry_build.py +``` + +Will build static library for Cgo wrapper `libenry`, then generate and build `enry.c` +- a CPython extension that + +## Run + +Example for single exposed API function is provided. + +``` +$ python enry.py +``` + +## TODOs + - [ ] try ABI mode, to aviod dependency on C compiler on install (+perf test?) + - [ ] ready `libenry.h` and generate `ffibuilder.cdef` content + - [ ] cover the rest of enry API + - [ ] add `setup.py` + - [ ] build/release automation on CI (publish on pypi) + - \ No newline at end of file diff --git a/python/enry.py b/python/enry.py new file mode 100644 index 00000000..0905e9e6 --- /dev/null +++ b/python/enry.py @@ -0,0 +1,40 @@ +""" +Python library calling enry Go implementation trough cFFI (API, out-of-line) and Cgo. +""" + +from _c_enry import ffi, lib + + +def go_str_to_py(go_str): + str_len = go_str.n + if str_len > 0: + return ffi.unpack(go_str.p, go_str.n).decode() + return "" + + +def py_str_to_go(py_str): + str_bytes = py_str.encode() + c_str = ffi.new("char[]", str_bytes) + go_str = ffi.new("_GoString_ *", [c_str, len(str_bytes)]) + return go_str[0] + + +def language_by_extension(filename: str) -> str: + fName = py_str_to_go(filename) + guess = lib.GetLanguageByExtension(fName) + lang = go_str_to_py(guess.r0) + return lang + + +## Test + + +def main(): + files = ["Parse.hs", "some.cpp", "and.go", "type.h"] + for filename in files: + lang = language_by_extension(filename) + print("file: {:10s} language: '{}'".format(filename, lang)) + + +if __name__ == "__main__": + main() diff --git a/python/enry_build.py b/python/enry_build.py new file mode 100644 index 00000000..637f8fec --- /dev/null +++ b/python/enry_build.py @@ -0,0 +1,33 @@ +from cffi import FFI +ffibuilder = FFI() + +# cdef() expects a single string declaring the C types, functions and +# globals needed to use the shared object. It must be in valid C syntax. +ffibuilder.cdef(""" +typedef struct { const char *p; ptrdiff_t n; } _GoString_; +typedef _GoString_ GoString; +typedef unsigned char GoUint8; + +/* Return type for GetLanguageByExtension */ +struct GetLanguageByExtension_return { + GoString r0; /* language */ + GoUint8 r1; /* safe */ +}; + +extern struct GetLanguageByExtension_return GetLanguageByExtension(GoString p0); +""") + +# set_source() gives the name of the python extension module to +# produce, and some C source code as a string. This C code needs +# to make the declarated functions, types and globals available, +# so it is often just the "#include". +ffibuilder.set_source("_c_enry", + """ + #include "../.shared/libenry.h" // the C header of the library +""", + libraries=['enry'], + library_dirs=['../.shared' + ]) # library name, for the linker + +if __name__ == "__main__": + ffibuilder.compile(verbose=True) From cff9c07009bf4036bf940a046f521a6024602775 Mon Sep 17 00:00:00 2001 From: Alexander Bezzubov Date: Sat, 1 Jun 2019 21:19:55 +0200 Subject: [PATCH 2/4] python: expose language_by_filename() Signed-off-by: Alexander Bezzubov --- python/enry.py | 16 +++++++++++++++- python/enry_build.py | 8 ++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/python/enry.py b/python/enry.py index 0905e9e6..77830f8f 100644 --- a/python/enry.py +++ b/python/enry.py @@ -26,15 +26,29 @@ def language_by_extension(filename: str) -> str: return lang +def language_by_filename(filename: str) -> str: + fName = py_str_to_go(filename) + guess = lib.GetLanguageByFilename(fName) + lang = go_str_to_py(guess.r0) + return lang + + ## Test def main(): - files = ["Parse.hs", "some.cpp", "and.go", "type.h"] + files = ["Parse.hs", "some.cpp", "and.go", "type.h", ".bashrc"] + + print("strategy: extension") for filename in files: lang = language_by_extension(filename) print("file: {:10s} language: '{}'".format(filename, lang)) + print("\nstrategy: filename") + for filename in files: + lang = language_by_filename(filename) + print("file: {:10s} language: '{}'".format(filename, lang)) + if __name__ == "__main__": main() diff --git a/python/enry_build.py b/python/enry_build.py index 637f8fec..8e5d7763 100644 --- a/python/enry_build.py +++ b/python/enry_build.py @@ -15,6 +15,14 @@ }; extern struct GetLanguageByExtension_return GetLanguageByExtension(GoString p0); + +/* Return type for GetLanguageByFilename */ +struct GetLanguageByFilename_return { + GoString r0; /* language */ + GoUint8 r1; /* safe */ +}; + +extern struct GetLanguageByFilename_return GetLanguageByFilename(GoString p0); """) # set_source() gives the name of the python extension module to From be583cad06e13ed957fab654e6fdcf92ce9a10fc Mon Sep 17 00:00:00 2001 From: Alexander Bezzubov Date: Sat, 1 Jun 2019 21:24:04 +0200 Subject: [PATCH 3/4] python: add dependencies Signed-off-by: Alexander Bezzubov --- python/README.md | 4 ++-- python/requirements.txt | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 python/requirements.txt diff --git a/python/README.md b/python/README.md index eab4f15d..fe4ec132 100644 --- a/python/README.md +++ b/python/README.md @@ -23,7 +23,7 @@ $ python enry.py ## TODOs - [ ] try ABI mode, to aviod dependency on C compiler on install (+perf test?) - [ ] ready `libenry.h` and generate `ffibuilder.cdef` content + - [ ] helpers for sending/recieving Go slices to C - [ ] cover the rest of enry API - [ ] add `setup.py` - - [ ] build/release automation on CI (publish on pypi) - - \ No newline at end of file + - [ ] build/release automation on CI (publish on pypi) \ No newline at end of file diff --git a/python/requirements.txt b/python/requirements.txt new file mode 100644 index 00000000..28dbbb14 --- /dev/null +++ b/python/requirements.txt @@ -0,0 +1,4 @@ +cffi==1.12.3 +Click==7.0 +pycparser==2.19 +yapf==0.27.0 From 6cf5bf2ca497fbe037a27d99662d5f946d1e4e06 Mon Sep 17 00:00:00 2001 From: Alexander Bezzubov Date: Sat, 1 Jun 2019 21:30:07 +0200 Subject: [PATCH 4/4] python: expose is_vendor() Signed-off-by: Alexander Bezzubov --- python/enry.py | 26 ++++++++++++++++++++++++-- python/enry_build.py | 2 ++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/python/enry.py b/python/enry.py index 77830f8f..cd388ba2 100644 --- a/python/enry.py +++ b/python/enry.py @@ -4,6 +4,8 @@ from _c_enry import ffi, lib +## Helpers + def go_str_to_py(go_str): str_len = go_str.n @@ -19,6 +21,13 @@ def py_str_to_go(py_str): return go_str[0] +def go_bool_to_py(go_bool): + return go_bool == 1 + + +## API + + def language_by_extension(filename: str) -> str: fName = py_str_to_go(filename) guess = lib.GetLanguageByExtension(fName) @@ -33,11 +42,19 @@ def language_by_filename(filename: str) -> str: return lang -## Test +def is_vendor(filename: str) -> bool: + fName = py_str_to_go(filename) + guess = lib.IsVendor(fName) + return go_bool_to_py(guess) + + +## Tests def main(): - files = ["Parse.hs", "some.cpp", "and.go", "type.h", ".bashrc"] + files = [ + "Parse.hs", "some.cpp", "and.go", "type.h", ".bashrc", ".gitignore" + ] print("strategy: extension") for filename in files: @@ -49,6 +66,11 @@ def main(): lang = language_by_filename(filename) print("file: {:10s} language: '{}'".format(filename, lang)) + print("\ncheck: is vendor?") + for filename in files: + vendor = is_vendor(filename) + print("file: {:10s} vendor: '{}'".format(filename, vendor)) + if __name__ == "__main__": main() diff --git a/python/enry_build.py b/python/enry_build.py index 8e5d7763..96ca8f64 100644 --- a/python/enry_build.py +++ b/python/enry_build.py @@ -23,6 +23,8 @@ }; extern struct GetLanguageByFilename_return GetLanguageByFilename(GoString p0); + +extern GoUint8 IsVendor(GoString p0); """) # set_source() gives the name of the python extension module to