feat(lexer): add no_lstrip option

Fix compile error on WIndows for missing head file `unistd.h`.
Add lexer config interface.
Add `no_lstrip` option to keep indentation.

.github/workflows/python-publish.yml

@@ -9,8 +9,7 @@
 name: Upload Python Package
 
 on:
-  release:
-    types: [created]
+  create:
   workflow_dispatch:
 
 permissions:

Makefile

@@ -17,16 +17,16 @@ build_cython:
 build:
 	python setup.py build_ext --inplace
 
-install: build
+install:
 	python setup.py install
 
 run:
 	python -m kola
 
-develop: build
+develop:
 	python setup.py develop
 
-build_dist: test
+build_dist:
 	python setup.py sdist bdist_wheel
 
 lint:

kola/__main__.py

@@ -26,20 +26,23 @@
 
 
 def _read_stdin() -> str:
-    sys.stdout.write("$kola: ")
-    sys.stdout.flush()
-    s = sys.stdin.readline()
-    while s.endswith("\\\n"):
-        sys.stdout.write("$...: ")
+    try:
+        sys.stdout.write("$kola: ")
         sys.stdout.flush()
-        s += sys.stdin.readline()
+        s = sys.stdin.readline()
+        while s.endswith("\\\n"):
+            sys.stdout.write("$...: ")
+            sys.stdout.flush()
+            s += sys.stdin.readline()
+    except KeyboardInterrupt:
+        sys.exit()
     return s
 
 
 if __name__ == "__main__":
     parser = ArgumentParser("kola")
     parser.add_argument("file", default=None, nargs="?")
-    parser.add_argument("-i", "--inline", help="parse inline string")
+    parser.add_argument("-i", "-c", "--inline", help="parse inline string")
     parser.add_argument("-s", "--script", help="parser script")
     parser.add_argument("-d", "--debug", help="dubugger type", choices=["token", "command"])
     parser.add_argument("--encoding", help="file encoding", default="utf-8")

kola/_cutil.h

@@ -20,9 +20,14 @@ extern "C" {
 typedef void* yyscan_t;
 #endif
 
+#define LFLAG_DISABLED      (1 << 0)
+#define LFLAG_ISANNOTATION  (1 << 1)
+#define LFLAG_NOLSTRIP        (1 << 2)
+
 typedef struct lexer_extra {
     const char* filename;
     uint8_t command_threshold;
+    uint8_t flag;
 } LexerData;
 
 #define YY_EXTRA_TYPE LexerData*
@@ -33,13 +38,13 @@ enum TokenSyn {
 };
 
 static const uint8_t yy_goto[7][8] = {
-    {15,  63,  0,   0,   0,  0,   0, 0},     // CMD | CMD_N | TEXT
-    {34,  162, 35,  117, 0,  151, 0, 40},    // LITERAL   
-    {17,  49,  35,  117, 0,  151, 0, 40},    // NUM | STRING
-    {0,   0,   134, 0,   0,  0,   0, 6},     // CLN
-    {0,   0,   100, 0,   4,  0,   8, 0},     // CMA
-    {0,   3,   0,   0,   0,  0,   0, 0},     // SLP
-    {0,   0,   65,  0,   81, 0,   81, 0}     // SRP
+    {15,    63,     0,      0,      0,      0,      0,  0},     // CMD | CMD_N | TEXT
+    {34,    162,    35,     117,    0,      151,    0,  40},    // LITERAL   
+    {17,    49,     35,     117,    0,      151,    0,  40},    // NUM | STRING
+    {0,     0,      134,    0,      0,      0,      0,  6},     // CLN
+    {0,     0,      100,    0,      4,      0,      8,  0},     // CMA
+    {0,     3,      0,      0,      0,      0,      0,  0},     // SLP
+    {0,     0,      65,     0,      81,     0,      81, 0}     // SRP
 };
 
 #ifdef Py_PYTHON_H

kola/_cutil.pxd

@@ -72,10 +72,15 @@ cdef extern from "_cutil.h":
         SRP
         ANNOTATION
     const uint8_t yy_goto[7][8]
+
+    const int LFLAG_DISABLED
+    const int LFLAG_ISANNOTATION
+    const int LFLAG_NOLSTRIP
 
     ctypedef struct LexerData:
         const char* filename
         uint8_t command_threshold
+        uint8_t flag
     ctypedef void* yyscan_t
 
     void kola_set_error(object exc_type, int errorno, const char* filename, int lineno, const char* text) except *

kola/klvm/commandset.py

@@ -75,7 +75,7 @@ def get(self, __key: str, default: Optional[Callable] = None) -> Optional[Callab
         """
         get command in the command set
 
-        NOTICE: This method will only try to get its own commands.
+        NOTE: This method will only try to get its own commands.
         If you want to get a command as a normal case, use `__getitem__` instead.
         """
         cache = self._bound_command_cache.get(__key, None)
@@ -91,6 +91,17 @@ def get(self, __key: str, default: Optional[Callable] = None) -> Optional[Callab
         return bound_cmd
 
     def __kola_caller__(self, command: Command, args: tuple, kwargs: Dict[str, Any], **kwds: Any) -> Any:
+        """hook function used to change the calling behavior of the `Command` class
+
+        :param command: the `Command` object being invoked
+        :type command: Command
+        :param args: call positional arguments
+        :type args: Tuple[Any]
+        :param kwargs: call keyword arguments
+        :type kwargs: Dict[str, Any]
+        :return: command return value
+        :rtype: Any
+        """
         return command.__func__(self, *args, **kwargs)
 
     def __getitem__(self, __key: str) -> Callable:

kola/klvm/writer.py

@@ -28,6 +28,12 @@ def inner(writer: BaseWriter, *args, **kwds) -> None:
 
 
 class KoiLangWriter(KoiLang):
+    """
+    writer class for the KoiLang class
+
+    The class should not be used directly.
+    The usual way to use it is to call YourKoiLang.writer().
+    """
     def __init__(self, ___writer: Union[str, bytes, os.PathLike, BaseWriter, None] = None) -> None:
         super().__init__()
         if ___writer is None:

kola/kolalexer.l

@@ -7,35 +7,48 @@
     #include <string.h>
     #include "_cutil.h"
 
-    #define yy_command_threshold yyextra->command_threshold
+    #define yy_command_threshold    yyextra->command_threshold
+    #define yy_lflag                yyextra->flag
+    #define yy_is_annotation        (yy_lflag & LFLAG_ISANNOTATION)
+    #define yy_lstrip               (yy_lflag & LFLAG_NOLSTRIP)
+    #define yyset_annotation()      do {yy_lflag |= LFLAG_ISANNOTATION;} while (0)
+    #define yyunset_annotation()    do {yy_lflag &= ~LFLAG_ISANNOTATION;} while (0)
 
     #define ECHO yyterminate()
+
+    #define YY_NO_UNISTD_H
+
+    #ifdef _MSC_VER
+        #include <io.h>
+        #include <process.h>
+    #else
+        #include <unistd.h>
+    #endif
 %}
 
 delim   [ \t]
 ws      {delim}+
-letter  [A-Za-z]
-letter_ ({letter}|_)
+letter  [A-Za-z_]
 digit   [0-9]
 uint    ([1-9]{digit}*|0)
 sigint  (-?{uint})
-hex     0x[0-9A-Za-z]+
+hex     0x[0-9A-Fa-f]+
 bin     0b[01]+
 float_e (e|E)-?{digit}+
 float   ((({sigint}\.{digit}*)|(-?\.{digit}+)){float_e}?)|({sigint}{float_e})
-literal {letter_}({letter_}|{digit})*
+literal {letter}({letter}|{digit})*
 string  \"([^\"]|(\\\r?\n)|(\\\"))*\"
 plain   ([^\r\n]|(\\\r?\n))+
-text    [^#\n\r \t]{plain}*
 
 %s COMMAND
-%s COMMENT
 %s PLAIN_TEXT
 %s ARGUMENT
 
 %%
 
-{ws}                            {}
+{ws}                            {
+    if (YY_START == INITIAL && yy_lstrip) yymore();
+}
 \\\r?\n                         {}
 \r?\n                           {
     if (YY_START == COMMAND) {
@@ -44,21 +57,24 @@ text    [^#\n\r \t]{plain}*
     BEGIN INITIAL;
 }
 <INITIAL>#+             {
-    if (yyleng == yy_command_threshold) {
+    size_t prefix_len = yyleng - (size_t)(strchr(yytext, '#') - yytext);
+
+    if (prefix_len == yy_command_threshold) {
         BEGIN COMMAND;
-    } else if (yyleng > yy_command_threshold) {
-        BEGIN COMMENT;
-        yymore();
-    } else {
+    }  else {
+        if (prefix_len > yy_command_threshold) {
+            yyset_annotation();
+        } else {
+            yyunset_annotation();
+        }
         BEGIN PLAIN_TEXT;
-        yymore();
+        yyless(0);
     }
 }
+<INITIAL>[^#\n\r \t]    {BEGIN PLAIN_TEXT; yyless(0);}
 <COMMAND>{literal}      {BEGIN ARGUMENT; return(CMD);}
 <COMMAND>{uint}         {BEGIN ARGUMENT; return(CMD_N);}
-<COMMENT>{plain}        {return(ANNOTATION);}
-<PLAIN_TEXT>{plain}     {return(TEXT);}
-<INITIAL>{text}         {return(TEXT);}
+<PLAIN_TEXT>{plain}     {return yy_is_annotation? ANNOTATION : TEXT;}
 
 <ARGUMENT>\(            {return(SLP);}
 <ARGUMENT>\)            {return(SRP);}
@@ -84,5 +100,6 @@ text    [^#\n\r \t]{plain}*
 
 int yylex_check(yyscan_t yyscanner) {
     struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+    if (yy_lflag & LFLAG_DISABLED) return 0;
     return YY_CURRENT_BUFFER != NULL;
-}
+}