fix hudson-ai/guidance#20

(empty lexemes)
microsoft · Jul 3, 2024 · aaeed3f · aaeed3f
1 parent cd7490a
commit aaeed3f
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 3 deletions.
diff --git a/controllers/llguidance_ctrl/src/earley/parser.rs b/controllers/llguidance_ctrl/src/earley/parser.rs
@@ -452,8 +452,12 @@ impl Parser {
     }
 
     pub fn lexer_allows_eos(&mut self) -> bool {
-        let curr = self.lexer_state();
-        self.lexer.allows_eos(curr.lexer_state)
+        if self.has_pending_lexeme_bytes() {
+            self.lexer.allows_eos(self.lexer_state().lexer_state)
+        } else {
+            // empty lexemes are not allowed
+            false
+        }
     }
 
     fn item_to_string(&self, idx: usize) -> String {

diff --git a/py/guidance b/py/guidance
+1 −1		guidance/_grammar.py
+18 −36		guidance/library/_gen.py
+2 −2		guidance/library/_substring.py
+1 −51		guidance/models/_model.py
+37 −0		tests/unit/library/test_greedy.py
+27 −0		tests/unit/test_grammar.py
+12 −1		tests/unit/test_ll.py