fix hudson-ai/guidance#19

controllers/llguidance_ctrl/run_g.py

@@ -220,6 +220,15 @@ def character_maker2(lm, id, description, valid_weapons):
     )
     grm = "Here: 2 + 2 = " + greedy_grammar(name="num", body=lexeme("[0-9]+"))
 
+    grm = "Here: 1 / 10 = " + greedy_grammar(
+        body=select(
+            [
+                lexeme(r"-?(?:0|[1-9][0-9]*)", contextual=True),
+                lexeme(r"-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)", contextual=True),
+            ]
+        )
+    )
+
     # grm = "Here: 2 + 2 = " + guidance.json(name="num", schema={"type": "integer"})
     # grm = guidance.json(name="num", schema={"type": "integer"})
     # m = grm.match("123<s>")

controllers/llguidance_ctrl/src/earley/regexvec.rs

@@ -124,26 +124,47 @@ impl RegexVec {
             + self.rx_sets.num_bytes()
     }
 
-    /// Return index of lowest matching regex if any.
-    /// Lazy regexes match as soon as they accept, while greedy only
-    /// if they accept and force EOI.
-    pub fn lowest_match(&mut self, state: StateID) -> Option<(usize, usize)> {
-        let desc = &mut self.state_descs[state.as_usize()];
-        if let Some(lowest_match) = desc.lowest_match {
-            return lowest_match;
-        }
-        let mut res = None;
+    fn lowest_match_inner(&mut self, state: StateID) -> Option<(usize, usize)> {
+        let mut all_eoi = true;
+        let mut eoi_candidate = None;
+        // fine the first lazy matching regex
+        // failing that, if all regexes are matching and force EOI, pick the first one
         for (idx, e) in iter_state(&self.rx_sets, state) {
             if !self.exprs.is_nullable(e) {
+                all_eoi = false;
                 continue;
             }
-            if self.lazy[idx] || self.next_byte.next_byte(&self.exprs, e) == NextByte::ForcedEOI {
+            if self.lazy[idx] {
                 let len = self.exprs.possible_lookahead_len(e);
-                res = Some((idx, len));
-                break;
+                return Some((idx, len));
+            }
+            if all_eoi {
+                if self.next_byte.next_byte(&self.exprs, e) == NextByte::ForcedEOI {
+                    if eoi_candidate.is_none() {
+                        eoi_candidate = Some((idx, self.exprs.possible_lookahead_len(e)));
+                    }
+                } else {
+                    all_eoi = false;
+                }
             }
         }
-        desc.lowest_match = Some(res);
+
+        if all_eoi {
+            eoi_candidate
+        } else {
+            None
+        }
+    }
+
+    /// Return index of lowest matching regex if any.
+    /// Lazy regexes match as soon as they accept, while greedy only
+    /// if they accept and force EOI.
+    pub fn lowest_match(&mut self, state: StateID) -> Option<(usize, usize)> {
+        if let Some(lowest_match) = self.state_descs[state.as_usize()].lowest_match {
+            return lowest_match;
+        }
+        let res = self.lowest_match_inner(state);
+        self.state_descs[state.as_usize()].lowest_match = Some(res);
         res
     }