remove SimpleVob dep from derivre

microsoft · Jul 3, 2024 · a8d036d · a8d036d
1 parent 19432bf
commit a8d036d
Show file tree

Hide file tree

Showing 7 changed files with 12 additions and 14 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/controllers/derivre/Cargo.toml b/controllers/derivre/Cargo.toml
@@ -8,7 +8,6 @@ name = "derivre"
 
 [dependencies]
 ahash = "0.8.11"
-aici_abi = { path = "../aici_abi" }
 anyhow = "1.0.75"
 bytemuck = "1.16.0"
 bytemuck_derive = "1.6.1"

diff --git a/controllers/derivre/src/bytecompress.rs b/controllers/derivre/src/bytecompress.rs
@@ -2,7 +2,7 @@ use std::{collections::HashMap, hash::Hash};
 
 use crate::{
     ast::{byteset_contains, byteset_set, Expr, ExprSet},
-    ExprRef, SimpleVob,
+    ExprRef,
 };
 
 pub struct ByteCompressor {
@@ -82,12 +82,12 @@ impl ByteCompressor {
         self.mapping = vec![INVALID_MAPPING; exprset.alphabet_size()];
 
         let mut todo = rx_list.to_vec();
-        let mut visited = SimpleVob::alloc(exprset.len());
+        let mut visited = vec![false; exprset.len()];
         while let Some(e) = todo.pop() {
-            if visited.get(e.as_usize()) {
+            if visited[e.as_usize()] {
                 continue;
             }
-            visited.set(e.as_usize(), true);
+            visited[e.as_usize()] = true;
             todo.extend_from_slice(exprset.get_args(e));
             match exprset.get(e) {
                 Expr::Byte(b) => {
@@ -109,7 +109,10 @@ impl ByteCompressor {
         if num <= 64 {
             self.compress_bytesets(|_| 0u64, |v, idx| *v |= 1 << idx);
         } else {
-            self.compress_bytesets(|size| SimpleVob::alloc(size), |v, idx| v.set(idx, true));
+            self.compress_bytesets(
+                |size| vec![0u32; (size + 31) / 32],
+                |v, idx| v[idx / 32] |= 1 << (idx % 32),
+            );
         }
 
         let mut trg = ExprSet::new(self.alphabet_size);

diff --git a/controllers/derivre/src/lib.rs b/controllers/derivre/src/lib.rs
@@ -14,9 +14,6 @@ mod syntax;
 pub use ast::{ExprRef, NextByte};
 pub use regex::{AlphabetInfo, Regex, StateID};
 
-use aici_abi::svob;
-pub use svob::{SimpleVob, SimpleVobIter, TokenId};
-
 pub use regexbuilder::{RegexAst, RegexBuilder};
 
 pub use mapper::map_ast; // utility function

diff --git a/controllers/llguidance_ctrl/src/earley/regexvec.rs b/controllers/llguidance_ctrl/src/earley/regexvec.rs
@@ -1,7 +1,8 @@
+use aici_abi::svob::SimpleVob;
 use derivre::raw::{DerivCache, ExprSet, NextByteCache, VecHashCons};
 use std::fmt::Debug;
 
-pub use derivre::{AlphabetInfo, ExprRef, NextByte, SimpleVob, StateID};
+pub use derivre::{AlphabetInfo, ExprRef, NextByte, StateID};
 
 #[derive(Clone)]
 pub struct RegexVec {

diff --git a/controllers/llguidance_ctrl/src/tokenparser.rs b/controllers/llguidance_ctrl/src/tokenparser.rs
@@ -4,9 +4,8 @@ use crate::{
     api::{GenGrammarOptions, TopLevelGrammar},
     earley::{grammars_from_json, CGrammar, CSymIdx, ModelVariable, Parser, ParserStats},
 };
-use aici_abi::{MidProcessArg, MidProcessResult, TokenId, TokenizerEnv};
+use aici_abi::{svob::SimpleVob, MidProcessArg, MidProcessResult, TokenId, TokenizerEnv};
 use anyhow::Result;
-use derivre::SimpleVob;
 use serde_json::json;
 
 macro_rules! infoln {

diff --git a/py/guidance b/py/guidance
+0 −15		tests/conftest.py
+2 −1		tests/model_integration/library/test_commit_point.py
+2 −2		tests/model_integration/library/test_gen.py
+2 −1		tests/unit/library/test_commit_point.py
+1 −1		tests/unit/library/test_gen.py
+0 −1		tests/unit/library/test_substring.py
+0 −6		tests/xfail.txt