Merge pull request #218 from bzz/tokenizer-flex-cgo

New, optional flex-based tokenizer
src-d · Apr 17, 2019 · b6daf5c · b6daf5c
2 parents ab3c26b + 7e136ba
commit b6daf5c
Show file tree

Hide file tree

Showing 9 changed files with 2,707 additions and 15 deletions.
diff --git a/.gitignore b/.gitignore
@@ -8,3 +8,4 @@ Makefile.main
 build/
 vendor/
 java/lib/
+.vscode/
diff --git a/internal/tokenizer/common.go b/internal/tokenizer/common.go
@@ -0,0 +1,7 @@
+// Package tokenizer implements file tokenization used by the enry content
+// classifier. This package is an implementation detail of enry and should not
+// be imported by other packages.
+package tokenizer
+
+// ByteLimit defines the maximum prefix of an input text that will be tokenized.
+const ByteLimit = 100000
-Original file line number
+Diff line change
@@ Expand Up / @@ -8,3 +8,4 @@ Makefile.main @@
     build/
     vendor/
     java/lib/
+    .vscode/