From 3f1b0565a1b0a1e896c399d9fc9c78e86abf2cc9 Mon Sep 17 00:00:00 2001 From: Hendrik van Antwerpen Date: Thu, 10 Oct 2024 19:27:01 +0200 Subject: [PATCH] fmt --- crates/bpe-openai/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bpe-openai/src/lib.rs b/crates/bpe-openai/src/lib.rs index fe16ee2..24d5455 100644 --- a/crates/bpe-openai/src/lib.rs +++ b/crates/bpe-openai/src/lib.rs @@ -47,7 +47,7 @@ pub use bpe::*; /// produce the same output as the tiktoken tokenizers. The type gives access /// to the regex and underlying bye-pair encoding if needed. Note that using /// the byte-pair encoding directly does not take the regex into account and -/// may result in output that differs from tiktoken. +/// may result in output that differs from tiktoken. pub struct Tokenizer { /// The byte-pair encoding for this tokenizer. pub bpe: BytePairEncoding,