bugfix count total sentences

huggingface · Oct 16, 2023 · c5f0d0a · c5f0d0a
1 parent 2be346e
commit c5f0d0a
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/datatrove/pipeline/dedup/sentence_dedup.py b/src/datatrove/pipeline/dedup/sentence_dedup.py
@@ -220,7 +220,7 @@ def filter(self, doc: Document, du_lines: set = None):
         filtered_sentences = [sent for idx, sent in enumerate(sentences) if not du_lines or idx not in du_lines]
         if len(filtered_sentences) < len(sentences):
             self.stat_update("removed_sentences", len(sentences) - len(filtered_sentences))
-        self.stat_update()
+        self.stat_update("original_sentences", len(sentences))
         doc.content = " ".join(filtered_sentences).strip()
         if len(word_tokenize(doc.content)) > self.min_doc_words:
             return True