fix xhtml+xml

huggingface · Dec 11, 2024 · b701935 · b701935
1 parent 842b241
commit b701935
Showing 1 changed file with 8 additions and 2 deletions.
diff --git a/src/datatrove/pipeline/readers/warc.py b/src/datatrove/pipeline/readers/warc.py
@@ -96,15 +96,21 @@ def process_record(record: "ArcWarcRecord") -> dict | None:
     # content type filtering
     mime_type = record.rec_headers.get("WARC-Identified-Payload-Type", None)
     if mime_type is not None and (
-        mime_type != "text/html" and (record.rec_type != "conversion" or mime_type != "text/plain")
+        mime_type != "text/html"
+        and mime_type != "application/xhtml+xml"
+        and (record.rec_type != "conversion" or mime_type != "text/plain")
     ):
         return
 
     content_bytes = record.content_stream().read()
     if mime_type is None:
         # fallback for older crawls without payload types
         mime_type = magic.from_buffer(content_bytes, mime=True)
-        if mime_type != "text/html" and (record.rec_type != "conversion" or mime_type != "text/plain"):
+        if (
+            mime_type != "text/html"
+            and mime_type != "application/xhtml+xml"
+            and (record.rec_type != "conversion" or mime_type != "text/plain")
+        ):
             return
 
     # Decode the response bytes