skip non-utf8 content

saveweb · Jul 26, 2024 · 9b0e3de · 9b0e3de
1 parent bb5e8b8
commit 9b0e3de
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/src/googlrot/main.py b/src/googlrot/main.py
@@ -110,6 +110,9 @@ async def code_mode(g: Github, googl_perfix_queue_collection: AsyncIOMotorCollec
                     continue
                 else:
                     raise e
+            except UnicodeDecodeError:
+                logger.error(f"UnicodeDecodeError: {result.repository.full_name}, skip")
+                continue
             print("conetnt: ", content[:256])
             for url in extractor.gen_urls(content):
                 assert isinstance(url, str)