From b0e9a77c5cdd711c9495e4e4d28fb03b63879c49 Mon Sep 17 00:00:00 2001 From: JohnserfSeed Date: Fri, 15 Nov 2024 17:02:32 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E6=9B=B4=E6=96=B0WebSocketCrawler?= =?UTF-8?q?=E7=9A=84=E6=97=A5=E5=BF=97=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- f2/crawlers/base_crawler.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/f2/crawlers/base_crawler.py b/f2/crawlers/base_crawler.py index 719e506..58e280f 100644 --- a/f2/crawlers/base_crawler.py +++ b/f2/crawlers/base_crawler.py @@ -522,19 +522,21 @@ async def receive_messages(self): ) timeout_count = 0 + while True: try: message = await asyncio.wait_for( self.websocket.recv(), timeout=self.timeout ) # 为wss连接设置10秒超时机制 + timestamp = timestamp_2_str(time.time(), "%Y-%m-%d %H:%M:%S") logger.info( - _("[ReceiveMessages] | [⏳ 接收消息 {0}]").format( - timestamp_2_str(time.time(), "%Y-%m-%d %H:%M:%S") - ) + _("[ReceiveMessages] | [⏳ 接收消息 {0}]").format(timestamp) ) + timeout_count = 0 # 重置超时计数 await self.on_message(message) + except asyncio.TimeoutError: timeout_count += 1 logger.warning( @@ -543,11 +545,18 @@ async def receive_messages(self): ) ) if timeout_count >= 3: - logger.warning(_("[ReceiveMessages] [❌ 多次超时,关闭连接]")) + logger.warning( + _( + "[ReceiveMessages] [❌ 超时关闭连接] | " + "[超时次数:{0}] [连接状态:{1}]" + ).format(timeout_count, self.websocket.closed) + ) return "closed" if self.websocket.closed: logger.warning( - _("[ReceiveMessages] [🔒 服务器关闭] | [WebSocket 连接结束]") + _( + "[ReceiveMessages] [🔒 远程服务器关闭] | [WebSocket 连接结束]" + ) ) return "closed" except ConnectionClosedError as exc: @@ -556,11 +565,13 @@ async def receive_messages(self): _("[ReceiveMessages] [🔌 连接关闭] | [原因:{0}]").format(exc) ) return "closed" + except ConnectionClosedOK: logger.info( _("[ReceiveMessages] [✔️ 正常关闭] | [WebSocket 连接正常关闭]") ) return "closed" + except Exception as exc: logger.debug(traceback.format_exc()) logger.error(