Skip to content

Commit

Permalink
fix: 增加try跳过编码有问题的log #3
Browse files Browse the repository at this point in the history 
fix: 增加一个环境变量用于指定需要运行的分析脚本 #4
feture:原来唯一性文件格式换为json，未来可在里面增加其他功能，比如记忆上次安装会报错的依赖不再无用功

Signed-off-by: keven <[email protected]>
  • Loading branch information
@keven1024
keven1024 committed Jun 7, 2022
1 parent 74d28d2 commit ff646f5
Show file tree
Hide file tree
Showing 2 changed files with 74 additions and 37 deletions.
31 changes: 25 additions & 6 deletions depend.py
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import json
import re
import sys
import os
Expand Down Expand Up @@ -63,21 +64,39 @@ def re_filter_list(string, filter_list):
return True
return False

def only_check(self, pyname, pyabspath):
def only_check(self, pyname, pyabspath,osenv=None):
only_path = self.get_ql_path() + pyname + '_by_keven1024'
result = "☺当前脚本目录为: " + str(pyabspath) + "\n"
j_data = {
"py_path": None
}
if osenv and self.get_env(osenv):
result += "😏检测到环境变量：" + str(osenv) + " = " + self.get_env(osenv) + " 将按照该路径为准\n"
pyabspath = self.get_env(osenv)
if os.path.exists(only_path):
with open(only_path, 'r') as f:
if f.read(2048) != pyabspath:
with open(only_path, 'r+') as f:
py_data = f.read(2097152)
if py_data == pyabspath:
# 对旧版转换为json格式
j_data["py_path"] = pyabspath
f.seek(0, os.SEEK_SET)
result += "😏检测到旧版检测文件，自动转换新版\n"
f.writelines(json.dumps(j_data))
try:
j_data = json.loads(py_data)
except:
pass
if j_data["py_path"] and j_data["py_path"] == pyabspath:
result += "😁脚本唯一性检测通过，继续运行!\n"
else:
result += "🙄检测到其他同类型的青龙日志分析脚本存在，拒绝运行!\n"
load_send()
send(pyname, result)
exit(0)
else:
result += "😁脚本唯一性检测通过，继续运行!\n"
else:
with open(only_path, "w") as f:
f.writelines(pyabspath)
j_data["py_path"] = pyabspath
f.writelines(json.dumps(j_data))
result += "🙄检测到第一次运行，已写入唯一性检测文件，如无特殊情况请勿删除\n"
return result

Expand Down
80 changes: 49 additions & 31 deletions ql_log_scan.py
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,20 @@
import json
import os
import re
import time
import sys
import subprocess
import traceback
from depend import Depend

'''
cron: 30 23 * * *
new Env('青龙日志分析 && 自动补全依赖');
########环境变量设置#########
## (非必填) 脚本唯一性检测，请在此处填写你想运行的脚本的绝对路径，其他脚本检测到路径与此变量不符将会停止运行
QL_LOG_SCAN_SCRIPT_PATH=
## (非必填)指定日志目录: 默认自动识别青龙目录，出现错误才需要手动指定日志目录
export QL_LOG_PATH="/ql/data/log/"
Expand All @@ -32,8 +37,8 @@

class QlLogScan(Depend):
def __init__(self):
self.pyname = os.path.basename(__file__)
print(self.only_check(self.pyname, os.path.abspath(__file__)))
self.pyname = os.path.basename(__file__).replace(".py", "")
print(self.only_check(self.pyname, os.path.abspath(__file__),"QL_LOG_SCAN_SCRIPT_PATH"))
self.ql_log_path = self.get_env("QL_LOG_PATH", self.get_ql_path() + "log/")
self.filter_dir_list = self.not2append(["^\.tmp$", "^update$", self.pyname + "$"],
self.str2list(self.get_env("QL_LOG_BLACK_DIR")))
Expand All @@ -48,7 +53,8 @@ def __init__(self):
"python_err": 0,
"err_dict": {},
"nodejs_depend": [],
"python_depend": []
"python_depend": [],
"readlog_err" :[]
}
self.LogNameHeadList = self.generateLogNameHeadList()
self.analysisLog()
Expand All @@ -72,35 +78,41 @@ def analysisLog(self):
:13] in self.LogNameHeadList:
# 读取日志
log_file = open(os.path.join(path, file_name), "r")
log_text = log_file.read()
log_file.close()
# 分析日志
nodejs_err_list = re.findall(r"Error\:(.*\s?)Require stack\:", log_text)
python_err_list = re.findall(
r"Traceback \(most recent call last\):([\n\s]+File[\s\S]*?, line [\d]+, in[\s\S]*?["
r"\s\S]*?\n[\s\S]*?\n)+(.*?)\n",
try:
log_text = log_file.read(2097152)
log_file.close()
# 分析日志
nodejs_err_list = re.findall(r"Error\:(.*\s?)Require stack\:", log_text)
python_err_list = re.findall(
r"Traceback \(most recent call last\):([\n\s]+File[\s\S]*?, line [\d]+, in[\s\S]*?["
r"\s\S]*?\n[\s\S]*?\n)+(.*?)\n",
log_text)
if nodejs_err_list:
self.log_stat["nodejs_err"] += len(nodejs_err_list)
self.log_stat["err_dict"][dir_name] = []
for i in nodejs_err_list:
v = i.strip()
self.log_stat["err_dict"][dir_name].append({"type": "NodeJs", "log": v})
# 依赖缺失判断
miss_depend = re.search(r"Cannot find module '([a-zA-Z\d_-]+)'", v)
if miss_depend and miss_depend.group(1) not in self.log_stat["nodejs_depend"]:
self.log_stat["nodejs_depend"].append(miss_depend.group(1))
elif python_err_list:
self.log_stat["python_err"] += len(python_err_list)
self.log_stat["err_dict"][dir_name] = []
for i in python_err_list:
v = i[-1].strip()
self.log_stat["err_dict"][dir_name].append({"type": "Python", "log": v})
# 依赖缺失判断
miss_depend = re.search(r"ModuleNotFoundError: No module named \'([a-zA-Z0-9_-]+)\'", v)
if miss_depend and miss_depend.group(1) not in self.log_stat["python_depend"]:
self.log_stat["python_depend"].append(miss_depend.group(1))
self.log_stat["all"] += 1
if nodejs_err_list:
self.log_stat["nodejs_err"] += len(nodejs_err_list)
self.log_stat["err_dict"][dir_name] = []
for i in nodejs_err_list:
v = i.strip()
self.log_stat["err_dict"][dir_name].append({"type": "NodeJs", "log": v})
# 依赖缺失判断
miss_depend = re.search(r"Cannot find module '([a-zA-Z\d_-]+)'", v)
if miss_depend and miss_depend.group(1) not in self.log_stat["nodejs_depend"]:
self.log_stat["nodejs_depend"].append(miss_depend.group(1))
elif python_err_list:
self.log_stat["python_err"] += len(python_err_list)
self.log_stat["err_dict"][dir_name] = []
for i in python_err_list:
v = i[-1].strip()
self.log_stat["err_dict"][dir_name].append({"type": "Python", "log": v})
# 依赖缺失判断
miss_depend = re.search(r"ModuleNotFoundError: No module named \'([a-zA-Z0-9_-]+)\'", v)
if miss_depend and miss_depend.group(1) not in self.log_stat["python_depend"]:
self.log_stat["python_depend"].append(miss_depend.group(1))
self.log_stat["all"] += 1
except Exception as e:
err_log = "读取日志" + str(os.path.join(path, file_name)) + "出现异常: " + str(e) + "\n"
self.log_stat["readlog_err"].append(err_log)
print(err_log)


@staticmethod
def format_log_date(text):
Expand All @@ -116,6 +128,8 @@ def showAnalysisLog(self):
self.LogNameHeadList[-1]) if len(self.LogNameHeadList) != 1 else
self.LogNameHeadList[
0]) + " 的日志报告：\n"
if len(self.log_stat["readlog_err"]) != 0:
result += "🔍脚本在读取日志过程中，出现了" + str(len(self.log_stat["readlog_err"])) + "个异常，详细信息将在最后展示\n"
result += "✅正常运行脚本：" + str(self.log_stat["all"]) + " 次\n"
if self.log_stat["all"] != 0:
result += "⛔异常运行脚本：" + str(self.log_stat["nodejs_err"] + self.log_stat["python_err"]) + " 次，占比 " + str(
Expand Down Expand Up @@ -144,6 +158,10 @@ def showAnalysisLog(self):
result += "🛑脚本：" + k + "：\n"
for i in v:
result += "- ⚠" + i["type"] + "错误：" + i["log"] + " \n\n\n"
if len(self.log_stat["readlog_err"]) != 0:
result += "👷‍♀️读取日志异常日志：\n\n"
for i in self.log_stat["readlog_err"]:
result += "⚠" + i + "\n"
send("🐲青龙日志分析", result)
return result

Expand Down

0 comments on commit ff646f5

Please sign in to comment.