Skip to content

Commit

Permalink
新增 网站监控异常恢复通知事件
Browse files Browse the repository at this point in the history
  • Loading branch information
@mouday
mouday committed May 7, 2024
1 parent 04497ee commit db7e624
Show file tree
Hide file tree
Showing 6 changed files with 287 additions and 13 deletions.
3 changes: 3 additions & 0 deletions doc/source/manual/notify.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -397,6 +397,8 @@ WebHook、企业微信、飞书、钉钉均支持自定义通知模板


### 6.4、监控异常
异常通知和异常恢复通知的字段一致

```json
{
Expand All @@ -419,3 +421,4 @@ WebHook、企业微信、飞书、钉钉均支持自定义通知模板
| monitor_row.http_url | string | 请求URL
| monitor_row.allow_error_count | int | 重试次数
| monitor_row.status | int | 状态：0未知，1成功，2失败

2 changes: 2 additions & 0 deletions doc/source/manual/plan.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
- `已完成` 支持内网用户自己设置过期时间
- `已完成` 增加筛选功能: 筛选域名状态、证书状态、网站状态
- `已完成` 增加批量删除域名的功能
- 支持Telegram 告警
- 监控网站恢复后发送恢复的报警邮件

- v1.5.x（开发中）
- 新增网站监控 [issues#17](https://github.com/mouday/domain-admin/issues/17)
Expand Down
3 changes: 3 additions & 0 deletions domain_admin/enums/event_enum.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,3 +22,6 @@ class EventEnum(object):

# 托管证书到期
SSL_CERT_FILE_EXPIRE = 3

# 监控异常恢复
MONITOR_EXCEPTION_RESTORE = 4
51 changes: 38 additions & 13 deletions domain_admin/service/monitor_service.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,26 +85,29 @@ def wrapper(monitor_row, *args, **kwargs):
handle_monitor_exception(monitor_row, error)
raise error
else:
handle_monitor_exception_restore(monitor_row)

return result

return wrapper


def handle_monitor_exception(monitor_row, error):
if monitor_row.allow_error_count > 0:
# 检查连续失败次数是否大于最大允许失败次数，增加容错
rows = LogMonitorModel.select().where(
LogMonitorModel.monitor_id == monitor_row.id,
).order_by(
LogMonitorModel.id.desc()
).limit(
monitor_row.allow_error_count + 1
)
def handle_monitor_exception_restore(monitor_row):
# 如果上一次记录是失败的，则需要触发监测异常恢复通知
if monitor_row.status != MonitorStatusEnum.ERROR:
return

# 检查连续失败次数是否大于最大允许失败次数，增加容错
if monitor_row.allow_error_count > 0 and is_between_allow_error_count(monitor_row):
return

error_count = len([row for row in rows if row.status == MonitorStatusEnum.ERROR])
notify_service.notify_user_about_monitor_exception_restore(monitor_row)

if error_count <= monitor_row.allow_error_count:
return

def handle_monitor_exception(monitor_row, error):
# 检查连续失败次数是否大于最大允许失败次数，增加容错
if monitor_row.allow_error_count > 0 and is_between_allow_error_count(monitor_row):
return

# 发送异常通知
notify_service.notify_user_about_monitor_exception(monitor_row, error)
Expand Down Expand Up @@ -267,3 +270,25 @@ def import_monitor_from_file(filename, user_id):
# https://github.com/mouday/domain-admin/issues/63
for batch in chunked(lst, 500):
MonitorModel.insert_many(batch).on_conflict_ignore().execute()


def is_between_allow_error_count(monitor_row):
"""
连续失败次数是否在允许范围内
:param monitor_row:
:return:
注意：先触发的事件，才写入日志
"""
# 检查连续失败次数是否大于最大允许失败次数，增加容错
rows = LogMonitorModel.select().where(
LogMonitorModel.monitor_id == monitor_row.id,
).order_by(
LogMonitorModel.id.desc()
).limit(
monitor_row.allow_error_count + 1
)

error_count = len([row for row in rows if row.status == MonitorStatusEnum.ERROR])

return error_count <= monitor_row.allow_error_count
24 changes: 24 additions & 0 deletions domain_admin/service/notify_service.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,11 @@
'email_template': 'monitor-email.html',
'email_subject': '[Domain Admin]监控异常提醒',
},
{
'event_id': EventEnum.MONITOR_EXCEPTION_RESTORE,
'email_template': 'monitor-restore-email.html',
'email_subject': '[Domain Admin]监控异常恢复提醒',
},
{
'event_id': EventEnum.SSL_CERT_FILE_EXPIRE,
'email_template': 'cert-email.html',
Expand Down Expand Up @@ -190,6 +195,25 @@ def notify_user_about_monitor_exception(monitor_row, error):
logger.error(traceback.format_exc())


def notify_user_about_monitor_exception_restore(monitor_row):
"""
监控异常恢复通知
:param monitor_row:
:param error:
:return:
"""
rows = NotifyModel.select().where(
NotifyModel.status == True,
NotifyModel.event_id == EventEnum.MONITOR_EXCEPTION_RESTORE
)

for row in rows:
try:
notify_user(notify_row=row, rows=rows, data={'monitor_row': monitor_row})
except:
logger.error(traceback.format_exc())


# @async_task_service.sync_task_decorator("触发通知用户")
def notify_user_about_some_event(notify_row):
"""
Expand Down
217 changes: 217 additions & 0 deletions domain_admin/templates/monitor-restore-email.html
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,217 @@
{# 查询域名到期情况 #}
<html lang="zh">
<head>
<meta charset=utf-8>
<meta name="viewport" content="width=device-width,initial-scale=1.0">
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
</head>

<body>
<div>
<div
style="
box-sizing: border-box;
text-align: center;
min-width: 320px;
max-width: 800px;
border: 1px solid #f6f6f6;
background-color: #f6f6f6;
margin: auto;
padding: 20px 0 30px;
font-family: 'helvetica neue', PingFangSC-Light, arial,
'hiragino sans gb', 'microsoft yahei ui', 'microsoft yahei', simsun,
sans-serif;
"
>
<div>
<table
style="
width: 100%;
font-weight: 300;
margin-bottom: 10px;
border-collapse: collapse;
"
>
<tbody>
<tr style="font-weight: 300">
<td style="width: 3%; max-width: 30px"></td>
<td style="max-width: 600px">
<div
style="
height: 25px;
text-align: left;
margin-bottom: 10px;
margin-top: 10px;
"
>
<a
href="https://github.com/mouday/domain-admin"
rel="noopener"
target="_blank"
style="text-decoration: none; color: black"
>
<div style="display: flex">
<span
style="
font-size: 30px;
margin-left: 3px;
font-family: sans-serif;
height: 24px;
line-height: 24px;
"
>Domain Admin</span
>
</div>
</a
>
</div>
<p
style="
height: 2px;
background-color: #006EFF;
border: 0;
font-size: 0;
padding: 0;
width: 100%;
margin-top: 10px;
"
></p>
<div
style="
background-color: #fff;
padding: 23px 0 20px;
box-shadow: 0px 1px 1px 0px rgba(122, 55, 55, 0.2);
text-align: left;
"
>
<table
style="
width: 100%;
font-weight: 300;
margin-bottom: 10px;
border-collapse: collapse;
text-align: left;
"
>
<tbody>
<tr style="font-weight: 300">
<td style="width: 3.2%; max-width: 30px"><br/></td>
<td style="max-width: 480px; text-align: left">
<div
style="
font-size: 14px;
color: #333;
line-height: 24px;
margin: 6px 0 0 0;
word-wrap: break-word;
word-break: break-all;
"
>

<div style="font-size: 12px;">监控异常恢复提醒：</div>

<div style="font-size: 14px; color: #000000; margin-top: 20px;">
<div>名称：{{monitor_row.title}}</div>
<div>URL：{{monitor_row.http_url}}</div>
<div>监控异常已恢复</div>
</div>

<div style="margin-top: 20px; font-size: 12px;">请持续关注</div>

</div>
<br/>
<dl
style="
font-size: 14px;
color: #333;
line-height: 18px;
"
></dl>
<p
style="
color: #333;
font-size: 14px;
line-height: 26px;
word-wrap: break-word;
word-break: break-all;
margin-top: 32px;
"
>
此致<br/><strong>Domain Admin团队</strong>
</p>
</td>
<td style="width: 3.2%; max-width: 30px"></td>
</tr>
</tbody>
</table>
</div>
<div
style="
text-align: center;
font-size: 12px;
line-height: 18px;
color: #999;
"
>
<table
style="
width: 100%;
font-weight: 300;
margin-bottom: 10px;
border-collapse: collapse;
"
>
<tbody>
<tr style="font-weight: 300">
<td style="width: 3.2%; max-width: 30px"></td>
<td style="max-width: 540px">
<p
style="
text-align: center;
margin: 20px auto 14px auto;
font-size: 12px;
color: #999;
font-family: sans-serif;
"
>
此为系统邮件，请勿回复。
</p>
<p
style="
text-align: center;
margin: 20px auto 5px auto;
font-size: 12px;
color: #999;
font-family: sans-serif;
"
>
Copyright © 2023 Domain Admin
</p>
<p
style="
text-align: center;
margin: 5px auto 0px auto;
font-size: 12px;
color: #999;
font-family: sans-serif;
"
>
All Rights Reserved Domain Admin 版权所有
</p>
</td>
<td style="width: 3.2%; max-width: 30px"></td>
</tr>
</tbody>
</table>
</div>
</td>
<td style="width: 3%; max-width: 30px"></td>
</tr>
</tbody>
</table>
</div>
</div>
</div>

</body>
</html>

0 comments on commit db7e624

Please sign in to comment.