From 6322036279b049553cbde22e4e362a99bbd6c8c5 Mon Sep 17 00:00:00 2001 From: pengshiyu <1940607002@qq.com> Date: Sat, 24 Feb 2024 21:07:44 +0800 Subject: [PATCH] =?UTF-8?q?=E7=BD=91=E7=AB=99=E7=9B=91=E6=8E=A7=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E9=94=99=E8=AF=AF=E9=87=8D=E8=AF=95=E6=AC=A1=E6=95=B0?= =?UTF-8?q?=EF=BC=9B=E7=BD=91=E7=AB=99=E7=9B=91=E6=8E=A7=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E5=AF=BC=E5=85=A5=E5=AF=BC=E5=87=BA=EF=BC=9B=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E8=80=97=E6=97=B6=E6=98=BE=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- domain_admin/api/monitor_api.py | 82 +++++++++++++++- domain_admin/api/system_api.py | 7 ++ domain_admin/config/env_config.py | 2 +- domain_admin/enums/version_enum.py | 4 + domain_admin/main.py | 3 +- .../migrate/history/migrate_1610_to_1611.py | 35 +++++++ domain_admin/migrate/migrate_config.py | 19 +++- domain_admin/model/monitor_model.py | 34 +++++++ domain_admin/router/api_map.py | 5 +- domain_admin/service/monitor_service.py | 95 ++++++++++++++++++- .../scheduler_service/scheduler_main.py | 8 ++ domain_admin/utils/datetime_util.py | 11 ++- 12 files changed, 289 insertions(+), 16 deletions(-) create mode 100644 domain_admin/migrate/history/migrate_1610_to_1611.py diff --git a/domain_admin/api/monitor_api.py b/domain_admin/api/monitor_api.py index 7fbc369b3d..e920edddfc 100644 --- a/domain_admin/api/monitor_api.py +++ b/domain_admin/api/monitor_api.py @@ -13,7 +13,7 @@ from domain_admin.model.log_monitor_model import LogMonitorModel from domain_admin.model.monitor_model import MonitorModel -from domain_admin.service import monitor_service +from domain_admin.service import monitor_service, file_service, async_task_service from domain_admin.service.scheduler_service import scheduler_main @@ -25,12 +25,14 @@ def add_monitor(): current_user_id = g.user_id title = request.json['title'] monitor_type = request.json['monitor_type'] + allow_error_count = request.json.get('allow_error_count') or 0 content = request.json['content'] interval = request.json['interval'] monitor_row = MonitorModel.create( user_id=current_user_id, title=title, + allow_error_count=allow_error_count, monitor_type=monitor_type, content=json.dumps(content), interval=interval @@ -50,11 +52,13 @@ def update_monitor_by_id(): title = request.json['title'] content = request.json['content'] interval = request.json['interval'] + allow_error_count = request.json.get('allow_error_count') or 0 MonitorModel.update( title=title, content=json.dumps(content), - interval=interval + interval=interval, + allow_error_count=allow_error_count, ).where( MonitorModel.id == monitor_id ).execute() @@ -119,6 +123,7 @@ def get_monitor_list(): order_prop = request.json.get('order_prop') or 'create_time' order_type = request.json.get('order_type') or 'desc' keyword = request.json.get('keyword') + status = request.json.get('status') query = MonitorModel.select().where( MonitorModel.user_id == current_user_id @@ -127,6 +132,9 @@ def get_monitor_list(): if keyword: query = query.where(MonitorModel.title.contains(keyword)) + if isinstance(status, int): + query = query.where(MonitorModel.status == status) + total = query.count() lst = [] @@ -147,3 +155,73 @@ def get_monitor_list(): 'list': lst, 'total': total } + + +def export_monitor_file(): + """ + 导出监控文件 + csv格式 + :return: + """ + current_user_id = g.user_id + + keyword = request.json.get('keyword') + status = request.json.get('status') + ext = request.json.get('ext', 'csv') + + order_prop = request.json.get('order_prop') or 'create_time' + order_type = request.json.get('order_type') or 'desc' + + params = { + 'keyword': keyword, + 'status': status, + 'user_id': current_user_id, + } + + query = monitor_service.get_monitor_list_query(**params) + + ordering = [ + SQL(f"`{order_prop}` {order_type}"), + MonitorModel.id.desc() + ] + + rows = query.order_by(*ordering) + + lst = [row.to_dict() for row in rows] + + filename = monitor_service.export_monitor_to_file(rows=lst, ext=ext) + + return { + 'name': filename, + 'url': file_service.resolve_temp_url(filename) + } + + +def import_monitor_from_file(): + """ + 从文件导入域名 + 支持 xlsx 和 csv格式 + :return: + """ + current_user_id = g.user_id + + update_file = request.files.get('file') + + filename = file_service.save_temp_file(update_file) + + # 导入数据 + monitor_service.import_monitor_from_file(filename, current_user_id) + + # 异步查询 + run_init_monitor_task_async(user_id=current_user_id) + + +@async_task_service.async_task_decorator("更新监控信息") +def run_init_monitor_task_async(user_id): + rows = MonitorModel.select().where( + MonitorModel.user_id == user_id, + MonitorModel.version == 0 + ) + + for row in rows: + scheduler_main.run_one_monitor_task(row) diff --git a/domain_admin/api/system_api.py b/domain_admin/api/system_api.py index 75c0b67466..deae396bc1 100644 --- a/domain_admin/api/system_api.py +++ b/domain_admin/api/system_api.py @@ -13,6 +13,7 @@ from domain_admin.model.monitor_model import MonitorModel from domain_admin.model.system_model import SystemModel from domain_admin.service import scheduler_service +from domain_admin.service.scheduler_service import scheduler_main from domain_admin.utils import datetime_util @@ -218,3 +219,9 @@ def get_system_data(): 'path': '/monitor/list' } ] + + +def get_monitor_task_next_run_time(): + return { + 'next_run_time': scheduler_main.get_monitor_task_next_run_time() + } diff --git a/domain_admin/config/env_config.py b/domain_admin/config/env_config.py index 108a008c11..774f52c9d9 100644 --- a/domain_admin/config/env_config.py +++ b/domain_admin/config/env_config.py @@ -28,5 +28,5 @@ # token_expire_days TOKEN_EXPIRE_DAYS = env.int("TOKEN_EXPIRE_DAYS", DEFAULT_TOKEN_EXPIRE_DAYS) -# APP_MODE +# APP_MODE : production, development APP_MODE = env.str("APP_MODE", 'production') diff --git a/domain_admin/enums/version_enum.py b/domain_admin/enums/version_enum.py index 8e67ee7267..4017004196 100644 --- a/domain_admin/enums/version_enum.py +++ b/domain_admin/enums/version_enum.py @@ -170,3 +170,7 @@ class VersionEnum(object): Version_167 = '1.6.7' Version_168 = '1.6.8' Version_169 = '1.6.9' + + Version_1610 = '1.6.10' + Version_1611 = '1.6.11' + Version_1612 = '1.6.12' diff --git a/domain_admin/main.py b/domain_admin/main.py index 1349b55e89..9de332328e 100644 --- a/domain_admin/main.py +++ b/domain_admin/main.py @@ -7,7 +7,7 @@ from domain_admin import compat from domain_admin.log import logger -from domain_admin.config import TEMP_DIR +from domain_admin.config import TEMP_DIR, APP_MODE from domain_admin.model.base_model import db from domain_admin.model.database import init_database from domain_admin.router import api_map, permission @@ -107,6 +107,7 @@ def init_app(flask_app): # 版本自动升级 version_service.update_version() + # if APP_MODE == 'production': # 启动定时器 scheduler_service.init_scheduler() diff --git a/domain_admin/migrate/history/migrate_1610_to_1611.py b/domain_admin/migrate/history/migrate_1610_to_1611.py new file mode 100644 index 0000000000..0512da0cc6 --- /dev/null +++ b/domain_admin/migrate/history/migrate_1610_to_1611.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +""" +@File : migrate_1610_to_1611.py +@Date : 2024-02-24 + +cmd: +$ python domain_admin/migrate/migrate_1610_to_1611.py +""" +from __future__ import print_function, unicode_literals, absolute_import, division + +from domain_admin.migrate import migrate_common +from domain_admin.model.address_model import AddressModel +from domain_admin.model.base_model import db +from domain_admin.model.domain_model import DomainModel +from domain_admin.model.monitor_model import MonitorModel +from domain_admin.model.notify_model import NotifyModel + + +def execute_migrate(): + """ + 版本升级 1.6.10 => 1.6.11 + :return: + """ + migrator = migrate_common.get_migrator(db) + + migrate_rows = [ + # version + migrator.add_column( + table=MonitorModel._meta.table_name, + column_name=MonitorModel.version.name, + field=MonitorModel.version + ) + ] + + migrate_common.try_execute_migrate(migrate_rows) diff --git a/domain_admin/migrate/migrate_config.py b/domain_admin/migrate/migrate_config.py index fd9049a0c6..cf4b5a34ee 100644 --- a/domain_admin/migrate/migrate_config.py +++ b/domain_admin/migrate/migrate_config.py @@ -24,11 +24,16 @@ migrate_1520_to_1521, migrate_154_to_155, migrate_106_to_110, - migrate_1533_to_1534, migrate_162_to_163, migrate_168_to_169) + migrate_1533_to_1534, + migrate_162_to_163, + migrate_168_to_169, + migrate_1610_to_1611 +) +# 参数说明 # local_versions 本地版本 # migrate_func 升级函数 -# update_version +# update_version 升级后的版本 MIGRATE_CONFIG = [ # 1.0.0 1.0.1 1.0.2 => 1.0.3 @@ -340,4 +345,14 @@ 'update_version': VersionEnum.Version_169 }, + # 2024-02-24 + # 1.6.10 => 1.6.11 + { + 'local_versions': [ + VersionEnum.Version_169, + VersionEnum.Version_1610, + ], + 'migrate_func': migrate_1610_to_1611.execute_migrate, + 'update_version': VersionEnum.Version_1611 + }, ] diff --git a/domain_admin/model/monitor_model.py b/domain_admin/model/monitor_model.py index 82b32626d6..ca0217a2ef 100644 --- a/domain_admin/model/monitor_model.py +++ b/domain_admin/model/monitor_model.py @@ -48,6 +48,9 @@ class MonitorModel(BaseModel): # 下一次运行时间 next_run_time = DateTimeField(default=None, null=True) + # 数据版本号 @since 1.6.11 + version = IntegerField(default=0, null=False) + # 创建时间 create_time = DateTimeField(default=datetime.now) @@ -73,6 +76,14 @@ def update_time_label(self): def content_dict(self): return json.loads(self.content) + @property + def http_url(self): + return self.content_dict.get('url') + + @property + def http_timeout(self): + return self.content_dict.get('timeout') + def to_dict(self): data = model_to_dict( model=self, @@ -81,8 +92,31 @@ def to_dict(self): 'update_time_label', 'content_dict', 'monitor_id', + 'http_url', + 'http_timeout', ] ) data['content'] = data.pop('content_dict') return data + + +# 数据导入导出字段关系 +FIELD_MAPPING = [ + { + 'name': '名称', + 'field': 'title', + }, + { + 'name': '请求URL', + 'field': 'http_url', + }, + { + 'name': '请求超时（秒）', + 'field': 'http_timeout', + }, + { + 'name': '检测频率（分钟）', + 'field': 'interval', + }, +] diff --git a/domain_admin/router/api_map.py b/domain_admin/router/api_map.py index b954f38b39..b28239e7af 100644 --- a/domain_admin/router/api_map.py +++ b/domain_admin/router/api_map.py @@ -41,7 +41,6 @@ "/api/updateAllDomainCertInfo": domain_api.update_all_domain_cert_info, "/api/getDomainGroupFilter": domain_api.get_domain_group_filter, - # "/api/updateDomainSetting": domain_api.update_domain_setting, "/api/updateAllDomainCertInfoOfUser": domain_api.update_all_domain_cert_info_of_user, @@ -83,6 +82,7 @@ '/api/updateCronConfig': system_api.update_cron_config, '/api/getSystemVersion': system_api.get_system_version, '/api/getSystemData': system_api.get_system_data, + '/api/getMonitorTaskNextRunTime': system_api.get_monitor_task_next_run_time, # 用户管理 (管理员权限) '/api/getUserList': user_api.get_user_list, @@ -189,6 +189,8 @@ '/api/removeMonitorById': monitor_api.remove_monitor_by_id, '/api/getMonitorById': monitor_api.get_monitor_by_id, '/api/getMonitorList': monitor_api.get_monitor_list, + '/api/exportMonitorFile': monitor_api.export_monitor_file, + '/api/importMonitorFromFile': monitor_api.import_monitor_from_file, # http监控日志 '/api/getLogMonitorList': log_monitor_api.get_log_monitor_list, @@ -196,4 +198,5 @@ '/api/clearAllLogMonitor': log_monitor_api.clear_all_log_monitor, '/api/getTagList': tag_api.get_tag_list, + } diff --git a/domain_admin/service/monitor_service.py b/domain_admin/service/monitor_service.py index 5c84165531..943a1cba24 100644 --- a/domain_admin/service/monitor_service.py +++ b/domain_admin/service/monitor_service.py @@ -4,21 +4,24 @@ @Date : 2024-01-28 @Author : Peng Shiyu """ +import json from datetime import datetime, timedelta from functools import wraps import requests import six -from peewee import fn +from peewee import fn, chunked from domain_admin.config import USER_AGENT from domain_admin.enums.monitor_status_enum import MonitorStatusEnum from domain_admin.enums.monitor_type_enum import MonitorTypeEnum +from domain_admin.model import monitor_model from domain_admin.model.base_model import db from domain_admin.model.log_monitor_model import LogMonitorModel from domain_admin.model.monitor_model import MonitorModel -from domain_admin.service import notify_service -from domain_admin.utils import datetime_util +from domain_admin.service import notify_service, file_service, async_task_service +from domain_admin.service.scheduler_service import scheduler_main +from domain_admin.utils import datetime_util, file_util def monitor_log_decorator(func): @@ -79,7 +82,7 @@ def wrapper(monitor_row, *args, **kwargs): # 继续抛出异常 if error: - notify_service.notify_user_about_monitor_exception(monitor_row, error) + handle_monitor_exception(monitor_row, error) raise error else: return result @@ -87,6 +90,26 @@ def wrapper(monitor_row, *args, **kwargs): return wrapper +def handle_monitor_exception(monitor_row, error): + if monitor_row.allow_error_count > 0: + # 检查连续失败次数是否大于最大允许失败次数，增加容错 + rows = LogMonitorModel.select().where( + LogMonitorModel.monitor_id == monitor_row.id, + ).order_by( + LogMonitorModel.id.desc() + ).limit( + monitor_row.allow_error_count + ) + + error_count = len([row for row in rows if row.status == MonitorStatusEnum.ERROR]) + + if error_count <= monitor_row.allow_error_count: + return + + # 发送异常通知 + notify_service.notify_user_about_monitor_exception(monitor_row, error) + + def run_monitor_warp(monitor_row): error = None @@ -102,6 +125,7 @@ def run_monitor_warp(monitor_row): MonitorModel.update( next_run_time=next_run_time, status=MonitorStatusEnum.ERROR if error else MonitorStatusEnum.SUCCESS, + version=MonitorModel.version + 1 ).where( MonitorModel.id == monitor_row.id ).execute() @@ -110,8 +134,8 @@ def run_monitor_warp(monitor_row): return next_run_time -@monitor_notify_decorator @monitor_log_decorator +@monitor_notify_decorator def run_monitor(monitor_row): """ :param monitor_row: @@ -182,3 +206,64 @@ def load_monitor_log_count(lst): for row in lst: row['log_count'] = monitor_groups_map.get(row['id']) + + +def get_monitor_list_query(**params): + query = MonitorModel.select() + + status = params.get("status") + if isinstance(status, int): + query = query.where(MonitorModel.status == status) + + keyword = params.get('keyword') + if keyword: + query = query.where(MonitorModel.title.contains(keyword)) + + user_id = params.get('user_id') + if user_id: + query = query.where(MonitorModel.user_id == user_id) + + return query + + +def export_monitor_to_file(rows, ext): + """ + 导出域名到文件 + :param rows: + :return: + """ + + filename = datetime.now().strftime("monitor_%Y%m%d%H%M%S") + '.' + ext + temp_filename = file_service.resolve_temp_file(filename) + + lst = file_util.convert_to_export(rows, monitor_model.FIELD_MAPPING) + + file_util.write_data_to_file(temp_filename, lst) + + return filename + + +def import_monitor_from_file(filename, user_id): + rows = file_util.read_data_from_file(filename) + + lst = file_util.convert_to_import(rows, monitor_model.FIELD_MAPPING) + print(lst) + + lst = [ + { + 'title': item['title'], + 'monitor_type': MonitorTypeEnum.HTTP, + 'interval': int(item.get('interval') or '60'), + 'content': json.dumps({ + 'url': item['http_url'], + 'method': 'GET', + 'timeout': int(item.get('http_timeout') or '3'), + }), + 'user_id': user_id, + } for item in lst + ] + + # fix: peewee.OperationalError: too many SQL variables + # https://github.com/mouday/domain-admin/issues/63 + for batch in chunked(lst, 500): + MonitorModel.insert_many(batch).on_conflict_ignore().execute() diff --git a/domain_admin/service/scheduler_service/scheduler_main.py b/domain_admin/service/scheduler_service/scheduler_main.py index 1727704109..1962c68764 100644 --- a/domain_admin/service/scheduler_service/scheduler_main.py +++ b/domain_admin/service/scheduler_service/scheduler_main.py @@ -72,6 +72,7 @@ def run_one_monitor_task(monitor_row): def update_monitor_task(next_run_time): monitor_job = scheduler.get_job(scheduler_config.MONITOR_TASK_JOB_ID) + # 如果下次运行时间比唤起时间早，就替换唤起时间 if monitor_job and datetime_util.is_greater_than(next_run_time, monitor_job.next_run_time): return @@ -83,6 +84,13 @@ def update_monitor_task(next_run_time): ) +def get_monitor_task_next_run_time(): + monitor_task = scheduler.get_job(job_id=scheduler_config.MONITOR_TASK_JOB_ID) + + if monitor_task: + return monitor_task.next_run_time + + def run_monitor_task(): next_run_time = monitor_service.run_monitor_task() diff --git a/domain_admin/utils/datetime_util.py b/domain_admin/utils/datetime_util.py index 6f57e3d9cc..97d7d1d20f 100644 --- a/domain_admin/utils/datetime_util.py +++ b/domain_admin/utils/datetime_util.py @@ -112,25 +112,28 @@ def microsecond_for_human(value): lst = [] - if value > DAY: + if value >= DAY: days, value = divmod(value, DAY) lst.append(str(days) + 'd') - if value > HOUR: + if value >= HOUR: hours, value = divmod(value, HOUR) lst.append(str(hours) + 'h') - if value > MINUTE: + if value >= MINUTE: minutes, value = divmod(value, MINUTE) lst.append(str(minutes) + 'm') - if value > SECOND: + if value >= SECOND: seconds, value = divmod(value, SECOND) lst.append(str(seconds) + 's') if value > 0: lst.append(str(value) + 'ms') + if len(lst) == 0: + lst.append('0ms') + return ' '.join(lst)