文件导入域名自动更新子域证书列表

CHANGELOG.md

@@ -4,7 +4,7 @@
     - 更新备注
     - 替换备案查询接口
     - 优化时间统计
-    - 完善问度娘
+    - 完善问文档
 
 - v1.6.1(2024-01-28)
     - 添加网站监控

domain_admin/api/domain_info_api.py

@@ -65,7 +65,7 @@ def add_domain_info():
 
     # 异步提交
     if is_auto_subdomain:
-        domain_service.auto_import_from_domain(
+        domain_service.auto_import_from_domain_async(
             root_domain=domain,
             group_id=group_id,
             user_id=current_user_id
@@ -139,7 +139,7 @@ def update_domain_info_by_id():
         domain_info_service.update_domain_info_row(domain_info_row)
 
     if is_auto_subdomain:
-        domain_service.auto_import_from_domain(
+        domain_service.auto_import_from_domain_async(
             root_domain=domain,
             group_id=group_id,
             user_id=current_user_id
@@ -315,7 +315,7 @@ def update_domain_row_icp():
 def import_domain_info_from_file():
     """
     从文件导入域名
-    支持 txt 和 csv格式
+    支持格式: txt、xlsx、csv
     :return:
     """
     current_user_id = g.user_id
@@ -328,8 +328,7 @@ def import_domain_info_from_file():
     domain_info_service.add_domain_from_file(filename, current_user_id)
 
     # 异步查询
-    domain_info_service.update_all_domain_info_of_user(current_user_id)
-
+    domain_info_service.handle_auto_import_domain_info(current_user_id)
     # async_task_service.submit_task(fn=domain_info_service.update_all_domain_info_of_user, user_id=current_user_id)
 
 

domain_admin/enums/version_enum.py

@@ -155,4 +155,11 @@ class VersionEnum(object):
     Version_1533 = '1.5.33'
     Version_1534 = '1.5.34'
     Version_1535 = '1.5.35'
+    Version_1536 = '1.5.36'
+    Version_1537 = '1.5.37'
+    Version_1538 = '1.5.38'
+    Version_1539 = '1.5.39'
     Version_160 = '1.6.0'
+    Version_161 = '1.6.1'
+    Version_162 = '1.6.2'
+    Version_163 = '1.6.3'

domain_admin/migrate/history/migrate_162_to_163.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+"""
+@File    : migrate_162_to_163.py
+@Date    : 2023-09-17
+
+cmd:
+$ python domain_admin/migrate/migrate_162_to_163.py
+"""
+from __future__ import print_function, unicode_literals, absolute_import, division
+
+from domain_admin.migrate import migrate_common
+from domain_admin.model.base_model import db
+from domain_admin.model.domain_info_model import DomainInfoModel
+from domain_admin.model.domain_model import DomainModel
+from domain_admin.model.notify_model import NotifyModel
+
+
+def execute_migrate():
+    """
+    版本升级 1.6.2 => 1.6.3
+    :return:
+    """
+    migrator = migrate_common.get_migrator(db)
+
+    migrate_rows = [
+        migrator.add_column(
+            DomainInfoModel._meta.table_name,
+            DomainInfoModel.version.name,
+            DomainInfoModel.version
+        ),
+
+        migrator.add_column(
+            DomainModel._meta.table_name,
+            DomainModel.version.name,
+            DomainModel.version
+        ),
+
+    ]
+
+    migrate_common.try_execute_migrate(migrate_rows)

domain_admin/migrate/migrate_config.py

@@ -24,7 +24,7 @@
     migrate_1520_to_1521,
     migrate_154_to_155,
     migrate_106_to_110,
-    migrate_1533_to_1534)
+    migrate_1533_to_1534, migrate_162_to_163)
 
 # local_versions 本地版本
 # migrate_func 升级函数
@@ -306,6 +306,24 @@
             VersionEnum.Version_1533,
         ],
         'migrate_func': migrate_1533_to_1534.execute_migrate,
-        'update_version': VersionEnum.Version_1524
+        'update_version': VersionEnum.Version_1534
+    },
+    # 2023-11-28
+    # 1.5.33 => 1.5.34
+    {
+        'local_versions': [
+            VersionEnum.Version_1534,
+            VersionEnum.Version_1535,
+            VersionEnum.Version_1536,
+            VersionEnum.Version_1537,
+            VersionEnum.Version_1538,
+            VersionEnum.Version_1539,
+            VersionEnum.Version_160,
+            VersionEnum.Version_161,
+            VersionEnum.Version_162,
+        ],
+        'migrate_func': migrate_162_to_163.execute_migrate,
+        'update_version': VersionEnum.Version_163
     },
+
 ]

domain_admin/model/domain_info_model.py

@@ -62,6 +62,9 @@ class DomainInfoModel(BaseModel):
     # 标签list @since 1.5.2
     tags_raw = TextField(default=None, null=True)
 
+    # 数据版本号 @since 1.6.3
+    version = IntegerField(default=0, null=False)
+
     # 创建时间
     create_time = DateTimeField(default=datetime.now)
 

domain_admin/model/domain_model.py

@@ -64,6 +64,9 @@ class DomainModel(BaseModel):
     # SSL有效期总天数，仅用于排序
     total_days = IntegerField(default=0, null=False)
 
+    # 数据版本号 @since 1.6.3
+    version = IntegerField(default=0, null=False)
+
     # 创建时间
     create_time = DateTimeField(default=datetime.now)
 

domain_admin/service/domain_info_service.py

@@ -19,7 +19,7 @@
 from domain_admin.model.domain_info_model import DomainInfoModel
 from domain_admin.model.group_model import GroupModel
 from domain_admin.model.group_user_model import GroupUserModel
-from domain_admin.service import render_service, file_service, group_service, async_task_service
+from domain_admin.service import render_service, file_service, group_service, async_task_service, domain_service
 from domain_admin.utils import whois_util, datetime_util, domain_util, icp_util, file_util
 
 
@@ -104,6 +104,8 @@ def update_domain_info_row(row):
     :param row: DomainInfoModel
     :return: [str, None]
     """
+    logger.info("domain: %s", row.domain)
+
     domain_whois = None
 
     try:
@@ -130,7 +132,8 @@ def update_domain_info_row(row):
         domain_expire_days=update_row.real_domain_expire_days,
         domain_registrar=update_row.domain_registrar,
         domain_registrar_url=update_row.domain_registrar_url,
-        update_time=datetime_util.get_datetime()
+        update_time=datetime_util.get_datetime(),
+        version=DomainInfoModel.version + 1
     ).where(
         DomainInfoModel.id == row.id
     ).execute()
@@ -182,6 +185,8 @@ def update_domain_row_icp(row):
     :param row:
     :return:
     """
+    logger.info("domain: %s", row.domain)
+
     res = None
 
     try:
@@ -302,7 +307,7 @@ def get_domain_info_query(keyword, group_ids, domain_expire_days, role, user_id)
     if keyword:
         query = query.where(
             (DomainInfoModel.domain.contains(keyword))
-            |(DomainInfoModel.tags_raw.contains(keyword))
+            | (DomainInfoModel.tags_raw.contains(keyword))
         )
 
     if group_ids:
@@ -379,4 +384,34 @@ def get_ordering(order_prop='expire_days', order_type='ascending'):
 
     ordering.append(DomainInfoModel.id.desc())
 
-    return ordering
+    return ordering
+
+
+@async_task_service.async_task_decorator("从文件导入域名")
+def handle_auto_import_domain_info(current_user_id):
+    rows = DomainInfoModel.select().where(
+        DomainInfoModel.user_id == current_user_id,
+        DomainInfoModel.is_auto_update == True,
+        DomainInfoModel.version == 0
+    )
+
+    lst = list(rows)
+
+    # 注册信息
+    for row in lst:
+        update_domain_info_row(row)
+
+    # icp信息
+    for row in lst:
+        update_domain_row_icp(row)
+
+    # 导入子域名
+    for row in lst:
+        try:
+            domain_service.auto_import_from_domain(
+                root_domain=row.domain,
+                group_id=row.group_id,
+                user_id=current_user_id
+            )
+        except Exception as e:
+            logger.error(traceback.format_exc())

domain_admin/service/domain_service.py

@@ -327,6 +327,10 @@ def check_permission_and_get_row(domain_id, user_id):
 
 
 @async_task_service.async_task_decorator("自动导入子域名证书")
+def auto_import_from_domain_async(root_domain, group_id=0, user_id=0):
+    return auto_import_from_domain(root_domain=root_domain, group_id=group_id, user_id=user_id)
+
+
 def auto_import_from_domain(root_domain, group_id=0, user_id=0):
     """
     自动导入顶级域名下包含的子域名到证书列表
@@ -335,6 +339,8 @@ def auto_import_from_domain(root_domain, group_id=0, user_id=0):
     :param user_id: int
     :return:
     """
+    logger.info("domain: %s", root_domain)
+
     lst = crtsh_api.search(root_domain)
 
     domain_set = list(set([domain['common_name'] for domain in lst]))
@@ -355,7 +361,8 @@ def auto_import_from_domain(root_domain, group_id=0, user_id=0):
 
     # 更新插入的证书
     rows = DomainModel.select().where(
-        DomainModel.domain.in_(domain_set)
+        DomainModel.version == 0,
+        DomainModel.user_id == user_id,
     )
 
     for row in rows:

domain_admin/utils/open_api/crtsh_api.py

@@ -39,5 +39,6 @@ def search(domain):
 
 
 if __name__ == '__main__':
-    lst = search('bilibili.com')
+    # lst = search('bilibili.com')
+    lst = search('baidu.com')
     print([row['common_name'] for row in lst])

tests/model/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+"""
+@File    : __init__.py.py
+@Date    : 2024-01-30
+@Author  : Peng Shiyu
+"""

tests/model/domain_info_model_test.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+"""
+@File    : domain_info_model_test.py
+@Date    : 2024-01-30
+@Author  : Peng Shiyu
+"""
+import unittest
+
+from peewee import SQL
+
+from domain_admin.model.domain_info_model import DomainInfoModel
+
+
+class MonitorServiceTest(unittest.TestCase):
+    def test_add(self):
+        DomainInfoModel.update(
+            version=DomainInfoModel.version + 1
+        ).where(
+            DomainInfoModel.id == 1
+        ).execute()
+
+    # ('UPDATE `tb_domain_info` SET `version` = (`tb_domain_info`.`version` + %s) WHERE (`tb_domain_info`.`id` = %s)', [1, 1])
+
+    def test_order(self):
+        rows = list(DomainInfoModel.select(
+            DomainInfoModel.id, DomainInfoModel.create_time
+        ).order_by(SQL("`create_time` desc")))
+
+        # ('SELECT `t1`.`id`, `t1`.`create_time` FROM `tb_domain_info` AS `t1` ORDER BY `create_time` desc', [])