diff --git a/domain_admin/api/domain_api.py b/domain_admin/api/domain_api.py index 6ef9588825..e98004e5b9 100644 --- a/domain_admin/api/domain_api.py +++ b/domain_admin/api/domain_api.py @@ -371,6 +371,7 @@ def export_domain_file(): group_ids = request.json.get('group_ids') expire_days = request.json.get('expire_days') role = request.json.get('role') + ext = request.json.get('ext', 'csv') order_prop = request.json.get('order_prop') or 'expire_days' order_type = request.json.get('order_type') or 'ascending' @@ -408,7 +409,7 @@ def export_domain_file(): group_service.load_group_name(lst) - filename = domain_service.export_domain_to_file(lst) + filename = domain_service.export_domain_to_file(rows=lst, ext=ext) return { 'name': filename, diff --git a/domain_admin/api/domain_info_api.py b/domain_admin/api/domain_info_api.py index bb4a88bb5d..753aa246a5 100644 --- a/domain_admin/api/domain_info_api.py +++ b/domain_admin/api/domain_info_api.py @@ -345,6 +345,7 @@ def export_domain_info_file(): group_ids = request.json.get('group_ids') domain_expire_days = request.json.get('domain_expire_days') role = request.json.get('role') + ext = request.json.get('ext', 'csv') order_prop = request.json.get('order_prop') or 'domain_expire_days' order_type = request.json.get('order_type') or 'ascending' @@ -358,7 +359,7 @@ def export_domain_info_file(): } # 列表数据 - query = domain_info_service.get_domain_inf_query(**params) + query = domain_info_service.get_domain_info_query(**params) ordering = domain_info_service.get_ordering(order_prop=order_prop, order_type=order_type) @@ -379,7 +380,7 @@ def export_domain_info_file(): # 分组名 group_service.load_group_name(lst) - filename = domain_info_service.export_domain_to_file(lst) + filename = domain_info_service.export_domain_to_file(ext=ext, rows=lst) return { 'name': filename, @@ -429,7 +430,7 @@ def get_domain_info_list(): } # 列表数据 - query = domain_info_service.get_domain_inf_query(**params) + query = domain_info_service.get_domain_info_query(**params) total = query.count() diff --git a/domain_admin/model/domain_info_model.py b/domain_admin/model/domain_info_model.py index 290703da62..72dc273327 100644 --- a/domain_admin/model/domain_info_model.py +++ b/domain_admin/model/domain_info_model.py @@ -117,3 +117,44 @@ def tags(self, value): def tags_str(self): if self.tags: return '、'.join(self.tags) + + +# 数据导入导出字段关系 +FIELD_MAPPING = [ + { + 'name': '域名', + 'field': 'domain', + }, + { + 'name': '注册时间', + 'field': 'domain_start_date', + }, + { + 'name': '到期时间', + 'field': 'domain_expire_date', + }, + { + 'name': '剩余天数', + 'field': 'real_domain_expire_days', + }, + { + 'name': '分组', + 'field': 'group_name', + }, + { + 'name': '标签', + 'field': 'tags_str', + }, + { + 'name': '主办单位名称', + 'field': 'icp_company', + }, + { + 'name': 'ICP备案/许可证号', + 'field': 'icp_licence', + }, + { + 'name': '备注', + 'field': 'comment', + }, +] diff --git a/domain_admin/model/domain_model.py b/domain_admin/model/domain_model.py index 0923025543..09481f2145 100644 --- a/domain_admin/model/domain_model.py +++ b/domain_admin/model/domain_model.py @@ -139,3 +139,36 @@ def expire_status(self): return None else: return False + + +# 数据导入导出字段关系 +FIELD_MAPPING = [ + { + 'name': '域名', + 'field': 'domain', + }, + { + 'name': '端口', + 'field': 'port', + }, + { + 'name': '证书颁发时间', + 'field': 'start_date', + }, + { + 'name': '证书过期时间', + 'field': 'expire_date', + }, + { + 'name': '证书天数', + 'field': 'real_time_expire_days', + }, + { + 'name': '分组', + 'field': 'group_name', + }, + { + 'name': '备注', + 'field': 'alias', + }, +] diff --git a/domain_admin/service/domain_info_service.py b/domain_admin/service/domain_info_service.py index 260e43edfd..5ca7f9e19e 100644 --- a/domain_admin/service/domain_info_service.py +++ b/domain_admin/service/domain_info_service.py @@ -11,6 +11,7 @@ from datetime import datetime, timedelta import random +from domain_admin.model import domain_info_model from peewee import chunked from domain_admin.enums.role_enum import RoleEnum @@ -19,7 +20,7 @@ from domain_admin.model.group_model import GroupModel from domain_admin.model.group_user_model import GroupUserModel from domain_admin.service import render_service, file_service, group_service, async_task_service -from domain_admin.utils import whois_util, datetime_util, domain_util, icp_util +from domain_admin.utils import whois_util, datetime_util, domain_util, icp_util, file_util def add_domain_info( @@ -235,68 +236,52 @@ def add_domain_from_file(filename, user_id): """ # logger.info('user_id: %s, filename: %s', user_id, filename) - lst = list(domain_util.parse_domain_from_file(filename)) + lst = list(domain_util.parse_domain_from_file(filename, domain_info_model.FIELD_MAPPING)) # 导入分组 - group_name_list = [item.group_name for item in lst] - group_map = group_service.get_or_create_group_map(group_name_list, user_id) + group_name_list = [item.get('group_name') for item in lst if item.get('group_name')] + if group_name_list: + group_map = group_service.get_or_create_group_map(group_name_list, user_id) + else: + group_map = {} lst = [ { - 'domain': item.root_domain, - 'comment': item.alias, + 'domain': item['domain'], + 'comment': item.get('comment'), + 'group_id': group_map.get(item.get('group_name'), 0), + 'tags_raw': json.dumps(item.get('tags'), ensure_ascii=False), 'user_id': user_id, - 'group_id': group_map.get(item.group_name, 0), - 'tags_raw': json.dumps(item.tags, ensure_ascii=False) - } for item in lst if item.root_domain + } for item in lst if item.get('root_domain') ] for batch in chunked(lst, 500): DomainInfoModel.insert_many(batch).on_conflict_ignore().execute() -def export_domain_to_file(rows): +def export_domain_to_file(rows, ext): """ 导出域名到文件 + :param ext: 导出格式 :param rows: :return: """ - # 域名数据 - # rows = DomainInfoModel.select().where( - # DomainInfoModel.user_id == user_id - # ).order_by( - # DomainInfoModel.domain_expire_time.asc(), - # DomainInfoModel.id.desc(), - # ) - # - # # 分组数据 - # group_rows = GroupModel.select( - # GroupModel.id, - # GroupModel.name, - # ).where( - # GroupModel.user_id == user_id - # ) - # - # group_map = {row.id: row.name for row in group_rows} - # - # lst = [] - # for row in list(rows): - # row.group_name = group_map.get(row.group_id, '') - # lst.append(row) - - content = render_service.render_template('domain-export.csv', {'list': rows}) - - filename = datetime.now().strftime("domain_%Y%m%d%H%M%S") + '.csv' + filename = datetime.now().strftime("domain_%Y%m%d%H%M%S") + '.' + ext temp_filename = file_service.resolve_temp_file(filename) - # print(temp_filename) - with io.open(temp_filename, 'w', encoding='utf-8') as f: - f.write(content) + + if ext == 'txt': + lst = [row['domain'] for row in rows] + else: + lst = file_util.convert_to_export(rows, domain_info_model.FIELD_MAPPING) + + # content = render_service.render_template('domain-export.csv', {'list': rows}) + file_util.write_data_to_file(temp_filename, lst) return filename -def get_domain_inf_query(keyword, group_ids, domain_expire_days, role, user_id): +def get_domain_info_query(keyword, group_ids, domain_expire_days, role, user_id): user_group_ids = None if role == RoleEnum.ADMIN: diff --git a/domain_admin/service/domain_service.py b/domain_admin/service/domain_service.py index 6ad5a4916d..6b0b88b00b 100644 --- a/domain_admin/service/domain_service.py +++ b/domain_admin/service/domain_service.py @@ -13,6 +13,7 @@ from domain_admin.enums.role_enum import RoleEnum from domain_admin.log import logger +from domain_admin.model import domain_model from domain_admin.model.address_model import AddressModel from domain_admin.model.domain_info_model import DomainInfoModel from domain_admin.model.domain_model import DomainModel @@ -21,7 +22,7 @@ from domain_admin.model.user_model import UserModel from domain_admin.service import file_service, async_task_service from domain_admin.service import render_service, group_service -from domain_admin.utils import datetime_util, cert_util +from domain_admin.utils import datetime_util, cert_util, file_util from domain_admin.utils import domain_util from domain_admin.utils.cert_util import cert_socket_v2, cert_openssl_v2 from domain_admin.utils.flask_ext.app_exception import ForbiddenAppException @@ -364,20 +365,23 @@ def auto_import_from_domain(root_domain, group_id=0, user_id=0): def add_domain_from_file(filename, user_id): logger.info('user_id: %s, filename: %s', user_id, filename) - lst = list(domain_util.parse_domain_from_file(filename)) + lst = list(domain_util.parse_domain_from_file(filename, domain_model.FIELD_MAPPING)) # 导入分组 - group_name_list = [item.group_name for item in lst] - group_map = group_service.get_or_create_group_map(group_name_list, user_id) + group_name_list = [item.get('group_name') for item in lst if item.get('group_name')] + if group_name_list: + group_map = group_service.get_or_create_group_map(group_name_list, user_id) + else: + group_map = {} lst = [ { - 'domain': item.domain, - 'root_domain': item.root_domain, - 'port': item.port, - 'alias': item.alias, + 'domain': item['domain'], + 'root_domain': domain_util.get_root_domain(item['domain']), + 'port': item.get('port'), + 'alias': item.get('alias', ''), 'user_id': user_id, - 'group_id': group_map.get(item.group_name, 0), + 'group_id': group_map.get(item.get('group_name'), 0), } for item in lst ] @@ -387,40 +391,29 @@ def add_domain_from_file(filename, user_id): DomainModel.insert_many(batch).on_conflict_ignore().execute() -def export_domain_to_file(rows): +def export_domain_to_file(rows, ext): """ 导出域名到文件 :param rows: :return: """ - # # 域名数据 - # rows = DomainModel.select().where( - # DomainModel.user_id == user_id - # ).order_by( - # DomainModel.expire_days.asc(), - # DomainModel.id.desc(), - # ) - # - # # 分组数据 - # group_rows = GroupModel.select().where( - # GroupModel.user_id == user_id - # ) - # - # group_map = {row.id: row.name for row in group_rows} - # - # lst = [] - # for row in list(rows): - # row.group_name = group_map.get(row.group_id, '') - # lst.append(row) - - content = render_service.render_template('cert-export.csv', {'list': rows}) - - filename = datetime.now().strftime("cert_%Y%m%d%H%M%S") + '.csv' + # content = render_service.render_template('cert-export.csv', {'list': rows}) + + filename = datetime.now().strftime("cert_%Y%m%d%H%M%S") + '.' + ext temp_filename = file_service.resolve_temp_file(filename) - # print(temp_filename) - with io.open(temp_filename, 'w', encoding='utf-8') as f: - f.write(content) + + if ext == 'txt': + lst = [row['domain'] for row in rows] + else: + lst = file_util.convert_to_export(rows, domain_model.FIELD_MAPPING) + + file_util.write_data_to_file(temp_filename, lst) + + # temp_filename = file_service.resolve_temp_file(filename) + # # print(temp_filename) + # with io.open(temp_filename, 'w', encoding='utf-8') as f: + # f.write(content) return filename @@ -477,7 +470,6 @@ def load_address_count(lst): def get_domain_list_query(keyword, group_id, group_ids, expire_days, user_id, role): - user_group_ids = None if role == RoleEnum.ADMIN: diff --git a/domain_admin/service/file_service.py b/domain_admin/service/file_service.py index 18acb92ed0..2a618e9a1a 100644 --- a/domain_admin/service/file_service.py +++ b/domain_admin/service/file_service.py @@ -26,7 +26,7 @@ def get_temp_filename(ext): def save_temp_file(update_file): """保存上传的文件""" - ext = update_file.filename.split('.')[-1] + ext = file_util.get_filename_ext(update_file.filename) filename = get_temp_filename(ext) update_file.save(filename) return filename diff --git a/domain_admin/utils/csv_util.py b/domain_admin/utils/csv_util.py new file mode 100644 index 0000000000..d065201111 --- /dev/null +++ b/domain_admin/utils/csv_util.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +""" +@File : csv_util.py +@Date : 2023-10-14 +""" +import io +import csv + + +def read_csv(filename): + """ + 读取csv文件 适合完整导入 + :param filename: str + :return: iterator + """ + with io.open(filename, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + for row in reader: + yield row + + +def write_csv(filename, rows): + """ + 读取csv文件 适合完整导入 + :param rows: list + :param filename: str + :return: + """ + if len(rows) == 0: + return + + with open(filename, "w") as f: + writer = csv.DictWriter(f, rows[0].keys()) + writer.writeheader() + writer.writerows(rows) + + +if __name__ == '__main__': + # write_csv('./demo.csv', [ + # {'name': 'Tom', 'age': 23}, + # {'name': 'Jack', 'age': 24}, + # ]) + + for row in read_csv('./demo.csv'): + print(row) diff --git a/domain_admin/utils/domain_util.py b/domain_admin/utils/domain_util.py index aafe571221..d5805c8a74 100644 --- a/domain_admin/utils/domain_util.py +++ b/domain_admin/utils/domain_util.py @@ -12,6 +12,8 @@ from tldextract.tldextract import ExtractResult from domain_admin.log import logger +from domain_admin.model import domain_info_model +from domain_admin.service import group_service from domain_admin.utils import file_util from domain_admin.utils.cert_util import cert_consts @@ -37,14 +39,11 @@ class ParsedDomain(object): def parse_domain(domain): """ 解析域名信息 - :param domain: - :return: + :param domain: str + :return: str / None """ - # print(domain) - ret = re.match('((http(s)?:)?//)?(?P[\\w\\._:-]+)/?.*?', domain) if ret: - # print(ret.groups()) return ret.groupdict().get("domain") else: return None @@ -126,18 +125,47 @@ def parse_domain_from_txt_file(filename): yield item -def parse_domain_from_file(filename): +def parse_domain_from_file(filename, field_mapping): """ 解析域名文件的工厂方法 :param filename: :return: ParsedDomain """ file_type = file_util.get_filename_ext(filename) + rows = file_util.read_data_from_file(filename) - if file_type == 'csv': - return parse_domain_from_csv_file(filename) + if file_type == 'txt': + rows = [ + {'domain': parse_domain(row.strip())} + for row in rows + ] else: - return parse_domain_from_txt_file(filename) + rows = file_util.convert_to_import(rows, field_mapping) + + for item in rows: + domain = parse_domain(item['domain']) + + if ':' in domain: + domain, port = domain.split(":") + else: + # SSL默认端口 + port = cert_consts.SSL_DEFAULT_PORT + + item['domain'] = domain + item['root_domain'] = get_root_domain(domain) + item.setdefault('port', port) + + # 标签 + tags = item.get('tags_str') or None + if tags: + item['tags'] = [tag.strip() for tag in tags.split("、") if tag and tag.strip() and tag.strip() != '-'] + + return rows + + # if file_type == 'csv': + # return parse_domain_from_csv_file(filename) + # else: + # return parse_domain_from_txt_file(filename) def extract_domain(domain): diff --git a/domain_admin/utils/excel_util.py b/domain_admin/utils/excel_util.py new file mode 100644 index 0000000000..d1788dbdc5 --- /dev/null +++ b/domain_admin/utils/excel_util.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +""" +@File : excel_util.py +@Date : 2023-11-16 +""" +import re + +from openpyxl import Workbook +from openpyxl.reader.excel import load_workbook +from openpyxl.utils import get_column_letter + + +def read_excel(filename): + """ + 读取excel文件为python对象 + :param filename: + :return: iterator + """ + book = load_workbook(filename) + worksheet = book.worksheets[0] + + titles = [] + row_num = 0 + + for row in worksheet.rows: + row_num += 1 + + if row_num == 1: + # 表头 + titles = [cell.value.strip() for cell in row] + else: + # 内容 + yield dict(zip(titles, [cell.value for cell in row])) + + book.close() + + +def write_excel(filename, rows): + """ + 将列表写入到文件 + :param filename: + :param rows: list + :return: + """ + workbook = Workbook() + worksheet = workbook.active + + + # 表头 + if len(rows) > 0: + for i, key in enumerate(rows[0].keys()): + worksheet.cell(row=1, column=i + 1, value=key) + + # 内容 + for x, row in enumerate(rows): + for y, value in enumerate(row.values()): + worksheet.cell(row=x + 2, column=y + 1, value=value) + + # 调整列宽 + # 参考:https://blog.csdn.net/gongzairen/article/details/130819231 + width = 3 # 手动加宽的数值 + # 单元格列宽处理 + dims = {} + for row in worksheet.rows: + for cell in row: + if cell.value: + cell_len = 0.7 * len(re.findall('([\u4e00-\u9fa5])', str(cell.value))) + len(str(cell.value)) + dims[cell.column] = max((dims.get(cell.column, 0), cell_len)) + + for col, value in dims.items(): + worksheet.column_dimensions[get_column_letter(col)].width = value + width + + workbook.save(filename) + workbook.close() + + +if __name__ == '__main__': + write_excel('./demo.xlsx', [ + {'name': 'Tom', 'age': 23}, + {'name': 'Jack', 'age': 24}, + ]) diff --git a/domain_admin/utils/file_util.py b/domain_admin/utils/file_util.py index c1751f2a93..d640385b87 100644 --- a/domain_admin/utils/file_util.py +++ b/domain_admin/utils/file_util.py @@ -2,6 +2,22 @@ from __future__ import print_function, unicode_literals, absolute_import, division import uuid +from domain_admin.utils import excel_util, csv_util, txt_util + +# 文件读取配置 +read_config = { + 'xlsx': excel_util.read_excel, + 'csv': csv_util.read_csv, + 'txt': txt_util.read_txt, +} + +# 文件写入配置 +write_config = { + 'xlsx': excel_util.write_excel, + 'csv': csv_util.write_csv, + 'txt': txt_util.write_txt, +} + def get_random_filename(ext): """ @@ -19,3 +35,45 @@ def get_filename_ext(filename): :return: """ return filename.split('.')[-1] + + +def read_data_from_file(filename): + file_type = get_filename_ext(filename) + + if file_type in read_config: + return read_config[file_type](filename) + else: + raise Exception('not support .{}'.format(file_type)) + + +def write_data_to_file(filename, rows): + file_type = get_filename_ext(filename) + + if file_type in write_config: + return write_config[file_type](filename, rows) + else: + raise Exception('not support .{}'.format(file_type)) + + +def convert_to_export(rows, field_mapping): + lst = [] + for row in rows: + data = {} + for item in field_mapping: + data[item['name']] = row.get(item['field'], '') + + lst.append(data) + + return lst + + +def convert_to_import(rows, field_mapping): + lst = [] + for row in rows: + data = {} + for item in field_mapping: + data[item['field']] = row.get(item['name'], '') + + lst.append(data) + + return lst diff --git a/domain_admin/utils/txt_util.py b/domain_admin/utils/txt_util.py new file mode 100644 index 0000000000..7df29ca571 --- /dev/null +++ b/domain_admin/utils/txt_util.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +@File : txt_util.py +@Date : 2023-11-30 +""" +import os + + +def read_txt(filename): + """ + 读取文本 + :param filename: + :return: + """ + with open(filename, 'r') as f: + for line in f.readlines(): + yield line.strip() + + +def write_txt(filename, rows): + """ + 写入到文本 + :param filename: + :param rows: + :return: + """ + with open(filename, 'w') as f: + for row in rows: + f.write(row + os.linesep) + + +if __name__ == '__main__': + write_txt('demo.txt', ['1', '2']) + + for row in read_txt('demo.txt'): + print(row) diff --git a/http/domain-info.http b/http/domain-info.http index e9a71f15f5..91d95ed715 100644 --- a/http/domain-info.http +++ b/http/domain-info.http @@ -130,3 +130,12 @@ X-TOKEN: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyX2lkIjoxLCJleHAiOjE2ODYyMD } ### +POST {{baseUrl}}/api/exportDomainInfoFile +Content-Type: application/json +X-TOKEN: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyX2lkIjoxLCJleHAiOjE3MDE5MzMyNjN9.M481dinWgiAuoKw12HNU6DuXPwseqGk-5qRNwei7jAU + +{ + "ext": "xlsx" +} + +### diff --git a/requirements/production.txt b/requirements/production.txt index 53e5d9fa62..5e7238c512 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -18,3 +18,4 @@ prometheus-client acme fabric dnspython +openpyxl