AutoFilm v1.0.0

.gitignore

@@ -1 +1,7 @@
-Media/*
+config/config.yaml
+strm/*
+media/*
+__pycache__/*
+
+*bak
+test-*

README.md

@@ -2,7 +2,7 @@
 **一个为Emby、Jellyfin服务器提供直链播放的小项目**
 
 # 说明文档
-详情见[Akimio的博客](https://blog.akimio.top/posts/1031/#使用教程)
+详情见[AutoFilm说明文档](https://blog.akimio.top/posts/1031/)
 
 # 优点
 - [x] 轻量化Emby服务器，降低Emby服务器的性能需求以及硬盘需求
@@ -12,9 +12,16 @@
 
 # TODO LIST
 - [x] 从config文件中读取多个参数
-- [ ] 优化程序运行效率
+- [x] 优化程序运行效率（多线程处理）
+- [ ] 增加Docker镜像
+- [ ] Strm模式/媒体库模式
+- [ ] 监控模式
 - [ ] 对接TMDB实现分类、重命名、刮削等功能
 
+# 更新日志
+- 2024.2.1： v1.0.0，完全重构AutoFilm，不再兼容v0.1，实现多线程，大幅度提升任务处理速度
+- 2024.1.28：v0.1.1，初始版本持续迭代
+
 # 开源许可证
 **本项目采用 GNU Affero General Public License（GNU AGPL）开源许可证。**
 

autofilm.py

@@ -1,154 +1,144 @@
+import os
+import queue
+import threading
+import requests
+import yaml
+import time
 from webdav3.client import Client
-import argparse, os, requests, time
-
-from version import APP_VERSION
-
-'''
-遍历Webdav服务器函数
-如果depth为None，则会递归遍历整个WebDAV服务器
-如果depth为正整数，则会递归遍历到指定深度
-如果depth为0，则只会遍历当前文件夹中的文件和文件夹，不会继续递归遍历下一级文件夹。
-'''
-def list_files(webdav_url, username, password, show_path, depth=None, path='', count=0, proxies=None):
-    options = {
-        'webdav_hostname': webdav_url,
-        'webdav_login': username,
-        'webdav_password': password,
-        'proxies': proxies
-    }
-
-    client = Client(options)
-    directory = []
-    files = []
-    q = 1
-    while q < 15:
-        try:
-            items = client.list()
-        except:
-            print(f'第{q}次连接失败，{q+1}秒后重试...')
-            q += 1
-            time.sleep(q)
-        else:
-            if q > 1:
-                print('重连成功...')
-            break
-
-    if q == 15:
-        print('连接失败，请检查网络设置！')
-        exit()
-
-    for item in items[1:]:
-        if item[-1] == '/':
-            if depth is None or depth > 0:
-                subdirectory, subfiles, count = list_files(webdav_url + item, username, password, depth=None if depth is None else depth - 1, path=path+item, count=count)
-                directory += [item + subitem for subitem in subdirectory]
-                files += [item + subitem for subitem in subfiles]
+
+def list_files(username: str, password: str, urls_queue:queue.Queue, files_queue:queue.Queue):
+    while not urls_queue.empty():
+        url = urls_queue.get()
+        print(f"{threading.current_thread().name}——正在处理:{url}，剩余{urls_queue.qsize()}个URL待处理")
+        options = {
+            "webdav_hostname": url,
+            "webdav_login": username,
+            "webdav_password": password
+        }
+        client = Client(options)
+
+        try_number = 1
+        try_max = 15
+
+        while try_number < try_max:
+            try:
+                items = client.list()
+            except Exception as e:
+                print(f"{threading.current_thread().name}遇到错误，第{try_number}尝试失败；错误信息：{str(e)}，传入URL：{url}，")
+                time.sleep(try_number)
+                try_number += 1
             else:
-                directory.append(item)
-        else:
-            files.append(item)
-            count += 1
-    if show_path and path:
-        print(f'当前文件夹路径：{path}')
-    return directory, files, count
-
-'''
-下载函数
-用于'ASS', 'SRT', 'SSA','NFO','JPG', 'PNG'文件的下载
-'''   
-def download_file(url, local_path, filename, total_count):
-    p = 1
-    while p < 10:
+                if try_number > 1:
+                    print(f"{url}重连成功")
+                break
+        for item in items[1:]:
+            if item.endswith("/"):
+                urls_queue.put(url + item)
+            else:
+                files_queue.put(url + item)
+    print(f"{threading.current_thread().name}处理完毕")
+
+def strm_file(url: str, output_path: str, filename: str):
+    strm_filename = filename.rsplit(".", 1)[0] + ".strm"
+    local_path = os.path.join(output_path, strm_filename)
+    if not os.path.exists(local_path):
+        try:
+            print(f"正在下载：{filename}")
+            os.makedirs(os.path.dirname(local_path), exist_ok=True)
+            with open(local_path, "wb") as file:
+                file.write((url.replace("/dav", "/d") + filename).encode())
+            print(f"{filename}处理成功")
+        except Exception as e:
+            print(f"{filename}处理失败，错误信息：{str(e)}")
+    else:
+        print(f"{filename}已存在，跳过处理")
+
+def download_file(url:str, output_path:str, filename:str):
+    local_path = os.path.join(output_path, filename)
+    if not os.path.exists(local_path):
         try:
-            print('正在下载：' + filename)
-            r = requests.get(url.replace('/dav', '/d'), proxies=proxies)
+            print(f"正在下载：{filename}")
             os.makedirs(os.path.dirname(local_path), exist_ok=True)
-            with open(local_path, 'wb') as f:
-                f.write(r.content)
-                f.close()
-        except:
-            print(f'第{p}次下载失败，{p + 1}秒后重试...')
-            p += 1
-            time.sleep(p)
+            response = requests.get(url.replace("/dav", "/d") + filename)
+            with open(local_path, "wb") as file:
+                file.write(response.content)
+        except Exception as e:
+            print(f"{filename}下载失败，错误信息：{str(e)}")
+    else:
+        print(f"{filename}已存在，跳过下载")
+
+def processing_file(output_path:str, url_base:str, files_queue:queue.Queue, subtitle:bool, img:bool, nfo:bool):
+    video_format = ["mp4", "mkv", "flv", "avi", "wmv", "ts", "rmvb", "webm"]
+    subtitle_format = ["ass", "srt", "ssa", "sub"]
+    img_format = ["png", "jpg"]
+
+    waite_number = 0
+    waite_max = 10
+    waite_time = 5
+
+    while waite_number < waite_max:
+        if not files_queue.empty():
+            file_url = files_queue.get()
+            print(f"{threading.current_thread().name}——正在处理:{file_url}，剩余{files_queue.qsize()}个文件待处理")
+            filename = file_url.replace(url_base, "")
+            if filename.lower().endswith(tuple(video_format)):
+                strm_file(url_base, output_path, filename)
+            elif filename.lower().endswith(tuple(subtitle_format)) & subtitle:
+                download_file(url_base, output_path, filename)
+            elif filename.lower().endswith(tuple(img_format)) & img:
+                download_file(url_base, output_path, filename)
+            elif filename.lower().endswith("nfo") & nfo:
+                download_file(url_base, output_path, filename)
         else:
-            if p > 1:
-                print('重新下载成功！')
-            print(filename + '下载成功！')
-            break
-        progress = int((p / 10) * 100)
-        print(f'已完成 {progress}%，共 {total_count} 个文件')
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Autofilm script')
-    parser.add_argument('--webdav_url', type=str, help='WebDAV服务器地址', required=True)
-    parser.add_argument('--username', type=str, help='WebDAV账号', required=True)
-    parser.add_argument('--password', type=str, help='WebDAV密码', required=True)
-    parser.add_argument('--output_path', type=str, help='输出文件目录', default='./Media/')
-    parser.add_argument('--subtitle', type=str, help='是否下载字幕文件', choices=['true', 'false'], default='true')
-    parser.add_argument('--nfo', type=str, help='是否下载NFO文件', choices=['true', 'false'], default='false')
-    parser.add_argument('--img', type=str, help='是否下载JPG和PNG文件', choices=['true', 'false'], default='false')
-    parser.add_argument('--show_path', type=str, help='遍历时是否显示文件夹路径', choices=['true', 'false'], default='false')
-    parser.add_argument('--proxy', type=str, help='HTTP代理服务器，格式为IP:端口号')
-    args = parser.parse_args()
-
-    print(f"当前的APP版本是：{APP_VERSION}")
-
-    print('启动参数：')
-    print(f'Webdav服务器地址：{args.webdav_url}')
-    print(f'Webdav登入用户名：{args.username}')
-    print(f'Webdav登入密码：{args.password}')
-    print(f'文件输出路径：{args.output_path}')
-    print(f'是否下载字幕：{args.subtitle}')
-    print(f'是否下载电影信息：{args.nfo}')
-    print(f'是否下载图片：{args.img}')
-    print(f'遍历时是否显示文件夹路径：{args.show_path}')
-
-    proxies = None
-    if args.proxy:
-        proxies = {
-            'http': f'http://{args.proxy}',
-            'https': f'http://{args.proxy}'
-        }
-
-    directory, files, count = list_files(args.webdav_url, args.username, args.password, args.show_path, depth=None, path='', count=0, proxies=proxies)
-
-    urls = [args.webdav_url + item for item in directory + files]
-
-    download_count = 0
-
-    for url in urls:
-        if url[-1] == '/':
-            continue
-        filename = os.path.basename(url)
-        local_path = os.path.join(args.output_path, url.replace(args.webdav_url, '').lstrip('/'))
-        file_ext = filename[-3:].upper()
-
-        valid_extensions = ['mp4', 'mkv', 'flv', 'avi', 'wmv', 'ts', 'rmvb', 'webm']
-        if filename.lower().endswith(tuple(valid_extensions)):
-            new_filename = filename.rsplit('.', 1)[0] + '.strm'
-            if not os.path.exists(os.path.join(args.output_path, new_filename)):
-                print('正在处理：' + filename)
-                try:
-                    os.makedirs(os.path.dirname(local_path), exist_ok=True)
-                    with open(os.path.join(local_path.rsplit('.', 1)[0] + '.strm'), "w", encoding='utf-8') as f:
-                        f.write(url.replace('/dav', '/d'))
-                except:
-                    print(filename + '处理失败，文件名包含特殊符号，建议重命名！')
-        elif args.subtitle == 'true' and file_ext in ['ASS', 'SRT', 'SSA', 'SUB']:
-            if not os.path.exists(local_path):
-                download_file(url, local_path, filename, count)
-                download_count += 1
-        elif args.nfo == 'true' and file_ext == 'NFO':
-            if not os.path.exists(local_path):
-                download_file(url, local_path, filename, count)
-                download_count += 1
-        elif args.img == 'true' and file_ext in ['JPG', 'PNG']:
-            if not os.path.exists(local_path):
-                download_file(url, local_path, filename, count)
-                download_count += 1
-
-        progress = int((download_count / count) * 100)
-        print(f'已完成 {progress}%，共 {count} 个文件')
-
-    print('处理完毕！')
+            waite_number += 1
+            print(f"files_queue列表为空，当前尝试次数：{waite_number}，共尝试{waite_max}次，{waite_time}秒后重试")
+            time.sleep(waite_time)
+    print(f"{threading.current_thread().name}处理完毕")
+
+def main(config_path:str):
+    with open(config_path, "r", encoding="utf-8") as file:
+        config_data = yaml.safe_load(file)
+    output_path = config_data["setting"]["output_path"]
+    l_threads = config_data["setting"]["l_threads"]
+    p_threads = config_data["setting"]["p_threads"]
+
+    print(f"{'='*65}\n输出目录：{output_path}\nlist_files线程数：{l_threads}\nprocessing_file线程数：{p_threads}\n{'='*65}")
+    subtitle = config_data["setting"]["subtitle"]
+    img = config_data["setting"]["img"]
+    nfo = config_data["setting"]["nfo"]
+
+    webdav_data = config_data["webdav"]
+    print(f"一共读取到{len(webdav_data)}个Webdav配置")
+
+    round = 1
+    for key, value in  webdav_data.items():
+        print(f"开始生成{key}Webdav服务器的Strm文件\n剩余{(len(webdav_data) - round)}个Webdav服务器未进行")
+        url = value["url"]
+        username = value["username"]
+        password = value["password"]
+
+        urls_queue = queue.Queue() 
+        files_queue = queue.Queue()
+        urls_queue.put(url)
+
+        list_files_interval = 10
+        lp_interval = 10
+        processing_file_interval = 10
+
+        for thread in range(l_threads):
+            print(f"list_files线程{thread}启动中")
+            t = threading.Thread(target=list_files, args=(username, password, urls_queue, files_queue), name=f"list_files线程{thread}")
+            t.start()
+            print(f"list_files线程{thread}已启动，{list_files_interval}秒后启动下一个线程")
+            time.sleep(list_files_interval)
+
+        time.sleep(lp_interval)
+
+        for thread in range(p_threads):
+            print(f"processing_file线程{thread}启动中")
+            t = threading.Thread(target=processing_file, args=(output_path, url, files_queue,subtitle, img, nfo), name=f"processing_file线程{thread}")
+            t.start()
+            print(f"processing_file线程{thread}已启动，{processing_file_interval}秒后启动下一个线程")
+            time.sleep(processing_file_interval)
+
+        round += 1