From 5fbaf3b16fe93a8b5e37d0e58d779b2b4c220b2f Mon Sep 17 00:00:00 2001 From: windows-server-2003 Date: Fri, 3 Mar 2023 14:36:38 +0900 Subject: [PATCH] Fix slow buffering --- .../network_decoder_multiple.cpp | 14 +++++++----- source/network_decoder/network_downloader.cpp | 22 +++++++++++++++---- source/network_decoder/network_downloader.hpp | 9 +++++--- source/youtube_parser/parser.hpp | 1 + source/youtube_parser/utils.cpp | 9 ++++++++ source/youtube_parser/video.cpp | 3 +-- 6 files changed, 43 insertions(+), 15 deletions(-) diff --git a/source/network_decoder/network_decoder_multiple.cpp b/source/network_decoder/network_decoder_multiple.cpp index 6f8903fa..e02c2f0f 100644 --- a/source/network_decoder/network_decoder_multiple.cpp +++ b/source/network_decoder/network_decoder_multiple.cpp @@ -1,5 +1,6 @@ #include "network_decoder_multiple.hpp" #include "headers.hpp" +#include "youtube_parser/parser.hpp" void NetworkMultipleDecoder::deinit() { initer_stop_request = true; @@ -72,8 +73,8 @@ Result_with_string NetworkMultipleDecoder::init(std::string video_url, std::stri NetworkDecoderFFmpegIOData tmp_ffmpeg_data; std::vector streams; if (video_audio_seperate) { - NetworkStream *video_stream = new NetworkStream(video_url + url_append, is_livestream, NULL); - NetworkStream *audio_stream = new NetworkStream(audio_url + url_append, is_livestream, NULL); + NetworkStream *video_stream = new NetworkStream(video_url + url_append, extract_stream_length(video_url), is_livestream, NULL); + NetworkStream *audio_stream = new NetworkStream(audio_url + url_append, extract_stream_length(audio_url), is_livestream, NULL); streams = {video_stream, audio_stream}; downloader.add_stream(video_stream); downloader.add_stream(audio_stream); @@ -85,7 +86,7 @@ Result_with_string NetworkMultipleDecoder::init(std::string video_url, std::stri video_url = get_base_url(video_stream->url); audio_url = get_base_url(audio_stream->url); } else { - NetworkStream *both_stream = new NetworkStream(both_url + url_append, is_livestream, NULL); + NetworkStream *both_stream = new NetworkStream(both_url + url_append, extract_stream_length(both_url), is_livestream, NULL); streams = { both_stream }; downloader.add_stream(both_stream); decoder.interrupt = false; @@ -244,9 +245,10 @@ void NetworkMultipleDecoder::livestream_initer_thread_func() { logger.info("net/live-init", "next : " + std::to_string(seq_next)); NetworkDecoderFFmpegIOData tmp_ffmpeg_data; + std::string url_prefix = "&sq=" + std::to_string(seq_next); if (video_audio_seperate) { - NetworkStream *video_stream = new NetworkStream(video_url + "&sq=" + std::to_string(seq_next), is_livestream, NULL); - NetworkStream *audio_stream = new NetworkStream(audio_url + "&sq=" + std::to_string(seq_next), is_livestream, NULL); + NetworkStream *video_stream = new NetworkStream(video_url + url_prefix, extract_stream_length(video_url), is_livestream, NULL); + NetworkStream *audio_stream = new NetworkStream(audio_url + url_prefix, extract_stream_length(audio_url), is_livestream, NULL); video_stream->disable_interrupt = audio_stream->disable_interrupt = true; downloader->add_stream(video_stream); downloader->add_stream(audio_stream); @@ -278,7 +280,7 @@ void NetworkMultipleDecoder::livestream_initer_thread_func() { } video_stream->disable_interrupt = audio_stream->disable_interrupt = false; } else { - NetworkStream *both_stream = new NetworkStream(both_url + "&sq=" + std::to_string(seq_next), is_livestream, NULL); + NetworkStream *both_stream = new NetworkStream(both_url + url_prefix, extract_stream_length(both_url), is_livestream, NULL); both_stream->disable_interrupt = true; downloader->add_stream(both_stream); Result_with_string result = tmp_ffmpeg_data.init(both_stream, &decoder); diff --git a/source/network_decoder/network_downloader.cpp b/source/network_decoder/network_downloader.cpp index 0277eb08..547d08b0 100644 --- a/source/network_decoder/network_downloader.cpp +++ b/source/network_decoder/network_downloader.cpp @@ -116,6 +116,17 @@ static void confirm_thread_network_session_list_inited() { } } +static std::string remove_url_parameter(const std::string &url, const std::string ¶m) { + std::string res; + for (size_t i = 0; i < url.size(); ) { + if (url[i] == '&' || url[i] == '?') { + if (url.substr(i + 1, param.size() + 1) == param + "=") { + i = std::find(url.begin() + i + 1, url.end(), '&') - url.begin(); + } else res.push_back(url[i++]); + } else res.push_back(url[i++]); + } + return res; +} #define LOG_THREAD_STR "net/dl" void NetworkStreamDownloader::downloader_thread() { @@ -238,12 +249,15 @@ void NetworkStreamDownloader::downloader_thread() { u64 expected_len = end - start; auto &session_list = cur_stream->session_list ? *cur_stream->session_list : thread_network_session_list; - auto result = session_list.perform(HttpRequest::GET(cur_stream->url, {{"Range", "bytes=" + std::to_string(start) + "-" + std::to_string(end - 1)}})); - if (result.redirected_url != "") cur_stream->url = result.redirected_url; + // length not sure -> use Range header to get the size (slower) + auto result = cur_stream->len == 0 ? + session_list.perform(HttpRequest::GET(cur_stream->url, {{"Range", "bytes=" + std::to_string(start) + "-" + std::to_string(end - 1)}})) : + session_list.perform(HttpRequest::GET(cur_stream->url + "&range=" + std::to_string(start) + "-" + std::to_string(end - 1), {})); + if (result.redirected_url != "") cur_stream->url = remove_url_parameter(result.redirected_url, "range"); if (!result.fail && result.status_code_is_success()) { - if (!cur_stream->ready) { + if (cur_stream->len == 0) { auto content_range_str = result.get_header("Content-Range"); char *slash = strchr(content_range_str.c_str(), '/'); bool ok = false; @@ -252,7 +266,7 @@ void NetworkStreamDownloader::downloader_thread() { cur_stream->len = strtoll(slash + 1, &end, 10); if (!*end) { ok = true; - cur_stream->block_num = (cur_stream->len + BLOCK_SIZE - 1) / BLOCK_SIZE; + cur_stream->block_num = NetworkStream::get_block_num(cur_stream->len); } else logger.error(LOG_THREAD_STR, "failed to parse Content-Range : " + std::string(slash + 1)); } else logger.error(LOG_THREAD_STR, "no slash in Content-Range response header : " + content_range_str); if (!ok) cur_stream->error = true; diff --git a/source/network_decoder/network_downloader.hpp b/source/network_decoder/network_downloader.hpp index b96d7ecf..5dce9399 100644 --- a/source/network_decoder/network_downloader.hpp +++ b/source/network_decoder/network_downloader.hpp @@ -12,16 +12,17 @@ struct NetworkStream { static constexpr u64 NEW3DS_MAX_CACHE_BLOCKS = 12 * 1000 * 1000 / BLOCK_SIZE; static constexpr u64 OLD3DS_MAX_CACHE_BLOCKS = 4 * 1000 * 1000 / BLOCK_SIZE; static constexpr int RETRY_CNT_MAX = 1; + static u64 get_block_num(u64 size) { return (size + BLOCK_SIZE - 1) / BLOCK_SIZE; } - u64 block_num = 0; std::string url; Mutex downloaded_data_lock; // std::map needs locking when searching and inserting at the same time + u64 len = 0; + u64 block_num = 0; std::map > downloaded_data; bool whole_download = false; NetworkSessionList *session_list = NULL; // anything above here is not supposed to be used from outside network_downloader.cpp and network_downloader.hpp - u64 len = 0; volatile bool ready = false; volatile bool suspend_request = false; volatile bool quit_request = false; @@ -37,8 +38,10 @@ struct NetworkStream { bool livestream_private = false; bool read_dead_tried = false; + // if `whole_download` is true, it will not use Range request but download the whole content at once (used for livestreams) - NetworkStream (std::string url, bool whole_download, NetworkSessionList *session_list) : url(url), whole_download(whole_download), session_list(session_list) {} + NetworkStream (std::string url, int64_t len, bool whole_download, NetworkSessionList *session_list) : url(url), len(len < 0 ? 0 : len), + block_num(get_block_num(this->len)), whole_download(whole_download), session_list(session_list) {} double get_download_percentage(); std::vector get_buffering_progress_bar(int res_len); diff --git a/source/youtube_parser/parser.hpp b/source/youtube_parser/parser.hpp index c98d9940..4836aa27 100644 --- a/source/youtube_parser/parser.hpp +++ b/source/youtube_parser/parser.hpp @@ -237,6 +237,7 @@ std::string get_video_id_from_thumbnail_url(const std::string &url); bool youtube_is_valid_video_id(const std::string &id); bool is_youtube_url(const std::string &url); bool is_youtube_thumbnail_url(const std::string &url); +int64_t extract_stream_length(const std::string &url); enum class YouTubePageType { VIDEO, diff --git a/source/youtube_parser/utils.cpp b/source/youtube_parser/utils.cpp index 97eebadb..978f887e 100644 --- a/source/youtube_parser/utils.cpp +++ b/source/youtube_parser/utils.cpp @@ -67,4 +67,13 @@ YouTubePageType youtube_get_page_type(std::string url) { if (starts_with(url, "https://m.youtube.com/results?", 0)) return YouTubePageType::SEARCH; return YouTubePageType::INVALID; } +int64_t extract_stream_length(const std::string &url) { + auto pos = url.find("&clen="); + if (pos == std::string::npos) pos = url.find("?clen="); + if (pos == std::string::npos) return -1; + pos += std::string("&clen=").size(); + int64_t res = 0; + while (pos < url.size() && isdigit(url[pos])) res = res * 10 + url[pos++] - '0'; + return res; +} diff --git a/source/youtube_parser/video.cpp b/source/youtube_parser/video.cpp index a7761b83..da182525 100644 --- a/source/youtube_parser/video.cpp +++ b/source/youtube_parser/video.cpp @@ -40,7 +40,6 @@ static bool extract_player_data(Document &json_root, RJson player_response, YouT url = url.substr(0, n_start) + next_n + url.substr(n_end, url.size() - n_end); if (url.find("ratebypass") == std::string::npos) url += "&ratebypass=yes"; - i.set_str(json_root, "url", url.c_str()); } for (auto &i : formats) i.set_str(json_root, "url", url_decode(i["url"].string_value()).c_str()); // something like %2C still appears in the url, so decode them back @@ -302,7 +301,7 @@ YouTubeVideoDetail youtube_load_video_page(std::string url) { # ifdef _WIN32 // for debug purpose, to check whether the extracted stream url is working if (res.audio_stream_url != "") { - auto tmp_data = http_get(res.audio_stream_url, {{"Range", "bytes=0-400000"}}).second; + auto tmp_data = http_get(res.audio_stream_url + "&range=0-400000").second; if (tmp_data.size() != 400001) { debug_error("!!!!!!!!!!!!!!!!!!!!! SIZE DIFFER : " + std::to_string(tmp_data.size()) + " !!!!!!!!!!!!!!!!!!!!!"); } else debug_info("----------------------- OK -----------------------");