Fix slow buffering

source/network_decoder/network_decoder_multiple.cpp

@@ -1,5 +1,6 @@
 #include "network_decoder_multiple.hpp"
 #include "headers.hpp"
+#include "youtube_parser/parser.hpp"
 
 void NetworkMultipleDecoder::deinit() {
 	initer_stop_request = true;
@@ -72,8 +73,8 @@ Result_with_string NetworkMultipleDecoder::init(std::string video_url, std::stri
 	NetworkDecoderFFmpegIOData tmp_ffmpeg_data;
 	std::vector<NetworkStream *> streams;
 	if (video_audio_seperate) {
-		NetworkStream *video_stream = new NetworkStream(video_url + url_append, is_livestream, NULL);
-		NetworkStream *audio_stream = new NetworkStream(audio_url + url_append, is_livestream, NULL);
+		NetworkStream *video_stream = new NetworkStream(video_url + url_append, extract_stream_length(video_url), is_livestream, NULL);
+		NetworkStream *audio_stream = new NetworkStream(audio_url + url_append, extract_stream_length(audio_url), is_livestream, NULL);
 		streams = {video_stream, audio_stream};
 		downloader.add_stream(video_stream);
 		downloader.add_stream(audio_stream);
@@ -85,7 +86,7 @@ Result_with_string NetworkMultipleDecoder::init(std::string video_url, std::stri
 		video_url = get_base_url(video_stream->url);
 		audio_url = get_base_url(audio_stream->url);
 	} else {
-		NetworkStream *both_stream = new NetworkStream(both_url + url_append, is_livestream, NULL);
+		NetworkStream *both_stream = new NetworkStream(both_url + url_append, extract_stream_length(both_url), is_livestream, NULL);
 		streams = { both_stream };
 		downloader.add_stream(both_stream);
 		decoder.interrupt = false;
@@ -244,9 +245,10 @@ void NetworkMultipleDecoder::livestream_initer_thread_func() {
 		logger.info("net/live-init", "next : " + std::to_string(seq_next));
 
 		NetworkDecoderFFmpegIOData tmp_ffmpeg_data;
+		std::string url_prefix = "&sq=" + std::to_string(seq_next);
 		if (video_audio_seperate) {
-			NetworkStream *video_stream = new NetworkStream(video_url + "&sq=" + std::to_string(seq_next), is_livestream, NULL);
-			NetworkStream *audio_stream = new NetworkStream(audio_url + "&sq=" + std::to_string(seq_next), is_livestream, NULL);
+			NetworkStream *video_stream = new NetworkStream(video_url + url_prefix, extract_stream_length(video_url), is_livestream, NULL);
+			NetworkStream *audio_stream = new NetworkStream(audio_url + url_prefix, extract_stream_length(audio_url), is_livestream, NULL);
 			video_stream->disable_interrupt = audio_stream->disable_interrupt = true;
 			downloader->add_stream(video_stream);
 			downloader->add_stream(audio_stream);
@@ -278,7 +280,7 @@ void NetworkMultipleDecoder::livestream_initer_thread_func() {
 			}
 			video_stream->disable_interrupt = audio_stream->disable_interrupt = false;
 		} else {
-			NetworkStream *both_stream = new NetworkStream(both_url + "&sq=" + std::to_string(seq_next), is_livestream, NULL);
+			NetworkStream *both_stream = new NetworkStream(both_url + url_prefix, extract_stream_length(both_url), is_livestream, NULL);
 			both_stream->disable_interrupt = true;
 			downloader->add_stream(both_stream);
 			Result_with_string result = tmp_ffmpeg_data.init(both_stream, &decoder);

source/network_decoder/network_downloader.cpp

@@ -116,6 +116,17 @@ static void confirm_thread_network_session_list_inited() {
 	}
 }
 
+static std::string remove_url_parameter(const std::string &url, const std::string &param) {
+	std::string res;
+	for (size_t i = 0; i < url.size(); ) {
+		if (url[i] == '&' || url[i] == '?') {
+			if (url.substr(i + 1, param.size() + 1) == param + "=") {
+				i = std::find(url.begin() + i + 1, url.end(), '&') - url.begin();
+			} else res.push_back(url[i++]);
+		} else res.push_back(url[i++]);
+	}
+	return res;
+}
 
 #define LOG_THREAD_STR "net/dl"
 void NetworkStreamDownloader::downloader_thread() {
@@ -238,12 +249,15 @@ void NetworkStreamDownloader::downloader_thread() {
 			u64 expected_len = end - start;
 
 			auto &session_list = cur_stream->session_list ? *cur_stream->session_list : thread_network_session_list;
-			auto result = session_list.perform(HttpRequest::GET(cur_stream->url, {{"Range", "bytes=" + std::to_string(start) + "-" + std::to_string(end - 1)}}));
-			if (result.redirected_url != "") cur_stream->url = result.redirected_url;
+			// length not sure -> use Range header to get the size (slower)
+			auto result = cur_stream->len == 0 ?
+				session_list.perform(HttpRequest::GET(cur_stream->url, {{"Range", "bytes=" + std::to_string(start) + "-" + std::to_string(end - 1)}})) :
+				session_list.perform(HttpRequest::GET(cur_stream->url + "&range=" + std::to_string(start) + "-" + std::to_string(end - 1), {}));
+			if (result.redirected_url != "") cur_stream->url = remove_url_parameter(result.redirected_url, "range");
 
 
 			if (!result.fail && result.status_code_is_success()) {
-				if (!cur_stream->ready) {
+				if (cur_stream->len == 0) {
 					auto content_range_str = result.get_header("Content-Range");
 					char *slash = strchr(content_range_str.c_str(), '/');
 					bool ok = false;
@@ -252,7 +266,7 @@ void NetworkStreamDownloader::downloader_thread() {
 						cur_stream->len = strtoll(slash + 1, &end, 10);
 						if (!*end) {
 							ok = true;
-							cur_stream->block_num = (cur_stream->len + BLOCK_SIZE - 1) / BLOCK_SIZE;
+							cur_stream->block_num = NetworkStream::get_block_num(cur_stream->len);
 						} else logger.error(LOG_THREAD_STR, "failed to parse Content-Range : " + std::string(slash + 1));
 					} else logger.error(LOG_THREAD_STR, "no slash in Content-Range response header : " + content_range_str);
 					if (!ok) cur_stream->error = true;

source/network_decoder/network_downloader.hpp

@@ -12,16 +12,17 @@ struct NetworkStream {
 	static constexpr u64 NEW3DS_MAX_CACHE_BLOCKS = 12 * 1000 * 1000 / BLOCK_SIZE;
 	static constexpr u64 OLD3DS_MAX_CACHE_BLOCKS = 4 * 1000 * 1000 / BLOCK_SIZE;
 	static constexpr int RETRY_CNT_MAX = 1;
+	static u64 get_block_num(u64 size) { return (size + BLOCK_SIZE - 1) / BLOCK_SIZE; }
 
-	u64 block_num = 0;
 	std::string url;
 	Mutex downloaded_data_lock; // std::map needs locking when searching and inserting at the same time
+	u64 len = 0;
+	u64 block_num = 0;
 	std::map<u64, std::vector<u8> > downloaded_data;
 	bool whole_download = false;
 	NetworkSessionList *session_list = NULL;
 
 	// anything above here is not supposed to be used from outside network_downloader.cpp and network_downloader.hpp
-	u64 len = 0;
 	volatile bool ready = false;
 	volatile bool suspend_request = false;
 	volatile bool quit_request = false;
@@ -37,8 +38,10 @@ struct NetworkStream {
 	bool livestream_private = false;
 	bool read_dead_tried = false;
 
+
 	// if `whole_download` is true, it will not use Range request but download the whole content at once (used for livestreams)
-	NetworkStream (std::string url, bool whole_download, NetworkSessionList *session_list) : url(url), whole_download(whole_download), session_list(session_list) {}
+	NetworkStream (std::string url, int64_t len, bool whole_download, NetworkSessionList *session_list) : url(url), len(len < 0 ? 0 : len),
+		block_num(get_block_num(this->len)), whole_download(whole_download), session_list(session_list) {}
 
 	double get_download_percentage();
 	std::vector<double> get_buffering_progress_bar(int res_len);

source/youtube_parser/parser.hpp

@@ -237,6 +237,7 @@ std::string get_video_id_from_thumbnail_url(const std::string &url);
 bool youtube_is_valid_video_id(const std::string &id);
 bool is_youtube_url(const std::string &url);
 bool is_youtube_thumbnail_url(const std::string &url);
+int64_t extract_stream_length(const std::string &url);
 
 enum class YouTubePageType {
 	VIDEO,

source/youtube_parser/utils.cpp

@@ -67,4 +67,13 @@ YouTubePageType youtube_get_page_type(std::string url) {
 	if (starts_with(url, "https://m.youtube.com/results?", 0)) return YouTubePageType::SEARCH;
 	return YouTubePageType::INVALID;
 }
+int64_t extract_stream_length(const std::string &url) {
+	auto pos = url.find("&clen=");
+	if (pos == std::string::npos) pos = url.find("?clen=");
+	if (pos == std::string::npos) return -1;
+	pos += std::string("&clen=").size();
+	int64_t res = 0;
+	while (pos < url.size() && isdigit(url[pos])) res = res * 10 + url[pos++] - '0';
+	return res;
+}
 

source/youtube_parser/video.cpp

@@ -40,7 +40,6 @@ static bool extract_player_data(Document &json_root, RJson player_response, YouT
 
 		url = url.substr(0, n_start) + next_n + url.substr(n_end, url.size() - n_end);
 		if (url.find("ratebypass") == std::string::npos) url += "&ratebypass=yes";
-
 		i.set_str(json_root, "url", url.c_str());
 	}
 	for (auto &i : formats) i.set_str(json_root, "url", url_decode(i["url"].string_value()).c_str()); // something like %2C still appears in the url, so decode them back
@@ -302,7 +301,7 @@ YouTubeVideoDetail youtube_load_video_page(std::string url) {
 #	ifdef _WIN32
 	// for debug purpose, to check whether the extracted stream url is working
 	if (res.audio_stream_url != "") {
-		auto tmp_data = http_get(res.audio_stream_url, {{"Range", "bytes=0-400000"}}).second;
+		auto tmp_data = http_get(res.audio_stream_url + "&range=0-400000").second;
 		if (tmp_data.size() != 400001) {
 			debug_error("!!!!!!!!!!!!!!!!!!!!! SIZE DIFFER : " + std::to_string(tmp_data.size()) + " !!!!!!!!!!!!!!!!!!!!!");
 		} else debug_info("----------------------- OK -----------------------");