From 1eac12d49bac9d5c9a36ffd2c902901aefd9227b Mon Sep 17 00:00:00 2001 From: windows-server-2003 Date: Fri, 3 Mar 2023 13:20:13 +0900 Subject: [PATCH 1/3] Update YT decrypter --- decrypter/0_latest.txt | 39 ++++++++++++++++++----------------- decrypter/archive/0/19417.txt | 26 +++++++++++++++++++++++ 2 files changed, 46 insertions(+), 19 deletions(-) create mode 100644 decrypter/archive/0/19417.txt diff --git a/decrypter/0_latest.txt b/decrypter/0_latest.txt index 93b6b73d..f5f90552 100644 --- a/decrypter/0_latest.txt +++ b/decrypter/0_latest.txt @@ -1,25 +1,26 @@ -# 19369 +# 19417 +s 17 +c 1 R -c 2 -R -c 3 +s 65 > +r -800848978 +R +e 330905968 +r -1801395041 +r -45203544 +r 969944705 +s -1266939812 +s 1071587255 +r 2070384319 +t hwadna7 Aa0-_ +s 61910453 R -s -962757806 +t UDjUn 0aA-_ +r -1700764381 R R -r -1343083959 -r 258676768 -s 1600529817 -s -321620231 -r -65500503 -r 301529828 -s 1621916887 -r -1772925964 -t else 0aA-_ -s 2024158092 -s 316275604 +r 859083917 +r -2140614363 +r -307172264 R -s 2108090471 -r 1565550281 -e -1494726146 diff --git a/decrypter/archive/0/19417.txt b/decrypter/archive/0/19417.txt new file mode 100644 index 00000000..f5f90552 --- /dev/null +++ b/decrypter/archive/0/19417.txt @@ -0,0 +1,26 @@ +# 19417 +s 17 +c 1 +R +s 65 +> +r -800848978 +R +e 330905968 +r -1801395041 +r -45203544 +r 969944705 +s -1266939812 +s 1071587255 +r 2070384319 +t hwadna7 Aa0-_ +s 61910453 +R +t UDjUn 0aA-_ +r -1700764381 +R +R +r 859083917 +r -2140614363 +r -307172264 +R From 5fbaf3b16fe93a8b5e37d0e58d779b2b4c220b2f Mon Sep 17 00:00:00 2001 From: windows-server-2003 Date: Fri, 3 Mar 2023 14:36:38 +0900 Subject: [PATCH 2/3] Fix slow buffering --- .../network_decoder_multiple.cpp | 14 +++++++----- source/network_decoder/network_downloader.cpp | 22 +++++++++++++++---- source/network_decoder/network_downloader.hpp | 9 +++++--- source/youtube_parser/parser.hpp | 1 + source/youtube_parser/utils.cpp | 9 ++++++++ source/youtube_parser/video.cpp | 3 +-- 6 files changed, 43 insertions(+), 15 deletions(-) diff --git a/source/network_decoder/network_decoder_multiple.cpp b/source/network_decoder/network_decoder_multiple.cpp index 6f8903fa..e02c2f0f 100644 --- a/source/network_decoder/network_decoder_multiple.cpp +++ b/source/network_decoder/network_decoder_multiple.cpp @@ -1,5 +1,6 @@ #include "network_decoder_multiple.hpp" #include "headers.hpp" +#include "youtube_parser/parser.hpp" void NetworkMultipleDecoder::deinit() { initer_stop_request = true; @@ -72,8 +73,8 @@ Result_with_string NetworkMultipleDecoder::init(std::string video_url, std::stri NetworkDecoderFFmpegIOData tmp_ffmpeg_data; std::vector streams; if (video_audio_seperate) { - NetworkStream *video_stream = new NetworkStream(video_url + url_append, is_livestream, NULL); - NetworkStream *audio_stream = new NetworkStream(audio_url + url_append, is_livestream, NULL); + NetworkStream *video_stream = new NetworkStream(video_url + url_append, extract_stream_length(video_url), is_livestream, NULL); + NetworkStream *audio_stream = new NetworkStream(audio_url + url_append, extract_stream_length(audio_url), is_livestream, NULL); streams = {video_stream, audio_stream}; downloader.add_stream(video_stream); downloader.add_stream(audio_stream); @@ -85,7 +86,7 @@ Result_with_string NetworkMultipleDecoder::init(std::string video_url, std::stri video_url = get_base_url(video_stream->url); audio_url = get_base_url(audio_stream->url); } else { - NetworkStream *both_stream = new NetworkStream(both_url + url_append, is_livestream, NULL); + NetworkStream *both_stream = new NetworkStream(both_url + url_append, extract_stream_length(both_url), is_livestream, NULL); streams = { both_stream }; downloader.add_stream(both_stream); decoder.interrupt = false; @@ -244,9 +245,10 @@ void NetworkMultipleDecoder::livestream_initer_thread_func() { logger.info("net/live-init", "next : " + std::to_string(seq_next)); NetworkDecoderFFmpegIOData tmp_ffmpeg_data; + std::string url_prefix = "&sq=" + std::to_string(seq_next); if (video_audio_seperate) { - NetworkStream *video_stream = new NetworkStream(video_url + "&sq=" + std::to_string(seq_next), is_livestream, NULL); - NetworkStream *audio_stream = new NetworkStream(audio_url + "&sq=" + std::to_string(seq_next), is_livestream, NULL); + NetworkStream *video_stream = new NetworkStream(video_url + url_prefix, extract_stream_length(video_url), is_livestream, NULL); + NetworkStream *audio_stream = new NetworkStream(audio_url + url_prefix, extract_stream_length(audio_url), is_livestream, NULL); video_stream->disable_interrupt = audio_stream->disable_interrupt = true; downloader->add_stream(video_stream); downloader->add_stream(audio_stream); @@ -278,7 +280,7 @@ void NetworkMultipleDecoder::livestream_initer_thread_func() { } video_stream->disable_interrupt = audio_stream->disable_interrupt = false; } else { - NetworkStream *both_stream = new NetworkStream(both_url + "&sq=" + std::to_string(seq_next), is_livestream, NULL); + NetworkStream *both_stream = new NetworkStream(both_url + url_prefix, extract_stream_length(both_url), is_livestream, NULL); both_stream->disable_interrupt = true; downloader->add_stream(both_stream); Result_with_string result = tmp_ffmpeg_data.init(both_stream, &decoder); diff --git a/source/network_decoder/network_downloader.cpp b/source/network_decoder/network_downloader.cpp index 0277eb08..547d08b0 100644 --- a/source/network_decoder/network_downloader.cpp +++ b/source/network_decoder/network_downloader.cpp @@ -116,6 +116,17 @@ static void confirm_thread_network_session_list_inited() { } } +static std::string remove_url_parameter(const std::string &url, const std::string ¶m) { + std::string res; + for (size_t i = 0; i < url.size(); ) { + if (url[i] == '&' || url[i] == '?') { + if (url.substr(i + 1, param.size() + 1) == param + "=") { + i = std::find(url.begin() + i + 1, url.end(), '&') - url.begin(); + } else res.push_back(url[i++]); + } else res.push_back(url[i++]); + } + return res; +} #define LOG_THREAD_STR "net/dl" void NetworkStreamDownloader::downloader_thread() { @@ -238,12 +249,15 @@ void NetworkStreamDownloader::downloader_thread() { u64 expected_len = end - start; auto &session_list = cur_stream->session_list ? *cur_stream->session_list : thread_network_session_list; - auto result = session_list.perform(HttpRequest::GET(cur_stream->url, {{"Range", "bytes=" + std::to_string(start) + "-" + std::to_string(end - 1)}})); - if (result.redirected_url != "") cur_stream->url = result.redirected_url; + // length not sure -> use Range header to get the size (slower) + auto result = cur_stream->len == 0 ? + session_list.perform(HttpRequest::GET(cur_stream->url, {{"Range", "bytes=" + std::to_string(start) + "-" + std::to_string(end - 1)}})) : + session_list.perform(HttpRequest::GET(cur_stream->url + "&range=" + std::to_string(start) + "-" + std::to_string(end - 1), {})); + if (result.redirected_url != "") cur_stream->url = remove_url_parameter(result.redirected_url, "range"); if (!result.fail && result.status_code_is_success()) { - if (!cur_stream->ready) { + if (cur_stream->len == 0) { auto content_range_str = result.get_header("Content-Range"); char *slash = strchr(content_range_str.c_str(), '/'); bool ok = false; @@ -252,7 +266,7 @@ void NetworkStreamDownloader::downloader_thread() { cur_stream->len = strtoll(slash + 1, &end, 10); if (!*end) { ok = true; - cur_stream->block_num = (cur_stream->len + BLOCK_SIZE - 1) / BLOCK_SIZE; + cur_stream->block_num = NetworkStream::get_block_num(cur_stream->len); } else logger.error(LOG_THREAD_STR, "failed to parse Content-Range : " + std::string(slash + 1)); } else logger.error(LOG_THREAD_STR, "no slash in Content-Range response header : " + content_range_str); if (!ok) cur_stream->error = true; diff --git a/source/network_decoder/network_downloader.hpp b/source/network_decoder/network_downloader.hpp index b96d7ecf..5dce9399 100644 --- a/source/network_decoder/network_downloader.hpp +++ b/source/network_decoder/network_downloader.hpp @@ -12,16 +12,17 @@ struct NetworkStream { static constexpr u64 NEW3DS_MAX_CACHE_BLOCKS = 12 * 1000 * 1000 / BLOCK_SIZE; static constexpr u64 OLD3DS_MAX_CACHE_BLOCKS = 4 * 1000 * 1000 / BLOCK_SIZE; static constexpr int RETRY_CNT_MAX = 1; + static u64 get_block_num(u64 size) { return (size + BLOCK_SIZE - 1) / BLOCK_SIZE; } - u64 block_num = 0; std::string url; Mutex downloaded_data_lock; // std::map needs locking when searching and inserting at the same time + u64 len = 0; + u64 block_num = 0; std::map > downloaded_data; bool whole_download = false; NetworkSessionList *session_list = NULL; // anything above here is not supposed to be used from outside network_downloader.cpp and network_downloader.hpp - u64 len = 0; volatile bool ready = false; volatile bool suspend_request = false; volatile bool quit_request = false; @@ -37,8 +38,10 @@ struct NetworkStream { bool livestream_private = false; bool read_dead_tried = false; + // if `whole_download` is true, it will not use Range request but download the whole content at once (used for livestreams) - NetworkStream (std::string url, bool whole_download, NetworkSessionList *session_list) : url(url), whole_download(whole_download), session_list(session_list) {} + NetworkStream (std::string url, int64_t len, bool whole_download, NetworkSessionList *session_list) : url(url), len(len < 0 ? 0 : len), + block_num(get_block_num(this->len)), whole_download(whole_download), session_list(session_list) {} double get_download_percentage(); std::vector get_buffering_progress_bar(int res_len); diff --git a/source/youtube_parser/parser.hpp b/source/youtube_parser/parser.hpp index c98d9940..4836aa27 100644 --- a/source/youtube_parser/parser.hpp +++ b/source/youtube_parser/parser.hpp @@ -237,6 +237,7 @@ std::string get_video_id_from_thumbnail_url(const std::string &url); bool youtube_is_valid_video_id(const std::string &id); bool is_youtube_url(const std::string &url); bool is_youtube_thumbnail_url(const std::string &url); +int64_t extract_stream_length(const std::string &url); enum class YouTubePageType { VIDEO, diff --git a/source/youtube_parser/utils.cpp b/source/youtube_parser/utils.cpp index 97eebadb..978f887e 100644 --- a/source/youtube_parser/utils.cpp +++ b/source/youtube_parser/utils.cpp @@ -67,4 +67,13 @@ YouTubePageType youtube_get_page_type(std::string url) { if (starts_with(url, "https://m.youtube.com/results?", 0)) return YouTubePageType::SEARCH; return YouTubePageType::INVALID; } +int64_t extract_stream_length(const std::string &url) { + auto pos = url.find("&clen="); + if (pos == std::string::npos) pos = url.find("?clen="); + if (pos == std::string::npos) return -1; + pos += std::string("&clen=").size(); + int64_t res = 0; + while (pos < url.size() && isdigit(url[pos])) res = res * 10 + url[pos++] - '0'; + return res; +} diff --git a/source/youtube_parser/video.cpp b/source/youtube_parser/video.cpp index a7761b83..da182525 100644 --- a/source/youtube_parser/video.cpp +++ b/source/youtube_parser/video.cpp @@ -40,7 +40,6 @@ static bool extract_player_data(Document &json_root, RJson player_response, YouT url = url.substr(0, n_start) + next_n + url.substr(n_end, url.size() - n_end); if (url.find("ratebypass") == std::string::npos) url += "&ratebypass=yes"; - i.set_str(json_root, "url", url.c_str()); } for (auto &i : formats) i.set_str(json_root, "url", url_decode(i["url"].string_value()).c_str()); // something like %2C still appears in the url, so decode them back @@ -302,7 +301,7 @@ YouTubeVideoDetail youtube_load_video_page(std::string url) { # ifdef _WIN32 // for debug purpose, to check whether the extracted stream url is working if (res.audio_stream_url != "") { - auto tmp_data = http_get(res.audio_stream_url, {{"Range", "bytes=0-400000"}}).second; + auto tmp_data = http_get(res.audio_stream_url + "&range=0-400000").second; if (tmp_data.size() != 400001) { debug_error("!!!!!!!!!!!!!!!!!!!!! SIZE DIFFER : " + std::to_string(tmp_data.size()) + " !!!!!!!!!!!!!!!!!!!!!"); } else debug_info("----------------------- OK -----------------------"); From ba153c4b514fce0e16ae71d76a49c59d0ff245db Mon Sep 17 00:00:00 2001 From: windows-server-2003 Date: Fri, 3 Mar 2023 14:39:33 +0900 Subject: [PATCH 3/3] Move onto v0.5.3 --- Makefile | 2 +- romfs/yt_decrypter.txt | 48 +++++++++++++++++++----------------------- source/definitions.hpp | 2 +- 3 files changed, 24 insertions(+), 28 deletions(-) diff --git a/Makefile b/Makefile index 180fa339..8fc1204a 100644 --- a/Makefile +++ b/Makefile @@ -50,7 +50,7 @@ ROMFS := romfs LIBRARY := library GFXBUILD := $(ROMFS)/gfx #--------------------------------------------------------------------------------- -APP_VER := 82 +APP_VER := 83 APP_TITLE := ThirdTube APP_DESCRIPTION := A YouTube client for the 3DS APP_AUTHOR := windows_server_2003 diff --git a/romfs/yt_decrypter.txt b/romfs/yt_decrypter.txt index 4c7beb5a..f5f90552 100644 --- a/romfs/yt_decrypter.txt +++ b/romfs/yt_decrypter.txt @@ -1,30 +1,26 @@ -# 19345 -s 9 -c 3 -R -s 23 -s 35 -s 8 -c 2 +# 19417 +s 17 +c 1 R +s 65 > -t shift 0aA-_ -e -982641188 +r -800848978 +R +e 330905968 +r -1801395041 +r -45203544 +r 969944705 +s -1266939812 +s 1071587255 +r 2070384319 +t hwadna7 Aa0-_ +s 61910453 +R +t UDjUn 0aA-_ +r -1700764381 +R R -s 24255491 -e 459973519 +r 859083917 +r -2140614363 +r -307172264 R -s 1636924065 -r 594168238 -e 342788075 -s 1232619296 -s -1202923071 -s 297886161 -s 399714127 -t else 0aA-_ -s -2138817396 -s 1123509674 -r -82914077 -r 2061406551 -r 686585324 -s 1968971545 diff --git a/source/definitions.hpp b/source/definitions.hpp index 2bacee18..91d66b57 100644 --- a/source/definitions.hpp +++ b/source/definitions.hpp @@ -9,7 +9,7 @@ #define DEF_SEND_APP_INFO_URL (std::string)"https://script.google.com/macros/s/AKfycbyn_blFyKWXCgJr6NIF8x6ETs7CHRN5FXKYEAAIrzV6jPYcCkI/exec" #define DEF_CHECK_UPDATE_URL (std::string)"https://script.google.com/macros/s/AKfycbwTd3jzV0npUE9MNKmZIv3isazVR5D9_7A8rexsG1vr9SE7iavDBxgtzlph8dZipwu9/exec" #define DEF_HTTP_USER_AGENT (std::string)"video player for 3ds v1.1.0" -#define DEF_CURRENT_APP_VER (std::string)"0.5.2" +#define DEF_CURRENT_APP_VER (std::string)"0.5.3" #define DEF_CURRENT_APP_VER_INT 0 #define GITHUB_URL std::string("https://github.com/windows-server-2003/ThirdTube")