diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index 8a72e5040..ebcce5e08 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -33,6 +33,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) set(OPENSSL_USE_STATIC_LIBS TRUE) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +set(CMAKE_TOOLCHAIN_FILE "${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake") # set(CMAKE_PREFIX_PATH ${CMAKE_CURRENT_SOURCE_DIR}/build-deps/_install) # This is the critical line for installing another package @@ -106,6 +107,7 @@ find_package(CLI11 CONFIG REQUIRED) find_package(unofficial-minizip CONFIG REQUIRED) find_package(LibArchive REQUIRED) find_package(tabulate CONFIG REQUIRED) +find_package(CURL REQUIRED) # Build using CMAKE-JS if(DEFINED CMAKE_JS_INC) @@ -150,6 +152,7 @@ target_link_libraries(${PROJECT_NAME} PRIVATE CLI11::CLI11) target_link_libraries(${PROJECT_NAME} PRIVATE unofficial::minizip::minizip) target_link_libraries(${PROJECT_NAME} PRIVATE LibArchive::LibArchive) target_link_libraries(${PROJECT_NAME} PRIVATE tabulate::tabulate) +target_link_libraries(${PROJECT_NAME} PRIVATE CURL::libcurl) # Build using CMAKE-JS if(DEFINED CMAKE_JS_INC) diff --git a/engine/commands/cortex_upd_cmd.cc b/engine/commands/cortex_upd_cmd.cc index a4343e55c..33d8bd3c8 100644 --- a/engine/commands/cortex_upd_cmd.cc +++ b/engine/commands/cortex_upd_cmd.cc @@ -1,6 +1,3 @@ -// clang-format off -#include "utils/cortex_utils.h" -// clang-format on #include "cortex_upd_cmd.h" #include "httplib.h" #include "nlohmann/json.hpp" @@ -84,24 +81,18 @@ bool CortexUpdCmd::GetStableAndBeta(const std::string& v) { for (auto& asset : assets) { auto asset_name = asset["name"].get(); if (asset_name == matched_variant) { - std::string host{"https://github.com"}; - auto full_url = asset["browser_download_url"].get(); - std::string path = full_url.substr(host.length()); - - auto fileName = asset["name"].get(); + auto file_name = asset["name"].get(); CTL_INF("URL: " << full_url); - auto download_task = DownloadTask{.id = "cortex", - .type = DownloadType::Cortex, - .error = std::nullopt, - .items = {DownloadItem{ - .id = "cortex", - .host = host, - .fileName = fileName, - .type = DownloadType::Cortex, - .path = path, - }}}; + auto download_task{DownloadTask{.id = "cortex", + .type = DownloadType::Cortex, + .items = {DownloadItem{ + .id = "cortex", + .fileName = file_name, + .type = DownloadType::Cortex, + .path = full_url, + }}}}; DownloadService download_service; download_service.AddDownloadTask( @@ -157,16 +148,14 @@ bool CortexUpdCmd::GetNightly(const std::string& v) { // Download file std::string version = v.empty() ? "latest" : std::move(v); std::ostringstream release_path; - release_path << "cortex/" << version << "/" << system_info.os << "-" - << system_info.arch << "/" << kNightlyFileName; + release_path << kNightlyHost << "/cortex/" << version << "/" << system_info.os + << "-" << system_info.arch << "/" << kNightlyFileName; CTL_INF("Engine release path: " << kNightlyHost << release_path.str()); auto download_task = DownloadTask{.id = "cortex", .type = DownloadType::Cortex, - .error = std::nullopt, .items = {DownloadItem{ .id = "cortex", - .host = kNightlyHost, .fileName = kNightlyFileName, .type = DownloadType::Cortex, .path = release_path.str(), @@ -195,4 +184,4 @@ bool CortexUpdCmd::GetNightly(const std::string& v) { return ReplaceBinaryInflight(src, dst); } -} // namespace commands \ No newline at end of file +} // namespace commands diff --git a/engine/commands/engine_init_cmd.cc b/engine/commands/engine_init_cmd.cc index 7b0153f69..1ce8d28cd 100644 --- a/engine/commands/engine_init_cmd.cc +++ b/engine/commands/engine_init_cmd.cc @@ -91,24 +91,18 @@ bool EngineInitCmd::Exec() const { for (auto& asset : assets) { auto assetName = asset["name"].get(); if (assetName == matched_variant) { - std::string host{"https://github.com"}; - auto full_url = asset["browser_download_url"].get(); - std::string path = full_url.substr(host.length()); - - auto fileName = asset["name"].get(); + auto file_name = asset["name"].get(); CTL_INF("URL: " << full_url); - auto downloadTask = DownloadTask{.id = engineName_, - .type = DownloadType::Engine, - .error = std::nullopt, - .items = {DownloadItem{ - .id = engineName_, - .host = host, - .fileName = fileName, - .type = DownloadType::Engine, - .path = path, - }}}; + auto downloadTask{DownloadTask{.id = engineName_, + .type = DownloadType::Engine, + .items = {DownloadItem{ + .id = engineName_, + .fileName = file_name, + .type = DownloadType::Engine, + .path = full_url, + }}}}; DownloadService download_service; download_service.AddDownloadTask(downloadTask, [this]( @@ -202,25 +196,23 @@ bool EngineInitCmd::Exec() const { } std::ostringstream cuda_toolkit_path; - cuda_toolkit_path << "dist/cuda-dependencies/" + cuda_toolkit_path << jan_host << "/" << "dist/cuda-dependencies/" << cuda_driver_version << "/" << system_info.os << "/" << cuda_toolkit_file_name; LOG_DEBUG << "Cuda toolkit download url: " << jan_host << cuda_toolkit_path.str(); - auto downloadCudaToolkitTask = DownloadTask{ + auto downloadCudaToolkitTask{DownloadTask{ .id = download_id, .type = DownloadType::CudaToolkit, - .error = std::nullopt, .items = {DownloadItem{ .id = download_id, - .host = jan_host, .fileName = cuda_toolkit_file_name, .type = DownloadType::CudaToolkit, .path = cuda_toolkit_path.str(), }}, - }; + }}; download_service.AddDownloadTask( downloadCudaToolkitTask, diff --git a/engine/controllers/command_line_parser.cc b/engine/controllers/command_line_parser.cc index 987aa693a..d3fb0cf0c 100644 --- a/engine/controllers/command_line_parser.cc +++ b/engine/controllers/command_line_parser.cc @@ -219,7 +219,7 @@ void CommandLineParser::EngineInstall(CLI::App* parent, std::string& version) { auto install_engine_cmd = parent->add_subcommand(engine_name, ""); - install_engine_cmd->callback([&] { + install_engine_cmd->callback([=] { commands::EngineInitCmd eic(engine_name, version); eic.Exec(); }); @@ -229,7 +229,7 @@ void CommandLineParser::EngineUninstall(CLI::App* parent, const std::string& engine_name) { auto uninstall_engine_cmd = parent->add_subcommand(engine_name, ""); - uninstall_engine_cmd->callback([&] { + uninstall_engine_cmd->callback([=] { commands::EngineUninstallCmd cmd(engine_name); cmd.Exec(); }); diff --git a/engine/controllers/engines.cc b/engine/controllers/engines.cc index dd7ba3036..c824df179 100644 --- a/engine/controllers/engines.cc +++ b/engine/controllers/engines.cc @@ -54,24 +54,18 @@ void Engines::InstallEngine( for (auto& asset : assets) { auto assetName = asset["name"].get(); if (assetName.find(os_arch) != std::string::npos) { - std::string host{"https://github.com"}; - auto full_url = asset["browser_download_url"].get(); - std::string path = full_url.substr(host.length()); - - auto fileName = asset["name"].get(); + auto name = asset["name"].get(); LOG_INFO << "URL: " << full_url; - auto downloadTask = DownloadTask{.id = engine, - .type = DownloadType::Engine, - .error = std::nullopt, - .items = {DownloadItem{ - .id = engine, - .host = host, - .fileName = fileName, - .type = DownloadType::Engine, - .path = path, - }}}; + auto downloadTask{DownloadTask{.id = engine, + .type = DownloadType::Engine, + .items = {DownloadItem{ + .id = engine, + .fileName = name, + .type = DownloadType::Engine, + .path = full_url, + }}}}; DownloadService().AddAsyncDownloadTask( downloadTask, diff --git a/engine/exceptions/failed_curl_exception.h b/engine/exceptions/failed_curl_exception.h new file mode 100644 index 000000000..095824334 --- /dev/null +++ b/engine/exceptions/failed_curl_exception.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +class FailedCurlException : public std::runtime_error { + + public: + FailedCurlException(const std::string& message) + : std::runtime_error(message) {} +}; diff --git a/engine/exceptions/failed_init_curl_exception.h b/engine/exceptions/failed_init_curl_exception.h new file mode 100644 index 000000000..1843f9b80 --- /dev/null +++ b/engine/exceptions/failed_init_curl_exception.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +class FailedInitCurlException : public std::runtime_error { + constexpr static auto kErrorMessage = "Failed to init CURL"; + + public: + FailedInitCurlException() : std::runtime_error(kErrorMessage) {} +}; diff --git a/engine/exceptions/failed_open_file_exception.h b/engine/exceptions/failed_open_file_exception.h new file mode 100644 index 000000000..8d014ead4 --- /dev/null +++ b/engine/exceptions/failed_open_file_exception.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +class FailedOpenFileException : public std::runtime_error { + + public: + FailedOpenFileException(const std::string& message) + : std::runtime_error(message) {} +}; diff --git a/engine/services/download_service.cc b/engine/services/download_service.cc index 78a0c4757..3efce4880 100644 --- a/engine/services/download_service.cc +++ b/engine/services/download_service.cc @@ -1,96 +1,159 @@ +#include #include +#include #include #include -#include #include #include #include "download_service.h" +#include "exceptions/failed_curl_exception.h" +#include "exceptions/failed_init_curl_exception.h" +#include "exceptions/failed_open_file_exception.h" #include "utils/file_manager_utils.h" +#include "utils/format_utils.h" #include "utils/logging_utils.h" void DownloadService::AddDownloadTask(const DownloadTask& task, std::optional callback) { - tasks.push_back(task); + CLI_LOG("Validating download items, please wait.."); + // preprocess to check if all the item are valid + auto total_download_size{0}; + for (const auto& item : task.items) { + try { + total_download_size += GetFileSize(item.path); + } catch (const FailedCurlException& e) { + CTL_ERR("Found invalid download item: " << item.path << " - " + << e.what()); + throw; + } + } + auto total_download_size_str{ + format_utils::BytesToHumanReadable(total_download_size)}; + CLI_LOG("Download items are valid. " << "Total: " << total_download_size_str + << ". Start downloading.."); + + // all items are valid, start downloading for (const auto& item : task.items) { - StartDownloadItem(task.id, item, callback); + Download(task.id, item, std::nullopt, callback); } } +uint64_t DownloadService::GetFileSize(const std::string& url) const { + CURL* curl; + curl = curl_easy_init(); + + if (!curl) { + throw FailedInitCurlException(); + } + + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + CURLcode res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + // if we have a failed here. it meant the url is invalid + throw FailedCurlException("CURL failed: " + + std::string(curl_easy_strerror(res))); + } + + curl_off_t content_length = 0; + res = curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, + &content_length); + return content_length; +} + void DownloadService::AddAsyncDownloadTask( const DownloadTask& task, std::optional callback) { tasks.push_back(task); for (const auto& item : task.items) { - // TODO: maybe apply std::async is better? std::thread([this, task, &callback, item]() { - this->StartDownloadItem(task.id, item, callback); + this->Download(task.id, item, std::nullopt, callback); }).detach(); } } -void DownloadService::StartDownloadItem( - const std::string& downloadId, const DownloadItem& item, - std::optional callback) { - CTL_INF("Downloading item: " << downloadId); +size_t WriteCallback(void* ptr, size_t size, size_t nmemb, FILE* stream) { + size_t written = fwrite(ptr, size, nmemb, stream); + return written; +} - auto containerFolderPath{file_manager_utils::GetContainerFolderPath( - file_manager_utils::downloadTypeToString(item.type))}; - CTL_INF("Container folder path: " << containerFolderPath.string() << "\n"); +size_t ProgressCallback(void* ptr, curl_off_t totalToDownload, + curl_off_t nowDownloaded, curl_off_t totalUpload, + curl_off_t nowUploaded) { + if (totalToDownload > 0) { + double percentage = (nowDownloaded * 100.0) / totalToDownload; + std::cout << "Downloaded: " << percentage << "%\r" << std::flush; + } + return 0; +} - auto itemFolderPath{containerFolderPath / std::filesystem::path(downloadId)}; - CTL_INF("itemFolderPath: " << itemFolderPath.string()); - if (!std::filesystem::exists(itemFolderPath)) { - CTL_INF("Creating " << itemFolderPath.string()); - std::filesystem::create_directory(itemFolderPath); +void DownloadService::Download( + const std::string& download_id, const DownloadItem& download_item, + std::optional progress_update_callback, + std::optional download_success_callback) { + + // TODO: check the local path inside download_item and create it + auto container_folder_path{file_manager_utils::GetContainerFolderPath( + file_manager_utils::DownloadTypeToString(download_item.type))}; + CTL_INF("Container folder path: " << container_folder_path.string() << "\n"); + + auto item_folder_path{container_folder_path / + std::filesystem::path(download_id)}; + CTL_INF("itemFolderPath: " << item_folder_path.string()); + // TODO: should move this out of this function + if (!std::filesystem::exists(item_folder_path)) { + CTL_INF("Creating " << item_folder_path.string()); + std::filesystem::create_directory(item_folder_path); + } + + auto output_file_path{item_folder_path / + std::filesystem::path(download_item.fileName)}; + CTL_INF("Absolute file output: " << output_file_path.string()); + + CURL* curl; + FILE* file; + CURLcode res; + + size_t (*writeCallbackPtr)(void*, size_t, size_t, FILE*) = &WriteCallback; + size_t (*progressCallbackPtr)(void*, curl_off_t, curl_off_t, curl_off_t, + curl_off_t) = &ProgressCallback; + + curl = curl_easy_init(); + if (!curl) { + throw FailedInitCurlException(); } - auto outputFilePath{itemFolderPath / std::filesystem::path(item.fileName)}; - CTL_INF("Absolute file output: " << outputFilePath.string()); - - uint64_t last = 0; - uint64_t tot = 0; - std::ofstream outputFile(outputFilePath, std::ios::binary); - - auto downloadUrl{item.host + "/" + item.path}; - CLI_LOG("Downloading url: " << downloadUrl); - - httplib::Client client(item.host); - - client.set_follow_location(true); - client.Get( - downloadUrl, - [](const httplib::Response& res) { - if (res.status != httplib::StatusCode::OK_200) { - LOG_ERROR << "HTTP error: " << res.reason; - return false; - } - return true; - }, - [&](const char* data, size_t data_length) { - tot += data_length; - outputFile.write(data, data_length); - return true; - }, - [&item, &last, &outputFile, &callback, outputFilePath, this]( - uint64_t current, uint64_t total) { - if (current - last > kUpdateProgressThreshold) { - last = current; - CLI_LOG("Downloading: " << current << " / " << total); - } - if (current == total) { - outputFile.flush(); - outputFile.close(); - CLI_LOG("Done download: " << static_cast(total) / 1024 / 1024 - << " MiB"); - if (callback.has_value()) { - auto need_parse_gguf = - item.path.find("cortexso") == std::string::npos; - callback.value()(outputFilePath.string(), need_parse_gguf); - } - return false; - } - return true; - }); + file = fopen(output_file_path.string().c_str(), "wb"); + if (!file) { + auto err_msg = "Failed to open output file " + output_file_path.string(); + throw FailedOpenFileException(err_msg); + } + + curl_easy_setopt(curl, CURLOPT_URL, download_item.path.c_str()); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallbackPtr); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, file); + curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, progressCallbackPtr); + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + + res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + fprintf(stderr, "curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + } + + fclose(file); + curl_easy_cleanup(curl); + if (download_success_callback.has_value()) { + // TODO: this should not be here + auto need_parse_gguf = + download_item.path.find("cortexso") == std::string::npos; + download_success_callback.value()(output_file_path.string(), + need_parse_gguf); + } } diff --git a/engine/services/download_service.h b/engine/services/download_service.h index b3c405c9a..02bb9fc87 100644 --- a/engine/services/download_service.h +++ b/engine/services/download_service.h @@ -16,8 +16,6 @@ enum class DownloadStatus { struct DownloadItem { std::string id; - std::string host; - std::string fileName; DownloadType type; @@ -25,23 +23,27 @@ struct DownloadItem { std::string path; std::optional checksum; + + // TODO: think of adding output file here. since it will leverage the logic to get the downloaded out of download service }; struct DownloadTask { std::string id; DownloadType type; - std::optional error; std::vector items; + + std::string ToString() const { + return "DownloadTask{id: " + id + + ", type: " + std::to_string(static_cast(type)) + "}"; + } }; class DownloadService { public: - /** - * @brief Synchronously download. - * - * @param task - */ using DownloadItemCb = std::function; + using ProgressUpdateCallback = std::function; + using DownloadSuccessCallback = std::function; + void AddDownloadTask(const DownloadTask& task, std::optional callback = std::nullopt); @@ -49,18 +51,20 @@ class DownloadService { const DownloadTask& task, std::optional callback = std::nullopt); - // TODO: [NamH] implement the following methods - // void removeTask(const std::string &id); - // void registerCallback - // setup folder path at runtime - // register action after downloaded + /** + * Getting file size for a provided url. + * + * @param url - url to get file size + */ + uint64_t GetFileSize(const std::string& url) const; private: - void StartDownloadItem(const std::string& downloadId, - const DownloadItem& item, - std::optional callback = std::nullopt); + void Download( + const std::string& download_id, const DownloadItem& download_item, + std::optional progress_update_callback = + std::nullopt, + std::optional download_success_callback = std::nullopt); // store tasks so we can abort it later std::vector tasks; - const int kUpdateProgressThreshold = 100000000; -}; \ No newline at end of file +}; diff --git a/engine/utils/cortexso_parser.h b/engine/utils/cortexso_parser.h index 91efa1fff..7167da889 100644 --- a/engine/utils/cortexso_parser.h +++ b/engine/utils/cortexso_parser.h @@ -31,24 +31,22 @@ inline std::optional getDownloadTask( for (auto& [key, value] : jsonResponse.items()) { std::ostringstream downloadUrlOutput; auto path = value["path"].get(); - downloadUrlOutput << repoAndModelIdStr << "/resolve/" << branch << "/" - << path; + downloadUrlOutput << kHuggingFaceHost << "/" << repoAndModelIdStr + << "/resolve/" << branch << "/" << path; const std::string downloadUrl = downloadUrlOutput.str(); DownloadItem downloadItem{}; downloadItem.id = path; - downloadItem.host = kHuggingFaceHost; downloadItem.fileName = path; downloadItem.type = DownloadType::Model; downloadItem.path = downloadUrl; downloadItems.push_back(downloadItem); } - DownloadTask downloadTask{}; - downloadTask.id = branch == "main" ? modelId : modelId + "-" + branch; - downloadTask.type = DownloadType::Model; - downloadTask.error = std::nullopt; - downloadTask.items = downloadItems; + DownloadTask downloadTask{ + .id = branch == "main" ? modelId : modelId + "-" + branch, + .type = DownloadType::Model, + .items = downloadItems}; return downloadTask; } catch (const json::parse_error& e) { @@ -61,4 +59,4 @@ inline std::optional getDownloadTask( } return std::nullopt; } -} // namespace cortexso_parser \ No newline at end of file +} // namespace cortexso_parser diff --git a/engine/utils/file_manager_utils.h b/engine/utils/file_manager_utils.h index b4d7ab07a..a14b17feb 100644 --- a/engine/utils/file_manager_utils.h +++ b/engine/utils/file_manager_utils.h @@ -27,10 +27,10 @@ inline std::filesystem::path GetExecutableFolderContainerPath() { uint32_t size = sizeof(buffer); if (_NSGetExecutablePath(buffer, &size) == 0) { - LOG_INFO << "Executable path: " << buffer; + CTL_INF("Executable path: " << buffer); return std::filesystem::path{buffer}.parent_path(); } else { - LOG_ERROR << "Failed to get executable path"; + CTL_ERR("Failed to get executable path"); return std::filesystem::current_path(); } #elif defined(__linux__) @@ -110,7 +110,7 @@ inline std::filesystem::path GetConfigurationPath() { } inline void CreateConfigFileIfNotExist() { - auto config_path = file_manager_utils::GetConfigurationPath(); + auto config_path = GetConfigurationPath(); if (std::filesystem::exists(config_path)) { // already exists return; @@ -243,7 +243,7 @@ inline std::filesystem::path GetContainerFolderPath( return container_folder_path; } -inline std::string downloadTypeToString(DownloadType type) { +inline std::string DownloadTypeToString(DownloadType type) { switch (type) { case DownloadType::Model: return "Model"; diff --git a/engine/utils/format_utils.h b/engine/utils/format_utils.h new file mode 100644 index 000000000..7824d138c --- /dev/null +++ b/engine/utils/format_utils.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include + +namespace format_utils { +inline std::string BytesToHumanReadable(uint64_t bytes) { + const uint64_t KB = 1024; + const uint64_t MB = KB * 1024; + const uint64_t GB = MB * 1024; + const uint64_t TB = GB * 1024; + + double result; + std::string unit; + + if (bytes >= TB) { + result = static_cast(bytes) / TB; + unit = "TB"; + } else if (bytes >= GB) { + result = static_cast(bytes) / GB; + unit = "GB"; + } else if (bytes >= MB) { + result = static_cast(bytes) / MB; + unit = "MB"; + } else if (bytes >= KB) { + result = static_cast(bytes) / KB; + unit = "KB"; + } else { + result = static_cast(bytes); + unit = "B"; + } + + std::ostringstream out; + // take 2 decimal points + out << std::fixed << std::setprecision(2) << result << " " << unit; + return out.str(); +} +} // namespace format_utils diff --git a/engine/vcpkg.json b/engine/vcpkg.json index fe4783ec8..8f7729524 100644 --- a/engine/vcpkg.json +++ b/engine/vcpkg.json @@ -1,5 +1,6 @@ { "dependencies": [ + "curl", "gtest", "cli11", {