From a7b2aa947144ece46341d009223a5c701d9ebc98 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 12 Sep 2024 14:54:58 +0700 Subject: [PATCH] [wip] some update --- engine/commands/cortex_upd_cmd.cc | 13 +-- engine/commands/engine_init_cmd.cc | 154 +++++++++++++++------------- engine/controllers/engines.cc | 1 - engine/services/download_service.cc | 82 ++++----------- engine/services/download_service.h | 55 +++++----- engine/utils/cortexso_parser.h | 16 +-- engine/utils/file_manager_utils.h | 9 ++ 7 files changed, 157 insertions(+), 173 deletions(-) diff --git a/engine/commands/cortex_upd_cmd.cc b/engine/commands/cortex_upd_cmd.cc index 33d8bd3c8..612dbdb00 100644 --- a/engine/commands/cortex_upd_cmd.cc +++ b/engine/commands/cortex_upd_cmd.cc @@ -90,14 +90,11 @@ bool CortexUpdCmd::GetStableAndBeta(const std::string& v) { .items = {DownloadItem{ .id = "cortex", .fileName = file_name, - .type = DownloadType::Cortex, .path = full_url, }}}}; - DownloadService download_service; - download_service.AddDownloadTask( - download_task, - [this](const std::string& absolute_path, bool unused) { + DownloadService().AddDownloadTask( + download_task, [](const DownloadTask finishedTask) { // try to unzip the downloaded file std::filesystem::path download_path{absolute_path}; CTL_INF("Downloaded engine path: " << download_path.string()); @@ -157,13 +154,11 @@ bool CortexUpdCmd::GetNightly(const std::string& v) { .items = {DownloadItem{ .id = "cortex", .fileName = kNightlyFileName, - .type = DownloadType::Cortex, .path = release_path.str(), }}}; - DownloadService download_service; - download_service.AddDownloadTask( - download_task, [this](const std::string& absolute_path, bool unused) { + DownloadService().AddDownloadTask( + download_task, [](const DownloadTask finishedTask) { // try to unzip the downloaded file std::filesystem::path download_path{absolute_path}; CTL_INF("Downloaded engine path: " << download_path.string()); diff --git a/engine/commands/engine_init_cmd.cc b/engine/commands/engine_init_cmd.cc index 1ce8d28cd..8dd82623e 100644 --- a/engine/commands/engine_init_cmd.cc +++ b/engine/commands/engine_init_cmd.cc @@ -9,9 +9,7 @@ // clang-format on #include "utils/cuda_toolkit_utils.h" #include "utils/engine_matcher_utils.h" -#if defined(_WIN32) || defined(__linux__) #include "utils/file_manager_utils.h" -#endif namespace commands { @@ -95,67 +93,84 @@ bool EngineInitCmd::Exec() const { auto file_name = asset["name"].get(); CTL_INF("URL: " << full_url); - auto downloadTask{DownloadTask{.id = engineName_, - .type = DownloadType::Engine, - .items = {DownloadItem{ - .id = engineName_, - .fileName = file_name, - .type = DownloadType::Engine, - .path = full_url, - }}}}; + std::filesystem::path engine_folder_path = + file_manager_utils::GetContainerFolderPath( + file_manager_utils::DownloadTypeToString( + DownloadType::Engine)) / + engineName_; + + if (!std::filesystem::exists(engine_folder_path)) { + CTL_INF("Creating " << engine_folder_path.string()); + std::filesystem::create_directory(engine_folder_path); + } + + CTL_INF("Engine folder path: " << engine_folder_path.string() + << "\n"); + + auto downloadTask{DownloadTask{ + .id = engineName_, + .type = DownloadType::Engine, + .items = {DownloadItem{ + .id = engineName_, + .fileName = file_name, + .path = full_url, + .localContainerPath = engine_folder_path.string(), + }}}}; DownloadService download_service; - download_service.AddDownloadTask(downloadTask, [this]( - const std::string& - absolute_path, - bool unused) { - // try to unzip the downloaded file - std::filesystem::path downloadedEnginePath{absolute_path}; - CTL_INF( - "Downloaded engine path: " << downloadedEnginePath.string()); - - std::filesystem::path extract_path = - downloadedEnginePath.parent_path().parent_path(); - - archive_utils::ExtractArchive(downloadedEnginePath.string(), - extract_path.string()); + download_service.AddDownloadTask( + downloadTask, [](const DownloadTask finishedTask) { + // try to unzip the downloaded file + std::filesystem::path engine_zip_path = + std::filesystem::path( + finishedTask.items[0].localContainerPath) / + finishedTask.items[0].fileName; + CTL_INF("Engine zip path: " << engine_zip_path.string()); + + std::filesystem::path extract_path = + engine_zip_path.parent_path().parent_path(); + + archive_utils::ExtractArchive(engine_zip_path.string(), + extract_path.string()); #if defined(_WIN32) || defined(__linux__) - // FIXME: hacky try to copy the file. Remove this when we are able to set the library path - auto engine_path = extract_path / engineName_; - LOG_INFO << "Source path: " << engine_path.string(); - auto executable_path = - file_manager_utils::GetExecutableFolderContainerPath(); - for (const auto& entry : - std::filesystem::recursive_directory_iterator(engine_path)) { - if (entry.is_regular_file() && - entry.path().extension() != ".gz") { - std::filesystem::path relative_path = - std::filesystem::relative(entry.path(), engine_path); - std::filesystem::path destFile = - executable_path / relative_path; - - std::filesystem::create_directories(destFile.parent_path()); - std::filesystem::copy_file( - entry.path(), destFile, - std::filesystem::copy_options::overwrite_existing); - - std::cout << "Copied: " << entry.path().filename().string() - << " to " << destFile.string() << std::endl; - } - } - std::cout << "DLL copying completed successfully." << std::endl; + // FIXME: hacky try to copy the file. Remove this when we are able to set the library path + auto engine_path = extract_path / engineName_; + LOG_INFO << "Source path: " << engine_path.string(); + auto executable_path = + file_manager_utils::GetExecutableFolderContainerPath(); + for (const auto& entry : + std::filesystem::recursive_directory_iterator( + engine_path)) { + if (entry.is_regular_file() && + entry.path().extension() != ".gz") { + std::filesystem::path relative_path = + std::filesystem::relative(entry.path(), engine_path); + std::filesystem::path destFile = + executable_path / relative_path; + + std::filesystem::create_directories( + destFile.parent_path()); + std::filesystem::copy_file( + entry.path(), destFile, + std::filesystem::copy_options::overwrite_existing); + + std::cout + << "Copied: " << entry.path().filename().string() + << " to " << destFile.string() << std::endl; + } + } + std::cout << "DLL copying completed successfully." + << std::endl; #endif - // remove the downloaded file - // TODO(any) Could not delete file on Windows because it is currently hold by httplib(?) - // Not sure about other platforms - try { - std::filesystem::remove(absolute_path); - } catch (const std::exception& e) { - CTL_WRN("Could not delete file: " << e.what()); - } - CTL_INF("Finished!"); - }); + // remove the downloaded file + try { + std::filesystem::remove(engine_zip_path); + } catch (const std::exception& e) { + CTL_WRN("Could not delete file: " << e.what()); + } + CTL_INF("Finished!"); + }); if (system_info.os == "mac" || engineName_ == "cortex.onnx") { // mac and onnx engine does not require cuda toolkit return true; @@ -195,28 +210,25 @@ bool EngineInitCmd::Exec() const { return false; } - std::ostringstream cuda_toolkit_path; - cuda_toolkit_path << jan_host << "/" << "dist/cuda-dependencies/" - << cuda_driver_version << "/" << system_info.os - << "/" << cuda_toolkit_file_name; + std::ostringstream cuda_toolkit_url; + cuda_toolkit_url << jan_host << "/" << "dist/cuda-dependencies/" + << cuda_driver_version << "/" << system_info.os + << "/" << cuda_toolkit_file_name; LOG_DEBUG << "Cuda toolkit download url: " << jan_host - << cuda_toolkit_path.str(); + << cuda_toolkit_url.str(); auto downloadCudaToolkitTask{DownloadTask{ .id = download_id, .type = DownloadType::CudaToolkit, - .items = {DownloadItem{ - .id = download_id, - .fileName = cuda_toolkit_file_name, - .type = DownloadType::CudaToolkit, - .path = cuda_toolkit_path.str(), - }}, + .items = {DownloadItem{.id = download_id, + .fileName = cuda_toolkit_file_name, + .path = cuda_toolkit_url.str(), + .localContainerPath = "TODO"}}, }}; download_service.AddDownloadTask( - downloadCudaToolkitTask, - [](const std::string& absolute_path, bool unused) { + downloadCudaToolkitTask, [](const DownloadTask finishedTask) { LOG_DEBUG << "Downloaded cuda path: " << absolute_path; // try to unzip the downloaded file std::filesystem::path downloaded_path{absolute_path}; diff --git a/engine/controllers/engines.cc b/engine/controllers/engines.cc index c824df179..1fec40b16 100644 --- a/engine/controllers/engines.cc +++ b/engine/controllers/engines.cc @@ -63,7 +63,6 @@ void Engines::InstallEngine( .items = {DownloadItem{ .id = engine, .fileName = name, - .type = DownloadType::Engine, .path = full_url, }}}}; diff --git a/engine/services/download_service.cc b/engine/services/download_service.cc index 3efce4880..d5dcc87c3 100644 --- a/engine/services/download_service.cc +++ b/engine/services/download_service.cc @@ -3,20 +3,17 @@ #include #include #include -#include #include #include "download_service.h" #include "exceptions/failed_curl_exception.h" #include "exceptions/failed_init_curl_exception.h" #include "exceptions/failed_open_file_exception.h" -#include "utils/file_manager_utils.h" -#include "utils/format_utils.h" #include "utils/logging_utils.h" -void DownloadService::AddDownloadTask(const DownloadTask& task, - std::optional callback) { - +void DownloadService::AddDownloadTask( + const DownloadTask& task, + std::optional callback) { CLI_LOG("Validating download items, please wait.."); // preprocess to check if all the item are valid auto total_download_size{0}; @@ -29,14 +26,14 @@ void DownloadService::AddDownloadTask(const DownloadTask& task, throw; } } - auto total_download_size_str{ - format_utils::BytesToHumanReadable(total_download_size)}; - CLI_LOG("Download items are valid. " << "Total: " << total_download_size_str - << ". Start downloading.."); // all items are valid, start downloading for (const auto& item : task.items) { - Download(task.id, item, std::nullopt, callback); + Download(task.id, item); + } + + if (callback.has_value()) { + callback.value()(task); } } @@ -66,14 +63,16 @@ uint64_t DownloadService::GetFileSize(const std::string& url) const { } void DownloadService::AddAsyncDownloadTask( - const DownloadTask& task, std::optional callback) { - tasks.push_back(task); + const DownloadTask& task, + std::optional callback) { for (const auto& item : task.items) { std::thread([this, task, &callback, item]() { - this->Download(task.id, item, std::nullopt, callback); + this->Download(task.id, item); }).detach(); } + + // TODO: how to call the callback when all the download has finished? } size_t WriteCallback(void* ptr, size_t size, size_t nmemb, FILE* stream) { @@ -81,62 +80,32 @@ size_t WriteCallback(void* ptr, size_t size, size_t nmemb, FILE* stream) { return written; } -size_t ProgressCallback(void* ptr, curl_off_t totalToDownload, - curl_off_t nowDownloaded, curl_off_t totalUpload, - curl_off_t nowUploaded) { - if (totalToDownload > 0) { - double percentage = (nowDownloaded * 100.0) / totalToDownload; - std::cout << "Downloaded: " << percentage << "%\r" << std::flush; - } - return 0; -} - -void DownloadService::Download( - const std::string& download_id, const DownloadItem& download_item, - std::optional progress_update_callback, - std::optional download_success_callback) { - - // TODO: check the local path inside download_item and create it - auto container_folder_path{file_manager_utils::GetContainerFolderPath( - file_manager_utils::DownloadTypeToString(download_item.type))}; - CTL_INF("Container folder path: " << container_folder_path.string() << "\n"); - - auto item_folder_path{container_folder_path / - std::filesystem::path(download_id)}; - CTL_INF("itemFolderPath: " << item_folder_path.string()); - // TODO: should move this out of this function - if (!std::filesystem::exists(item_folder_path)) { - CTL_INF("Creating " << item_folder_path.string()); - std::filesystem::create_directory(item_folder_path); - } +void DownloadService::Download(const std::string& download_id, + const DownloadItem& download_item) { + std::filesystem::path ofp = + std::filesystem::path(download_item.localContainerPath) / + download_item.fileName; - auto output_file_path{item_folder_path / - std::filesystem::path(download_item.fileName)}; - CTL_INF("Absolute file output: " << output_file_path.string()); + CTL_INF("Absolute file output: " << ofp.string().c_str()); CURL* curl; FILE* file; CURLcode res; - size_t (*writeCallbackPtr)(void*, size_t, size_t, FILE*) = &WriteCallback; - size_t (*progressCallbackPtr)(void*, curl_off_t, curl_off_t, curl_off_t, - curl_off_t) = &ProgressCallback; - curl = curl_easy_init(); if (!curl) { throw FailedInitCurlException(); } - file = fopen(output_file_path.string().c_str(), "wb"); + file = fopen(ofp.string().c_str(), "wb"); if (!file) { - auto err_msg = "Failed to open output file " + output_file_path.string(); + auto err_msg{"Failed to open output file " + ofp.string()}; throw FailedOpenFileException(err_msg); } curl_easy_setopt(curl, CURLOPT_URL, download_item.path.c_str()); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallbackPtr); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &WriteCallback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, file); - curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, progressCallbackPtr); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); @@ -149,11 +118,4 @@ void DownloadService::Download( fclose(file); curl_easy_cleanup(curl); - if (download_success_callback.has_value()) { - // TODO: this should not be here - auto need_parse_gguf = - download_item.path.find("cortexso") == std::string::npos; - download_success_callback.value()(output_file_path.string(), - need_parse_gguf); - } } diff --git a/engine/services/download_service.h b/engine/services/download_service.h index 02bb9fc87..d734fc902 100644 --- a/engine/services/download_service.h +++ b/engine/services/download_service.h @@ -2,54 +2,63 @@ #include #include +#include #include enum class DownloadType { Model, Engine, Miscellaneous, CudaToolkit, Cortex }; -enum class DownloadStatus { - Pending, - Downloading, - Error, - Downloaded, -}; - struct DownloadItem { std::string id; std::string fileName; - DownloadType type; - std::string path; + std::string localContainerPath; + std::optional checksum; - // TODO: think of adding output file here. since it will leverage the logic to get the downloaded out of download service + std::string ToString() const { + std::ostringstream output; + output << "DownloadItem{id: " << id << ", fileName: " << fileName + << ", path: " << path + << ", localContainerPath: " << localContainerPath + << ", checksum: " << checksum.value_or("N/A") << "}"; + return output.str(); + } }; struct DownloadTask { std::string id; + DownloadType type; + std::vector items; std::string ToString() const { - return "DownloadTask{id: " + id + - ", type: " + std::to_string(static_cast(type)) + "}"; + std::ostringstream output; + output << "DownloadTask{id: " << id << ", type: " << static_cast(type) + << ", items: ["; + for (const auto& item : items) { + output << item.ToString() << ", "; + } + output << "]}"; + return output.str(); } }; class DownloadService { public: - using DownloadItemCb = std::function; - using ProgressUpdateCallback = std::function; - using DownloadSuccessCallback = std::function; + using OnDownloadTaskSuccessfully = + std::function; - void AddDownloadTask(const DownloadTask& task, - std::optional callback = std::nullopt); + void AddDownloadTask( + const DownloadTask& task, + std::optional callback = std::nullopt); void AddAsyncDownloadTask( const DownloadTask& task, - std::optional callback = std::nullopt); + std::optional callback = std::nullopt); /** * Getting file size for a provided url. @@ -59,12 +68,6 @@ class DownloadService { uint64_t GetFileSize(const std::string& url) const; private: - void Download( - const std::string& download_id, const DownloadItem& download_item, - std::optional progress_update_callback = - std::nullopt, - std::optional download_success_callback = std::nullopt); - - // store tasks so we can abort it later - std::vector tasks; + void Download(const std::string& download_id, + const DownloadItem& download_item); }; diff --git a/engine/utils/cortexso_parser.h b/engine/utils/cortexso_parser.h index 7167da889..56a9c78e5 100644 --- a/engine/utils/cortexso_parser.h +++ b/engine/utils/cortexso_parser.h @@ -28,6 +28,11 @@ inline std::optional getDownloadTask( auto jsonResponse = json::parse(res->body); std::vector downloadItems{}; + std::filesystem::path model_container_path = + file_manager_utils::GetModelsContainerPath() / modelId; + file_manager_utils::CreateDirectoryRecursively( + model_container_path.string()); + for (auto& [key, value] : jsonResponse.items()) { std::ostringstream downloadUrlOutput; auto path = value["path"].get(); @@ -35,12 +40,11 @@ inline std::optional getDownloadTask( << "/resolve/" << branch << "/" << path; const std::string downloadUrl = downloadUrlOutput.str(); - DownloadItem downloadItem{}; - downloadItem.id = path; - downloadItem.fileName = path; - downloadItem.type = DownloadType::Model; - downloadItem.path = downloadUrl; - downloadItems.push_back(downloadItem); + downloadItems.push_back(DownloadItem{ + .id = path, + .fileName = path, + .path = downloadUrl, + .localContainerPath = model_container_path.string()}); } DownloadTask downloadTask{ diff --git a/engine/utils/file_manager_utils.h b/engine/utils/file_manager_utils.h index a14b17feb..0911d553d 100644 --- a/engine/utils/file_manager_utils.h +++ b/engine/utils/file_manager_utils.h @@ -192,6 +192,15 @@ inline std::filesystem::path GetCortexLogPath() { return log_folder_path; } +inline void CreateDirectoryRecursively(const std::string& path) { + // Create the directories if they don't exist + if (std::filesystem::create_directories(path)) { + CTL_INF(path + " successfully created!"); + } else { + CTL_INF(path + " already exist!"); + } +} + inline std::filesystem::path GetModelsContainerPath() { auto cortex_path = GetCortexDataPath(); auto models_container_path = cortex_path / "models";