From 51ef27b103fab402b5dfb5772d91b571b3d6087a Mon Sep 17 00:00:00 2001 From: Marek Blaha Date: Fri, 15 Nov 2024 07:48:51 +0100 Subject: [PATCH] Implement reposync plugin Implements command `dnf5 reposync` for creating local copies of remote repositories. The syntax is mostly compatible with the dnf4 version of the command. --- dnf5-plugins/CMakeLists.txt | 1 + dnf5-plugins/reposync_plugin/CMakeLists.txt | 16 + .../aliases.d/compatibility-reposync.conf | 7 + .../reposync_plugin/po/CMakeLists.txt | 5 + dnf5-plugins/reposync_plugin/po/cs.po | 17 + .../po/dnf5-plugin-reposync.pot | 62 +++ dnf5-plugins/reposync_plugin/reposync.cpp | 420 ++++++++++++++++++ dnf5-plugins/reposync_plugin/reposync.hpp | 73 +++ .../reposync_plugin/reposync_cmd_plugin.cpp | 93 ++++ dnf5.spec | 9 +- doc/CMakeLists.txt | 1 + doc/changes_from_dnf4.7.rst | 3 + doc/conf.py.in | 1 + doc/dnf5.8.rst | 3 + doc/dnf5_plugins/index.rst | 1 + doc/dnf5_plugins/reposync.8.rst | 122 +++++ 16 files changed, 832 insertions(+), 2 deletions(-) create mode 100644 dnf5-plugins/reposync_plugin/CMakeLists.txt create mode 100644 dnf5-plugins/reposync_plugin/config/usr/share/dnf5/aliases.d/compatibility-reposync.conf create mode 100644 dnf5-plugins/reposync_plugin/po/CMakeLists.txt create mode 100644 dnf5-plugins/reposync_plugin/po/cs.po create mode 100644 dnf5-plugins/reposync_plugin/po/dnf5-plugin-reposync.pot create mode 100644 dnf5-plugins/reposync_plugin/reposync.cpp create mode 100644 dnf5-plugins/reposync_plugin/reposync.hpp create mode 100644 dnf5-plugins/reposync_plugin/reposync_cmd_plugin.cpp create mode 100644 doc/dnf5_plugins/reposync.8.rst diff --git a/dnf5-plugins/CMakeLists.txt b/dnf5-plugins/CMakeLists.txt index 503bfe4bc..bf3b14181 100644 --- a/dnf5-plugins/CMakeLists.txt +++ b/dnf5-plugins/CMakeLists.txt @@ -17,3 +17,4 @@ add_subdirectory("config-manager_plugin") add_subdirectory("copr_plugin") add_subdirectory("needs_restarting_plugin") add_subdirectory("repoclosure_plugin") +add_subdirectory("reposync_plugin") diff --git a/dnf5-plugins/reposync_plugin/CMakeLists.txt b/dnf5-plugins/reposync_plugin/CMakeLists.txt new file mode 100644 index 000000000..7e3f7c155 --- /dev/null +++ b/dnf5-plugins/reposync_plugin/CMakeLists.txt @@ -0,0 +1,16 @@ +# set gettext domain for translations +set(GETTEXT_DOMAIN dnf5-plugin-reposync) +add_definitions(-DGETTEXT_DOMAIN=\"${GETTEXT_DOMAIN}\") + +add_library(reposync_cmd_plugin MODULE reposync.cpp reposync_cmd_plugin.cpp) + +# disable the 'lib' prefix in order to create reposync_cmd_plugin.so +set_target_properties(reposync_cmd_plugin PROPERTIES PREFIX "") + +target_link_libraries(reposync_cmd_plugin PRIVATE libdnf5 libdnf5-cli) +target_link_libraries(reposync_cmd_plugin PRIVATE dnf5) + +install(TARGETS reposync_cmd_plugin LIBRARY DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/dnf5/plugins/) +install(DIRECTORY "config/usr/" DESTINATION "${CMAKE_INSTALL_PREFIX}") + +add_subdirectory(po) diff --git a/dnf5-plugins/reposync_plugin/config/usr/share/dnf5/aliases.d/compatibility-reposync.conf b/dnf5-plugins/reposync_plugin/config/usr/share/dnf5/aliases.d/compatibility-reposync.conf new file mode 100644 index 000000000..4290867dc --- /dev/null +++ b/dnf5-plugins/reposync_plugin/config/usr/share/dnf5/aliases.d/compatibility-reposync.conf @@ -0,0 +1,7 @@ +version = '1.0' + +['reposync.downloadpath'] +type = 'cloned_named_arg' +long_name = 'download-path' +short_name = 'p' +source = 'reposync.destdir' diff --git a/dnf5-plugins/reposync_plugin/po/CMakeLists.txt b/dnf5-plugins/reposync_plugin/po/CMakeLists.txt new file mode 100644 index 000000000..cb10bc8bd --- /dev/null +++ b/dnf5-plugins/reposync_plugin/po/CMakeLists.txt @@ -0,0 +1,5 @@ +if(NOT WITH_TRANSLATIONS) + return() +endif() + +include(Translations) diff --git a/dnf5-plugins/reposync_plugin/po/cs.po b/dnf5-plugins/reposync_plugin/po/cs.po new file mode 100644 index 000000000..cf603b198 --- /dev/null +++ b/dnf5-plugins/reposync_plugin/po/cs.po @@ -0,0 +1,17 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2024-11-06 03:02+0000\n" +"PO-Revision-Date: 2024-02-02 13:11+0000\n" +"Language-Team: Czech \n" +"Language: cs\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=3; plural=(n==1) ? 0 : (n>=2 && n<=4) ? 1 : 2;\n" +"X-Generator: Weblate 5.3.1\n" diff --git a/dnf5-plugins/reposync_plugin/po/dnf5-plugin-reposync.pot b/dnf5-plugins/reposync_plugin/po/dnf5-plugin-reposync.pot new file mode 100644 index 000000000..59428526e --- /dev/null +++ b/dnf5-plugins/reposync_plugin/po/dnf5-plugin-reposync.pot @@ -0,0 +1,62 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2024-11-21 10:44+0100\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=CHARSET\n" +"Content-Transfer-Encoding: 8bit\n" + +#: reposync.cpp:64 +msgid "Synchronize a remote DNF repository to a local directory." +msgstr "" + +#: reposync.cpp:159 +msgid "Can't use --norepopath with multiple repositories enabled" +msgstr "" + +#: reposync.cpp:243 +#, c++-format +msgid "" +"Download destination '{0}' for location '{1}' of '{2}' package from '{3}' " +"repo is outside of safe write path '{4}'." +msgstr "" + +#: reposync.cpp:280 +#, c++-format +msgid "Failed to create directory '{0}' iterator: {1}" +msgstr "" + +#: reposync.cpp:298 +#, c++-format +msgid "Failed to delete file {0}: {1}" +msgstr "" + +#: reposync.cpp:301 +#, c++-format +msgid "[DELETED] {}" +msgstr "" + +#: reposync.cpp:317 +#, c++-format +msgid "Removing '{}' with failing PGP check: {}" +msgstr "" + +#: reposync.cpp:378 +#, c++-format +msgid "Failed to get mirror for package: \"{}\"" +msgstr "" + +#: reposync.cpp:395 +msgid "PGP signature check failed" +msgstr "" diff --git a/dnf5-plugins/reposync_plugin/reposync.cpp b/dnf5-plugins/reposync_plugin/reposync.cpp new file mode 100644 index 000000000..05f2cdfbb --- /dev/null +++ b/dnf5-plugins/reposync_plugin/reposync.cpp @@ -0,0 +1,420 @@ +/* +Copyright Contributors to the libdnf project. + +This file is part of libdnf: https://github.com/rpm-software-management/libdnf/ + +Libdnf is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 2 of the License, or +(at your option) any later version. + +Libdnf is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with libdnf. If not, see . +*/ + +#include "reposync.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +static std::string join_url(const std::string & base, const std::string & path) { + if (base.back() == '/' && path.front() == '/') { + return base + path.substr(1); + } else if (base.back() != '/' && path.front() != '/') { + return base + "/" + path; + } else { + return base + path; + } +} + +} // namespace + +namespace dnf5 { + +void ReposyncCommand::set_parent_command() { + auto * arg_parser_parent_cmd = get_session().get_argument_parser().get_root_command(); + auto * arg_parser_this_cmd = get_argument_parser_command(); + arg_parser_parent_cmd->register_command(arg_parser_this_cmd); +} + +void ReposyncCommand::set_argument_parser() { + auto & ctx = get_context(); + auto & parser = ctx.get_argument_parser(); + + auto & cmd = *get_argument_parser_command(); + cmd.set_description(_("Synchronize a remote DNF repository to a local directory.")); + + auto * arch_arg = parser.add_new_named_arg("arch"); + arch_arg->set_long_name("arch"); + arch_arg->set_short_name('a'); + arch_arg->set_description("Limit downloaded packages to given architectures"); + arch_arg->set_has_value(true); + arch_arg->set_arg_value_help(",..."); + arch_arg->set_parse_hook_func([this]( + [[maybe_unused]] libdnf5::cli::ArgumentParser::NamedArg * arg, + [[maybe_unused]] const char * option, + const char * value) { + const libdnf5::OptionStringList list_value(value); + for (const auto & arch : list_value.get_value()) { + arch_option.emplace(arch); + } + return true; + }); + cmd.register_named_arg(arch_arg); + + auto * source_arg = parser.add_new_named_arg("source"); + source_arg->set_long_name("source"); + source_arg->set_description("Download source packages"); + source_arg->set_has_value(false); + source_arg->set_parse_hook_func([this]( + [[maybe_unused]] libdnf5::cli::ArgumentParser::NamedArg * arg, + [[maybe_unused]] const char * option, + [[maybe_unused]] const char * value) { + arch_option.emplace("src"); + return true; + }); + cmd.register_named_arg(source_arg); + + newest_option = std::make_unique( + *this, "newest-only", 'n', "Download only newest packages per-repo", false); + + remote_time_option = std::make_unique( + *this, "remote-time", '\0', "Set timestamps of the downloaded files according to remote side", false); + + norepopath_option = std::make_unique( + *this, "norepopath", '\0', "Don't add the reponame to the download path", false); + delete_option = std::make_unique( + *this, "delete", '\0', "Delete local packages no longer present in repository", false); + + urls_option = std::make_unique( + *this, "urls", 'u', "Print URLs where the rpms can be downloaded instead of downloading", false); + + gpgcheck_option = std::make_unique( + *this, "gpgcheck", 'g', "Remove packages that fail GPG signature checking after downloading", false); + + download_metadata_option = std::make_unique( + *this, "download-metadata", '\0', "Download all repository metadata", false); + + auto * destdir_arg = parser.add_new_named_arg("destdir"); + destdir_arg->set_long_name("destdir"); + destdir_arg->set_description("Root path under which the downloaded repositories are stored"); + destdir_arg->set_has_value(true); + destdir_arg->set_arg_value_help(""); + destdir_arg->link_value(&ctx.get_base().get_config().get_destdir_option()); + cmd.register_named_arg(destdir_arg); + + safe_write_path_option = + dynamic_cast(parser.add_init_value(std::make_unique(""))); + auto safe_write_path_arg = parser.add_new_named_arg("safe_write_path"); + safe_write_path_arg->set_long_name("safe-write-path"); + safe_write_path_arg->set_description("Filesystem path considered safe for writing"); + safe_write_path_arg->set_has_value(true); + safe_write_path_arg->set_arg_value_help(""); + safe_write_path_arg->link_value(safe_write_path_option); + cmd.register_named_arg(safe_write_path_arg); + + metadata_path_option = + dynamic_cast(parser.add_init_value(std::make_unique(""))); + auto metadata_path_arg = parser.add_new_named_arg("metadata_path"); + metadata_path_arg->set_long_name("metadata-path"); + metadata_path_arg->set_description("Root path under which the downloaded metadata are stored"); + metadata_path_arg->set_has_value(true); + metadata_path_arg->set_arg_value_help(""); + metadata_path_arg->link_value(metadata_path_option); + cmd.register_named_arg(metadata_path_arg); +} + +void ReposyncCommand::configure() { + auto & context = get_context(); + auto & base = context.get_base(); + if (arch_option.contains("src")) { + base.get_repo_sack()->enable_source_repos(); + } + + libdnf5::repo::RepoQuery repos_query(base); + repos_query.filter_enabled(true); + + if (norepopath_option->get_value() && repos_query.size() > 1) { + throw libdnf5::cli::ArgumentParserConflictingArgumentsError( + M_("Can't use --norepopath with multiple repositories enabled")); + } + + // Default destination for downloaded repos is the current directory + context.get_base().get_config().get_destdir_option().set(libdnf5::Option::Priority::DEFAULT, "."); + + const bool preserve_remote_time = remote_time_option->get_value(); + for (const auto & repo : repos_query) { + repo->set_preserve_remote_time(preserve_remote_time); + // expire all the enabled repos before downloading to ensure that the fresh + // metadata are used. + repo->expire(); + } + + context.set_load_system_repo(false); + context.set_load_available_repos(Context::LoadAvailableRepos::ENABLED); +} + + +std::filesystem::path ReposyncCommand::repo_download_path(const libdnf5::repo::Repo & repo) { + // first convert the destdir to the absolute path + std::filesystem::path repo_path = + std::filesystem::absolute(get_context().get_base().get_config().get_destdir_option().get_value()); + if (!norepopath_option->get_value()) { + repo_path /= repo.get_id(); + } + // resolve '.', '..', and existing symlinks in the repo_path + return std::filesystem::weakly_canonical(repo_path); +} + +void ReposyncCommand::limit_to_latest(libdnf5::rpm::PackageQuery & query) { + // TODO(mblaha): implement modularity support + // https://github.com/rpm-software-management/dnf5/issues/1902 + // Returned query should contain a union of these queries: + // - the latest NEVRAs from non-modular packages + // - all packages from stream version with the latest package NEVRA + // (this should not be needed but the latest package NEVRAs might be + // part of an older module version) + // - all packages from the latest stream version + + query.filter_latest_evr(); +} + +ReposyncCommand::download_list_type ReposyncCommand::get_packages_list(const libdnf5::repo::Repo & repo) { + auto & ctx = get_context(); + download_list_type result; + + const auto repo_path = repo_download_path(repo); + + // Safe path is either repository download path or --safe-write-path option value. + std::filesystem::path safe_write_path; + if (!safe_write_path_option->get_value().empty()) { + safe_write_path = + std::filesystem::weakly_canonical(std::filesystem::absolute(safe_write_path_option->get_value())); + } else { + safe_write_path = repo_path; + } + // Ensure the safe write path ends with a directory separator by appending + // an empty path. + // The download location is validated to ensure it resides within this safe + // path by checking that the download path string starts with the safe + // path. To avoid false positives, we ensure the safe path ends with a + // directory separator. For example, if "/tmp/path" is the safe path, + // "/tmp/path2/evil" would incorrectly match as within the safe path unless + // a separator is enforced. + safe_write_path /= ""; + + libdnf5::rpm::PackageQuery query(ctx.get_base(), libdnf5::sack::ExcludeFlags::IGNORE_MODULAR_EXCLUDES); + query.filter_available(); + query.filter_repo_id(repo.get_id()); + + if (newest_option->get_value()) { + limit_to_latest(query); + } + + if (!arch_option.empty()) { + query.filter_arch(std::vector(arch_option.begin(), arch_option.end())); + } + + std::unordered_set seen_paths; + for (auto pkg : query) { + auto pkg_path = std::filesystem::weakly_canonical(std::filesystem::absolute(repo_path / pkg.get_location())); + + // check that the location is safe + if (!pkg_path.string().starts_with(safe_write_path.c_str())) { + throw libdnf5::cli::CommandExitError( + 1, + M_("Download destination '{0}' for location '{1}' of '{2}' package from '{3}' repo is outside of safe " + "write path '{4}'."), + pkg_path.string(), + pkg.get_location(), + pkg.get_full_nevra(), + repo.get_id(), + safe_write_path.string()); + } + + if (seen_paths.contains(pkg_path)) { + // skip packages that would have been downloaded to the same location + continue; + } + seen_paths.emplace(pkg_path); + result.emplace_back(std::move(pkg_path), std::move(pkg)); + } + + return result; +} + +void ReposyncCommand::download_packages(const ReposyncCommand::download_list_type & pkg_list) { + libdnf5::repo::PackageDownloader downloader(get_context().get_base()); + downloader.force_keep_packages(true); + // do not stop on the first error but download as much packages as available + downloader.set_fail_fast(false); + for (const auto & [pth, pkg] : pkg_list) { + downloader.add(pkg, pth.parent_path()); + } + downloader.download(); +} + +void ReposyncCommand::delete_old_local_packages( + const libdnf5::repo::Repo & repo, const ReposyncCommand::download_list_type & pkg_list) { + const auto repo_path = repo_download_path(repo); + + std::error_code ec; + std::filesystem::recursive_directory_iterator delete_iterator(repo_path, ec); + if (ec) { + std::cerr << libdnf5::utils::sformat( + _("Failed to create directory '{0}' iterator: {1}"), repo_path.string(), ec.message()) + << std::endl; + return; + } + + std::unordered_set downloaded_paths; + for (const auto & [pth, pkg] : pkg_list) { + downloaded_paths.emplace(pth); + } + + for (const auto & entry : delete_iterator) { + if (entry.is_regular_file(ec)) { + const auto & file_path = entry.path(); + if (file_path.extension() == ".rpm" && !downloaded_paths.contains(file_path)) { + // Remove every *.rpm file that was not downloaded from the repo + std::filesystem::remove(file_path, ec); + if (ec) { + std::cerr << libdnf5::utils::sformat( + _("Failed to delete file {0}: {1}"), file_path.string(), ec.message()) + << std::endl; + } else { + std::cout << libdnf5::utils::sformat(_("[DELETED] {}"), file_path.string()) << std::endl; + } + } + } + } +} + +bool ReposyncCommand::pgp_check_packages(const download_list_type & pkg_list) { + bool ret = true; + std::error_code ec; + libdnf5::rpm::RpmSignature rpm_signature(get_context().get_base()); + for (const auto & [pth, pkg] : pkg_list) { + if (std::filesystem::exists(pth, ec)) { + auto check_result = rpm_signature.check_package_signature(pth); + if (check_result != libdnf5::rpm::RpmSignature::CheckResult::OK) { + std::cerr << libdnf5::utils::sformat( + _("Removing '{}' with failing PGP check: {}"), + pth.string(), + rpm_signature.check_result_to_string(check_result)) + << std::endl; + std::filesystem::remove(pth, ec); + ret = false; + } + } + } + return ret; +} + +void ReposyncCommand::download_metadata(libdnf5::repo::Repo & repo) { + std::filesystem::path repo_path; + const auto metadata_path = metadata_path_option->get_value(); + if (!metadata_path.empty()) { + repo_path = std::filesystem::absolute(metadata_path); + if (!norepopath_option->get_value()) { + repo_path /= repo.get_id(); + } + // resolve '.', '..', and existing symlinks in the repo_path + repo_path = std::filesystem::weakly_canonical(repo_path); + } else { + repo_path = repo_download_path(repo); + } + auto & optional_metadata_option = get_context().get_base().get_config().get_optional_metadata_types_option(); + if (!optional_metadata_option.get_value().contains(libdnf5::METADATA_TYPE_ALL)) { + optional_metadata_option.set(libdnf5::METADATA_TYPE_ALL); + } + repo.download_metadata(repo_path); +} + +void ReposyncCommand::run() { + auto & context = get_context(); + libdnf5::repo::RepoQuery repos_query(context.get_base()); + repos_query.filter_enabled(true); + std::vector schemes{"https://", "file://", "http://", "ftp://"}; + for (const auto & repo : repos_query) { + const auto pkg_list = get_packages_list(*repo); + if (urls_option->get_value()) { + if (download_metadata_option->get_value()) { + // get list of repository remote locations (mirrors + base_url) + std::vector remote_locations; + for (const auto & mirror : repo->get_mirrors()) { + remote_locations.emplace_back(mirror); + } + for (const auto & base_url : repo->get_config().get_baseurl_option().get_value()) { + remote_locations.emplace_back(base_url); + } + // find first available mirror prefering file and https schemes + std::string repo_location{}; + for (const auto & scheme : schemes) { + for (const auto & mirror : remote_locations) { + if (mirror.starts_with(scheme)) { + repo_location = mirror; + break; + } + } + if (!repo_location.empty()) { + break; + } + } + if (repo_location.empty()) { + std::cerr << libdnf5::utils::sformat(_("Failed to get mirror for metadata.")) << std::endl; + continue; + } + for (const auto & [md_type, md_location] : repo->get_metadata_locations()) { + std::cout << join_url(repo_location, md_location) << std::endl; + } + } + for (const auto & [pth, pkg] : pkg_list) { + auto urls = pkg.get_remote_locations(); + if (urls.empty()) { + std::cerr << libdnf5::utils::sformat(_("Failed to get mirror for package: \"{}\""), pkg.get_name()) + << std::endl; + continue; + } else { + std::cout << urls[0] << std::endl; + } + } + } else { + if (download_metadata_option->get_value()) { + download_metadata(*repo); + } + download_packages(pkg_list); + if (delete_option->get_value()) { + delete_old_local_packages(*repo, pkg_list); + } + if (gpgcheck_option->get_value()) { + if (!pgp_check_packages(pkg_list)) { + throw libdnf5::cli::CommandExitError(1, M_("PGP signature check failed")); + } + } + } + } +} + +} // namespace dnf5 diff --git a/dnf5-plugins/reposync_plugin/reposync.hpp b/dnf5-plugins/reposync_plugin/reposync.hpp new file mode 100644 index 000000000..89062731c --- /dev/null +++ b/dnf5-plugins/reposync_plugin/reposync.hpp @@ -0,0 +1,73 @@ +/* +Copyright Contributors to the libdnf project. + +This file is part of libdnf: https://github.com/rpm-software-management/libdnf/ + +Libdnf is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 2 of the License, or +(at your option) any later version. + +Libdnf is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with libdnf. If not, see . +*/ + + +#ifndef DNF5_PLUGINS_REPOSYNC_PLUGIN_REPOSYNC_HPP +#define DNF5_PLUGINS_REPOSYNC_PLUGIN_REPOSYNC_HPP + + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace dnf5 { + + +class ReposyncCommand : public Command { +public: + explicit ReposyncCommand(Context & context) : Command(context, "reposync") {} + void set_parent_command() override; + void set_argument_parser() override; + void configure() override; + void run() override; + +private: + using download_list_type = std::vector>; + + std::unique_ptr newest_option{nullptr}; + std::unique_ptr remote_time_option{nullptr}; + std::unique_ptr norepopath_option{nullptr}; + std::unique_ptr delete_option{nullptr}; + std::unique_ptr urls_option{nullptr}; + std::unique_ptr gpgcheck_option{nullptr}; + std::unique_ptr download_metadata_option{nullptr}; + std::unordered_set arch_option; + libdnf5::OptionString * safe_write_path_option{nullptr}; + libdnf5::OptionString * metadata_path_option{nullptr}; + + std::filesystem::path repo_download_path(const libdnf5::repo::Repo & repo); + void limit_to_latest(libdnf5::rpm::PackageQuery & query); + download_list_type get_packages_list(const libdnf5::repo::Repo & repo); + void download_packages(const download_list_type & pkg_list); + void delete_old_local_packages(const libdnf5::repo::Repo & repo, const download_list_type & pkg_list); + bool pgp_check_packages(const download_list_type & pkg_list); + void download_metadata(libdnf5::repo::Repo & repo); +}; + + +} // namespace dnf5 + + +#endif // DNF5_PLUGINS_REPOSYNC_PLUGIN_REPOSYNC_HPP diff --git a/dnf5-plugins/reposync_plugin/reposync_cmd_plugin.cpp b/dnf5-plugins/reposync_plugin/reposync_cmd_plugin.cpp new file mode 100644 index 000000000..a62a87c2f --- /dev/null +++ b/dnf5-plugins/reposync_plugin/reposync_cmd_plugin.cpp @@ -0,0 +1,93 @@ +/* +Copyright Contributors to the libdnf project. + +This file is part of libdnf: https://github.com/rpm-software-management/libdnf/ + +Libdnf is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 2 of the License, or +(at your option) any later version. + +Libdnf is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with libdnf. If not, see . +*/ + +#include "reposync.hpp" + +#include + +#include + +using namespace dnf5; + +namespace { + +constexpr const char * PLUGIN_NAME{"reposync"}; +constexpr PluginVersion PLUGIN_VERSION{.major = 1, .minor = 0, .micro = 0}; + +constexpr const char * attrs[]{"author.name", "author.email", "description", nullptr}; +constexpr const char * attrs_value[]{"Marek Blaha", "mblaha@redhat.com", "reposync command."}; + +class ReposyncCmdPlugin : public IPlugin { +public: + using IPlugin::IPlugin; + + PluginAPIVersion get_api_version() const noexcept override { return PLUGIN_API_VERSION; } + + const char * get_name() const noexcept override { return PLUGIN_NAME; } + + PluginVersion get_version() const noexcept override { return PLUGIN_VERSION; } + + const char * const * get_attributes() const noexcept override { return attrs; } + + const char * get_attribute(const char * attribute) const noexcept override { + for (size_t i = 0; attrs[i]; ++i) { + if (std::strcmp(attribute, attrs[i]) == 0) { + return attrs_value[i]; + } + } + return nullptr; + } + + std::vector> create_commands() override; + + void finish() noexcept override {} +}; + + +std::vector> ReposyncCmdPlugin::create_commands() { + std::vector> commands; + commands.push_back(std::make_unique(get_context())); + return commands; +} + + +} // namespace + + +PluginAPIVersion dnf5_plugin_get_api_version(void) { + return PLUGIN_API_VERSION; +} + +const char * dnf5_plugin_get_name(void) { + return PLUGIN_NAME; +} + +PluginVersion dnf5_plugin_get_version(void) { + return PLUGIN_VERSION; +} + +IPlugin * dnf5_plugin_new_instance([[maybe_unused]] ApplicationVersion application_version, Context & context) try { + return new ReposyncCmdPlugin(context); +} catch (...) { + return nullptr; +} + +void dnf5_plugin_delete_instance(IPlugin * plugin_object) { + delete plugin_object; +} diff --git a/dnf5.spec b/dnf5.spec index 9b302c074..c835386cc 100644 --- a/dnf5.spec +++ b/dnf5.spec @@ -722,25 +722,29 @@ Provides: dnf5-command(config-manager) Provides: dnf5-command(copr) Provides: dnf5-command(needs-restarting) Provides: dnf5-command(repoclosure) +Provides: dnf5-command(reposync) %description -n dnf5-plugins Core DNF5 plugins that enhance dnf5 with builddep, changelog, -config-manager, copr, and repoclosure commands. +config-manager, copr, repoclosure, and reposync commands. -%files -n dnf5-plugins -f dnf5-plugin-builddep.lang -f dnf5-plugin-changelog.lang -f dnf5-plugin-config-manager.lang -f dnf5-plugin-copr.lang -f dnf5-plugin-needs-restarting.lang -f dnf5-plugin-repoclosure.lang +%files -n dnf5-plugins -f dnf5-plugin-builddep.lang -f dnf5-plugin-changelog.lang -f dnf5-plugin-config-manager.lang -f dnf5-plugin-copr.lang -f dnf5-plugin-needs-restarting.lang -f dnf5-plugin-repoclosure.lang -f dnf5-plugin-reposync.lang %{_libdir}/dnf5/plugins/builddep_cmd_plugin.so %{_libdir}/dnf5/plugins/changelog_cmd_plugin.so %{_libdir}/dnf5/plugins/config-manager_cmd_plugin.so %{_libdir}/dnf5/plugins/copr_cmd_plugin.so %{_libdir}/dnf5/plugins/needs_restarting_cmd_plugin.so %{_libdir}/dnf5/plugins/repoclosure_cmd_plugin.so +%{_libdir}/dnf5/plugins/reposync_cmd_plugin.so %{_mandir}/man8/dnf*-builddep.8.* %{_mandir}/man8/dnf*-changelog.8.* %{_mandir}/man8/dnf*-config-manager.8.* %{_mandir}/man8/dnf*-copr.8.* %{_mandir}/man8/dnf*-needs-restarting.8.* %{_mandir}/man8/dnf*-repoclosure.8.* +%{_mandir}/man8/dnf*-reposync.8.* %{_datadir}/dnf5/aliases.d/compatibility-plugins.conf +%{_datadir}/dnf5/aliases.d/compatibility-reposync.conf # ========== dnf5-automatic plugin ========== @@ -886,6 +890,7 @@ popd %find_lang dnf5-plugin-copr %find_lang dnf5-plugin-needs-restarting %find_lang dnf5-plugin-repoclosure +%find_lang dnf5-plugin-reposync %find_lang dnf5daemon-client %find_lang dnf5daemon-server %find_lang libdnf5 diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 33f8d3301..6cb59ad16 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -104,6 +104,7 @@ if(WITH_MAN) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/man/dnf5-copr.8 DESTINATION share/man/man8) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/man/dnf5-needs-restarting.8 DESTINATION share/man/man8) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/man/dnf5-repoclosure.8 DESTINATION share/man/man8) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/man/dnf5-reposync.8 DESTINATION share/man/man8) endif() endif() diff --git a/doc/changes_from_dnf4.7.rst b/doc/changes_from_dnf4.7.rst index b49303786..88071c0ec 100644 --- a/doc/changes_from_dnf4.7.rst +++ b/doc/changes_from_dnf4.7.rst @@ -251,6 +251,9 @@ Changes to individual commands ``repoclosure`` * Dropped ``--pkg`` option. Positional arguments can now be used to specify packages to check closure for. +``reposync`` + * Dropped ``--downloadcomps`` option. Consider using ``--download-metadata`` option which downloads all available repository metadata, not only comps groups. + ``repolist`` * The ``repolist`` and ``repoinfo`` commands are now subcommands of the ``repo`` command: ``repo list`` and ``repo info``. diff --git a/doc/conf.py.in b/doc/conf.py.in index 32ba7971d..fff2d7869 100644 --- a/doc/conf.py.in +++ b/doc/conf.py.in @@ -149,6 +149,7 @@ man_pages = [ ('dnf5_plugins/copr.8', 'dnf5-copr', 'Copr Command', AUTHORS, 8), ('dnf5_plugins/needs_restarting.8', 'dnf5-needs-restarting', 'Needs-restarting Command', AUTHORS, 8), ('dnf5_plugins/repoclosure.8', 'dnf5-repoclosure', 'Repoclosure Command', AUTHORS, 8), + ('dnf5_plugins/reposync.8', 'dnf5-reposync', 'Reposync Command', AUTHORS, 8), ('libdnf5_plugins/actions.8', 'libdnf5-actions', 'Actions Plugin', AUTHORS, 8), ('misc/aliases.7', 'dnf5-aliases', 'Aliases for command line arguments', AUTHORS, 7), ('misc/caching.7', 'dnf5-caching', 'Caching', AUTHORS, 7), diff --git a/doc/dnf5.8.rst b/doc/dnf5.8.rst index 7064957b3..e79405d41 100644 --- a/doc/dnf5.8.rst +++ b/doc/dnf5.8.rst @@ -167,6 +167,9 @@ These are available after installing the ``dnf5-plugins`` package. :ref:`repoclosure ` | Display a list of unresolved dependencies for repositories. +:ref:`reposync ` + | Synchronize packages and metadata of a remote DNF repository to a local directory. + Options ======= diff --git a/doc/dnf5_plugins/index.rst b/doc/dnf5_plugins/index.rst index 77fbb61ab..852f17bee 100644 --- a/doc/dnf5_plugins/index.rst +++ b/doc/dnf5_plugins/index.rst @@ -15,6 +15,7 @@ DNF5 Plugins copr.8 needs_restarting.8 repoclosure.8 + reposync.8 .. # TODO(jkolarik): config-manager diff --git a/doc/dnf5_plugins/reposync.8.rst b/doc/dnf5_plugins/reposync.8.rst new file mode 100644 index 000000000..2f292ac58 --- /dev/null +++ b/doc/dnf5_plugins/reposync.8.rst @@ -0,0 +1,122 @@ +.. + Copyright Contributors to the libdnf project. + + This file is part of libdnf: https://github.com/rpm-software-management/libdnf/ + + Libdnf is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + Libdnf is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libdnf. If not, see . + +.. _reposync_plugin_ref-label: + +################## + Reposync Command +################## + +Synopsis +======== + +``dnf5 [GLOBAL OPTIONS] reposync [OPTIONS]`` + + +Description +=========== + +The ``reposync`` command creates local copies of remote repositories. It avoids re-downloading packages that are already present in the local directory. + +By default, ``reposync`` synchronizes all enabled repositories. However, you can customize the set of repositories to be synchronized using standard DNF5 options such as ``--repo``, ``--enable-repo``, or ``--disable-repo``. + + +Options +======= + +``--arch=, -a `` + Download only packages for the specified architecture. This option can be specified multiple times. The default behavior is to download packages for all architectures. + +``--delete`` + Remove local packages that are no longer present in the remote repository. + +``--destdir=`` + Specifies the root path where downloaded repositories are stored, relative + to the current working directory. Defaults to the current working + directory. Each downloaded repository will have a subdirectory named after + its ID within this path. + +``--download-metadata`` + Download all repository metadata. The downloaded copy is instantly usable + as a repository without the need to run ``createrepo_c``. When used with + ``--newest-only``, only the latest package versions are downloaded. However, + the metadata will still reference also older packages. To avoid issues caused + by missing RPM files, consider updating the metadata using ``createrepo_c --update``. + Otherwise, DNF will encounter errors when attempting to install older packages. + +``--gpgcheck, -g`` + Remove packages that fail PGP signature verification after downloading. The + command exits with a code of ``1`` if at least one package is removed. + + Note: For repositories configured with ``gpgcheck=0``, PGP signatures are not + checked, even when this option is used. + +``--metadata-path=`` + Specifies the root path where downloaded metadata files are stored. If not + specified, it defaults to the value of ``--destdir``. + +``--newest-only, -n`` + Download only the latest package versions from each repository. + +``--norepopath`` + Prevents the repository id from being added to the download path. This + option can only be used when syncing a single repository. (The default + behavior adds the repository id to the path.) + +``--remote-time`` + Attempts to set the timestamps of local downloaded files to match those on + the remote side. + +``--safe-write-path=`` + Defines the filesystem path prefix where reposync is allowed to write + files. If not specified, it defaults to the repository's download path. + This option is useful for repositories that use relative locations of + packages leading outside of the repository directory (e.g., + ``../packages_store/foo.rpm``). + + Caution: Any file under the ``safe-write-path`` can be overwritten. This option + can only be used when syncing a single repository. + +``--source`` + Downloads source packages. Equivalent to using ``--arch=src``. + +``--urls, -u`` + Prints the URLs of the files that would be downloaded without actually + downloading them. + + + +Examples +======== + +``dnf reposync --repoid=the_repo`` + Synchronize all packages from the repository with id ``the_repo``. The + synchronized copy is saved in ``the_repo`` subdirectory of the current + working directory. + +``dnf reposync --destdir=/my/repos/path --repoid=the_repo`` + Synchronize all packages from the repository with id ``the_repo``. In this + case files are saved in ``/my/repos/path/the_repo`` directory. + +``dnf reposync --repoid=the_repo --download-metadata`` + Synchronize all packages and metadata from ``the_repo`` repository. + + Repository synchronized with ``--download-metadata`` option can be directly + used with DNF for example by using ``--repofrompath`` option: + + ``dnf --repofrompath=syncedrepo,the_repo --repoid=syncedrepo list --available``