From 15f7bf6625d010a8d425806db67ee4e3f285ced5 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Wed, 30 Aug 2023 17:33:27 +0200 Subject: [PATCH] Parse repodata.json directly into libsolv --- libmamba/src/core/repo.cpp | 256 ++++++++++++++++++++++++++++--------- 1 file changed, 199 insertions(+), 57 deletions(-) diff --git a/libmamba/src/core/repo.cpp b/libmamba/src/core/repo.cpp index 8f5070b2e0..7cee70c8de 100644 --- a/libmamba/src/core/repo.cpp +++ b/libmamba/src/core/repo.cpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include @@ -28,7 +30,6 @@ extern "C" // Incomplete header #include "mamba/core/prefix_data.hpp" #include "mamba/core/repo.hpp" #include "mamba/core/util.hpp" -#include "mamba/specs/repo_data.hpp" #include "mamba/util/url_manip.hpp" #include "solv-cpp/pool.hpp" #include "solv-cpp/repo.hpp" @@ -132,12 +133,27 @@ namespace mamba solv.add_self_provide(); } - void set_solvable( + template + void deserialize_maybe_missing(Json&& j, std::string_view name, T& t) + { + if (j.contains(name)) + { + t = std::forward(j)[name].template get(); + } + else + { + t = {}; + } + } + + [[nodiscard]] bool set_solvable( MPool& pool, solv::ObjSolvableView solv, const std::string& repo_url, const std::string& filename, - const specs::RepoDataPackage& pkg + const std::string& default_subdir, + const nl::json& pkg, + std::string& tmp_buffer ) { // Not available from RepoDataPackage @@ -145,70 +161,190 @@ namespace mamba solv.set_url(util::join_url(repo_url, filename)); solv.set_channel(repo_url); - solv.set_name(pkg.name); - solv.set_version(pkg.version.str()); - solv.set_build_string(pkg.build_string); - // Un parse Noarch - if (pkg.noarch == specs::NoArchType::Generic) + if (auto iter = pkg.find("name"); (iter != pkg.end()) && iter->is_string()) { - solv.set_noarch("generic"); + solv.set_name(iter->get()); } - else if (pkg.noarch == specs::NoArchType::Python) + else { - solv.set_noarch("python"); + LOG_WARNING << R"(Found invalid name in ")" << filename << R"(")"; + return false; } - solv.set_build_number(pkg.build_number); - if (pkg.subdir) + + if (auto iter = pkg.find("version"); (iter != pkg.end()) && iter->is_string()) + { + solv.set_version(iter->get()); + } + else + { + LOG_WARNING << R"(Found invalid version in ")" << filename << R"(")"; + return false; + } + + if (auto iter = pkg.find("build"); (iter != pkg.end()) && iter->is_string()) + { + solv.set_build_string(iter->get()); + } + else + { + LOG_WARNING << R"(Found invalid build in ")" << filename << R"(")"; + return false; + } + + if (auto iter = pkg.find("build_number"); (iter != pkg.end()) && iter->is_number()) + { + solv.set_build_number(iter->get()); + } + else + { + LOG_WARNING << R"(Found invalid build_number in ")" << filename << R"(")"; + return false; + } + + if (auto iter = pkg.find("subdir"); (iter != pkg.end()) && iter->is_string()) + { + tmp_buffer = iter->get(); + solv.set_subdir(tmp_buffer); + } + else { - solv.set_subdir(*pkg.subdir); + solv.set_subdir(default_subdir); } - if (pkg.license) + + if (auto iter = pkg.find("size"); (iter != pkg.end()) && iter->is_number()) { - solv.set_license(*pkg.license); + solv.set_size(iter->get()); } - if (pkg.size) + + if (auto iter = pkg.find("md5"); (iter != pkg.end()) && iter->is_string()) { - solv.set_size(*pkg.size); + tmp_buffer = iter->get(); + solv.set_md5(tmp_buffer); } + + if (auto iter = pkg.find("sha256"); (iter != pkg.end()) && iter->is_string()) + { + tmp_buffer = iter->get(); + solv.set_sha256(tmp_buffer); + } + + if (auto iter = pkg.find("noarch"); (iter != pkg.end()) && !iter->is_null()) + { + if (iter->is_boolean() && iter->get()) + { + solv.set_noarch("generic"); + } + if (iter->is_string()) + { + tmp_buffer = iter->get(); + solv.set_noarch(tmp_buffer); + } + } + + if (auto iter = pkg.find("license"); (iter != pkg.end()) && iter->is_string()) + { + tmp_buffer = iter->get(); + solv.set_license(tmp_buffer); + } + // TODO conda timestamp are not Unix timestamp. // Libsolv normalize them this way, we need to do the same here otherwise the current // package may get arbitrary priority. - if (pkg.timestamp) + if (auto iter = pkg.find("timestamp"); (iter != pkg.end()) && iter->is_number()) { - solv.set_timestamp( - (*pkg.timestamp > 253402300799ULL) ? (*pkg.timestamp / 1000) : *pkg.timestamp - ); + const auto time = iter->get(); + solv.set_timestamp((time > 253402300799ULL) ? (time / 1000) : time); } - if (pkg.md5) + + if (auto iter = pkg.find("depends"); (iter != pkg.end()) && iter->is_array()) { - solv.set_md5(*pkg.md5); + for (const auto& dep : *iter) + { + if (dep.is_string()) + { + tmp_buffer = dep.get(); + if (const auto dep_id = pool_conda_matchspec(pool, tmp_buffer.c_str())) + { + solv.add_dependency(dep_id); + } + } + } } - if (pkg.sha256) + + if (auto iter = pkg.find("depends"); (iter != pkg.end()) && iter->is_array()) { - solv.set_sha256(*pkg.sha256); + for (const auto& dep : *iter) + { + if (dep.is_string()) + { + tmp_buffer = dep.get(); + if (const auto dep_id = pool_conda_matchspec(pool, tmp_buffer.c_str())) + { + solv.add_dependency(dep_id); + } + } + } } - for (const auto& dep : pkg.depends) + if (auto iter = pkg.find("constrains"); (iter != pkg.end()) && iter->is_array()) { - // TODO pool's matchspec2id - solv::DependencyId const dep_id = pool_conda_matchspec(pool, dep.c_str()); - assert(dep_id); - solv.add_dependency(dep_id); + for (const auto& dep : *iter) + { + if (dep.is_string()) + { + tmp_buffer = dep.get(); + if (const auto dep_id = pool_conda_matchspec(pool, tmp_buffer.c_str())) + { + solv.add_dependency(dep_id); + } + } + } } - for (const auto& cons : pkg.constrains) + if (auto iter = pkg.find("track_features"); iter != pkg.end()) { - // TODO pool's matchspec2id - solv::DependencyId const dep_id = pool_conda_matchspec(pool, cons.c_str()); - assert(dep_id); - solv.add_constraint(dep_id); + if (iter->is_array()) + { + for (const auto& feat : *iter) + { + if (feat.is_string()) + { + solv.add_track_feature(feat.get()); + } + } + } } - solv.add_track_features(pkg.track_features); - solv.add_self_provide(); + return true; } + void set_repo_solvables( + MPool& pool, + solv::ObjRepoView repo, + const std::string& repo_url, + const std::string& default_subdir, + const nl::json& packages, + std::string& tmp_buffer + ) + { + for (const auto& [fn, pkg] : packages.items()) + { + auto [id, solv] = repo.add_solvable(); + const bool parsed = set_solvable(pool, solv, repo_url, fn, default_subdir, pkg, tmp_buffer); + if (parsed) + { + LOG_INFO << "Adding package record to repo " << fn; + } + else + { + repo.remove_solvable(id, /* reuse_id= */ true); + LOG_WARNING << "Failed to parse from repodata " << fn; + } + } + } + + void set_solvables_url(solv::ObjRepoView repo, const std::string& repo_url) { // WARNING cannot call ``url()`` at this point because it has not been internalized. @@ -343,23 +479,35 @@ namespace mamba << " using mamba"; auto filename_stream = std::ifstream(filename.std_path()); - const auto repodata = nl::json::parse(filename_stream).get(); + const auto repodata = nl::json::parse(filename_stream); - for (const auto& [fn, pkg] : repodata.packages) - { - LOG_INFO << "Adding package record to repo " << fn; - auto [id, solv] = srepo(*this).add_solvable(); - set_solvable(m_pool, solv, m_metadata.url, fn, pkg); - } + // An override for missing package subdir is found in at the top level + std::string default_subdir = repodata.value( + nl::json::json_pointer("/info/subdir"), + "" + ); + // An temporary buffer for managing null terminated strings + std::string tmp_buffer = {}; + + set_repo_solvables( + m_pool, + srepo(*this), + m_metadata.url, + default_subdir, + repodata.at("packages"), + tmp_buffer + ); if (!Context::instance().use_only_tar_bz2) { - for (const auto& [fn, pkg] : repodata.conda_packages) - { - LOG_INFO << "Adding package record to repo " << fn; - auto [id, solv] = srepo(*this).add_solvable(); - set_solvable(m_pool, solv, m_metadata.url, fn, pkg); - } + set_repo_solvables( + m_pool, + srepo(*this), + m_metadata.url, + default_subdir, + repodata.at("packages.conda"), + tmp_buffer + ); } } @@ -481,12 +629,6 @@ namespace mamba { m_pool.remove_repo(id(), reuse_ids); } -} - -#include - -namespace mamba -{ auto MRepo::py_name() const -> std::string_view {