Skip to content

Commit

Permalink
Parse repodata.json directly into libsolv
Browse files Browse the repository at this point in the history
  • Loading branch information
@AntoinePrv
AntoinePrv committed Aug 31, 2023
1 parent 8dba3de commit 15f7bf6
Showing 1 changed file with 199 additions and 57 deletions.
256 changes: 199 additions & 57 deletions libmamba/src/core/repo.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
#include <algorithm>
#include <array>
#include <fstream>
#include <map>
#include <string_view>
#include <tuple>

#include <nlohmann/json.hpp>
Expand All @@ -28,7 +30,6 @@ extern "C" // Incomplete header
#include "mamba/core/prefix_data.hpp"
#include "mamba/core/repo.hpp"
#include "mamba/core/util.hpp"
#include "mamba/specs/repo_data.hpp"
#include "mamba/util/url_manip.hpp"
#include "solv-cpp/pool.hpp"
#include "solv-cpp/repo.hpp"
Expand Down Expand Up @@ -132,83 +133,218 @@ namespace mamba
solv.add_self_provide();
}

void set_solvable(
template <typename Json, typename T>
void deserialize_maybe_missing(Json&& j, std::string_view name, T& t)
{
if (j.contains(name))
{
t = std::forward<Json>(j)[name].template get<T>();
}
else
{
t = {};
}
}

[[nodiscard]] bool set_solvable(
MPool& pool,
solv::ObjSolvableView solv,
const std::string& repo_url,
const std::string& filename,
const specs::RepoDataPackage& pkg
const std::string& default_subdir,
const nl::json& pkg,
std::string& tmp_buffer
)
{
// Not available from RepoDataPackage
solv.set_file_name(filename);
solv.set_url(util::join_url(repo_url, filename));
solv.set_channel(repo_url);

solv.set_name(pkg.name);
solv.set_version(pkg.version.str());
solv.set_build_string(pkg.build_string);
// Un parse Noarch
if (pkg.noarch == specs::NoArchType::Generic)
if (auto iter = pkg.find("name"); (iter != pkg.end()) && iter->is_string())
{
solv.set_noarch("generic");
solv.set_name(iter->get<std::string_view>());
}
else if (pkg.noarch == specs::NoArchType::Python)
else
{
solv.set_noarch("python");
LOG_WARNING << R"(Found invalid name in ")" << filename << R"(")";
return false;
}
solv.set_build_number(pkg.build_number);
if (pkg.subdir)

if (auto iter = pkg.find("version"); (iter != pkg.end()) && iter->is_string())
{
solv.set_version(iter->get<std::string_view>());
}
else
{
LOG_WARNING << R"(Found invalid version in ")" << filename << R"(")";
return false;
}

if (auto iter = pkg.find("build"); (iter != pkg.end()) && iter->is_string())
{
solv.set_build_string(iter->get<std::string_view>());
}
else
{
LOG_WARNING << R"(Found invalid build in ")" << filename << R"(")";
return false;
}

if (auto iter = pkg.find("build_number"); (iter != pkg.end()) && iter->is_number())
{
solv.set_build_number(iter->get<std::size_t>());
}
else
{
LOG_WARNING << R"(Found invalid build_number in ")" << filename << R"(")";
return false;
}

if (auto iter = pkg.find("subdir"); (iter != pkg.end()) && iter->is_string())
{
tmp_buffer = iter->get<std::string_view>();
solv.set_subdir(tmp_buffer);
}
else
{
solv.set_subdir(*pkg.subdir);
solv.set_subdir(default_subdir);
}
if (pkg.license)

if (auto iter = pkg.find("size"); (iter != pkg.end()) && iter->is_number())
{
solv.set_license(*pkg.license);
solv.set_size(iter->get<std::size_t>());
}
if (pkg.size)

if (auto iter = pkg.find("md5"); (iter != pkg.end()) && iter->is_string())
{
solv.set_size(*pkg.size);
tmp_buffer = iter->get<std::string_view>();
solv.set_md5(tmp_buffer);
}

if (auto iter = pkg.find("sha256"); (iter != pkg.end()) && iter->is_string())
{
tmp_buffer = iter->get<std::string_view>();
solv.set_sha256(tmp_buffer);
}

if (auto iter = pkg.find("noarch"); (iter != pkg.end()) && !iter->is_null())
{
if (iter->is_boolean() && iter->get<bool>())
{
solv.set_noarch("generic");
}
if (iter->is_string())
{
tmp_buffer = iter->get<std::string_view>();
solv.set_noarch(tmp_buffer);
}
}

if (auto iter = pkg.find("license"); (iter != pkg.end()) && iter->is_string())
{
tmp_buffer = iter->get<std::string_view>();
solv.set_license(tmp_buffer);
}

// TODO conda timestamp are not Unix timestamp.
// Libsolv normalize them this way, we need to do the same here otherwise the current
// package may get arbitrary priority.
if (pkg.timestamp)
if (auto iter = pkg.find("timestamp"); (iter != pkg.end()) && iter->is_number())
{
solv.set_timestamp(
(*pkg.timestamp > 253402300799ULL) ? (*pkg.timestamp / 1000) : *pkg.timestamp
);
const auto time = iter->get<std::size_t>();
solv.set_timestamp((time > 253402300799ULL) ? (time / 1000) : time);
}
if (pkg.md5)

if (auto iter = pkg.find("depends"); (iter != pkg.end()) && iter->is_array())
{
solv.set_md5(*pkg.md5);
for (const auto& dep : *iter)
{
if (dep.is_string())
{
tmp_buffer = dep.get<std::string_view>();
if (const auto dep_id = pool_conda_matchspec(pool, tmp_buffer.c_str()))
{
solv.add_dependency(dep_id);
}
}
}
}
if (pkg.sha256)

if (auto iter = pkg.find("depends"); (iter != pkg.end()) && iter->is_array())
{
solv.set_sha256(*pkg.sha256);
for (const auto& dep : *iter)
{
if (dep.is_string())
{
tmp_buffer = dep.get<std::string_view>();
if (const auto dep_id = pool_conda_matchspec(pool, tmp_buffer.c_str()))
{
solv.add_dependency(dep_id);
}
}
}
}

for (const auto& dep : pkg.depends)
if (auto iter = pkg.find("constrains"); (iter != pkg.end()) && iter->is_array())
{
// TODO pool's matchspec2id
solv::DependencyId const dep_id = pool_conda_matchspec(pool, dep.c_str());
assert(dep_id);
solv.add_dependency(dep_id);
for (const auto& dep : *iter)
{
if (dep.is_string())
{
tmp_buffer = dep.get<std::string_view>();
if (const auto dep_id = pool_conda_matchspec(pool, tmp_buffer.c_str()))
{
solv.add_dependency(dep_id);
}
}
}
}

for (const auto& cons : pkg.constrains)
if (auto iter = pkg.find("track_features"); iter != pkg.end())
{
// TODO pool's matchspec2id
solv::DependencyId const dep_id = pool_conda_matchspec(pool, cons.c_str());
assert(dep_id);
solv.add_constraint(dep_id);
if (iter->is_array())
{
for (const auto& feat : *iter)
{
if (feat.is_string())
{
solv.add_track_feature(feat.get<std::string_view>());
}
}
}
}

solv.add_track_features(pkg.track_features);

solv.add_self_provide();
return true;
}

void set_repo_solvables(
MPool& pool,
solv::ObjRepoView repo,
const std::string& repo_url,
const std::string& default_subdir,
const nl::json& packages,
std::string& tmp_buffer
)
{
for (const auto& [fn, pkg] : packages.items())
{
auto [id, solv] = repo.add_solvable();
const bool parsed = set_solvable(pool, solv, repo_url, fn, default_subdir, pkg, tmp_buffer);
if (parsed)
{
LOG_INFO << "Adding package record to repo " << fn;
}
else
{
repo.remove_solvable(id, /* reuse_id= */ true);
LOG_WARNING << "Failed to parse from repodata " << fn;
}
}
}


void set_solvables_url(solv::ObjRepoView repo, const std::string& repo_url)
{
// WARNING cannot call ``url()`` at this point because it has not been internalized.
Expand Down Expand Up @@ -343,23 +479,35 @@ namespace mamba
<< " using mamba";

auto filename_stream = std::ifstream(filename.std_path());
const auto repodata = nl::json::parse(filename_stream).get<specs::RepoData>();
const auto repodata = nl::json::parse(filename_stream);

for (const auto& [fn, pkg] : repodata.packages)
{
LOG_INFO << "Adding package record to repo " << fn;
auto [id, solv] = srepo(*this).add_solvable();
set_solvable(m_pool, solv, m_metadata.url, fn, pkg);
}
// An override for missing package subdir is found in at the top level
std::string default_subdir = repodata.value<std::string>(
nl::json::json_pointer("/info/subdir"),
""
);
// An temporary buffer for managing null terminated strings
std::string tmp_buffer = {};

set_repo_solvables(
m_pool,
srepo(*this),
m_metadata.url,
default_subdir,
repodata.at("packages"),
tmp_buffer
);

if (!Context::instance().use_only_tar_bz2)
{
for (const auto& [fn, pkg] : repodata.conda_packages)
{
LOG_INFO << "Adding package record to repo " << fn;
auto [id, solv] = srepo(*this).add_solvable();
set_solvable(m_pool, solv, m_metadata.url, fn, pkg);
}
set_repo_solvables(
m_pool,
srepo(*this),
m_metadata.url,
default_subdir,
repodata.at("packages.conda"),
tmp_buffer
);
}
}

Expand Down Expand Up @@ -481,12 +629,6 @@ namespace mamba
{
m_pool.remove_repo(id(), reuse_ids);
}
}

#include <map>

namespace mamba
{

auto MRepo::py_name() const -> std::string_view
{
Expand Down

0 comments on commit 15f7bf6

Please sign in to comment.