Read repodata.json using nl::json (rerun) (#2753)

* Add simdjson dependency * Add experimental_repodata_parsing option * Add simdjson repodata parser * Add missing dependency * Deactivate simdjson by default
mamba-org · Oct 4, 2023 · 91b6c44 · 91b6c44
1 parent 6fe4f4d
commit 91b6c44
Show file tree

Hide file tree

Showing 12 changed files with 391 additions and 110 deletions.
diff --git a/.github/workflows/static_build.yml b/.github/workflows/static_build.yml
@@ -134,6 +134,7 @@ jobs:
             cli11>=2.2,<3
             cpp-expected
             nlohmann_json
+            simdjson-static>=3.3.0
             spdlog
             fmt
             yaml-cpp-static>=0.8.0

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -47,10 +47,6 @@ endif()
 
 # libmamba library and tests
 if (BUILD_LIBMAMBA)
-    if (BUILD_LIBMAMBA_TESTS)
-        set(BUILD_TESTS ON)
-    endif()
-
     add_subdirectory(libmamba)
 endif()
 

diff --git a/libmamba/CMakeLists.txt b/libmamba/CMakeLists.txt
@@ -322,6 +322,7 @@ find_package(fmt REQUIRED)
 find_package(spdlog REQUIRED)
 find_package(tl-expected REQUIRED)
 find_package(nlohmann_json REQUIRED)
+find_package(simdjson REQUIRED)
 find_package(yaml-cpp REQUIRED)
 find_package(reproc REQUIRED)
 find_package(reproc++ REQUIRED)
@@ -359,6 +360,8 @@ macro(libmamba_create_target target_name linkage output_name)
     if (${linkage_upper} STREQUAL "STATIC")
         message("   -> Statically linking against libmamba (static) dependencies")
 
+        target_compile_definitions(${target_name} PUBLIC SPDLOG_FMT_EXTERNAL)
+
         mamba_target_check_type(yaml-cpp::yaml-cpp STATIC_LIBRARY FATAL_ERROR)
         mamba_target_check_type(reproc STATIC_LIBRARY FATAL_ERROR)
         mamba_target_check_type(reproc++ STATIC_LIBRARY FATAL_ERROR)
@@ -376,6 +379,7 @@ macro(libmamba_create_target target_name linkage output_name)
             PRIVATE
                 reproc
                 reproc++
+                simdjson::simdjson_static
         )
 
         if (UNIX)
@@ -529,6 +533,7 @@ macro(libmamba_create_target target_name linkage output_name)
             PRIVATE
                 reproc
                 reproc++
+                simdjson::simdjson
         )
     endif ()
 
@@ -616,7 +621,7 @@ if (NOT (BUILD_SHARED OR BUILD_STATIC))
 endif ()
 
 # Tests
-if (BUILD_TESTS)
+if (BUILD_LIBMAMBA_TESTS)
     add_subdirectory(tests)
 endif()
 

diff --git a/libmamba/environment-dev.yml b/libmamba/environment-dev.yml
@@ -6,6 +6,7 @@ dependencies:
   - cmake >=3.16
   - ninja
   - nlohmann_json
+  - simdjson >=3.3.0
   - libsolv >=0.7.18
   - libarchive
   - libsodium

diff --git a/libmamba/include/mamba/core/context.hpp b/libmamba/include/mamba/core/context.hpp
@@ -131,6 +131,7 @@ namespace mamba
 
         // Configurable
         bool experimental = false;
+        bool experimental_repodata_parsing = false;
         bool debug = false;
 
         // TODO check writable and add other potential dirs

diff --git a/libmamba/include/mamba/core/repo.hpp b/libmamba/include/mamba/core/repo.hpp
@@ -62,7 +62,27 @@ namespace mamba
     {
     public:
 
-        MRepo(MPool& pool, const std::string& name, const fs::u8path& filename, const RepoMetadata& meta);
+        enum class RepodataParser
+        {
+            automatic,
+            mamba,
+            libsolv,
+        };
+
+        enum class LibsolvCache
+        {
+            yes,
+            no,
+        };
+
+        MRepo(
+            MPool& pool,
+            const std::string& name,
+            const fs::u8path& filename,
+            const RepoMetadata& meta,
+            RepodataParser parser = RepodataParser::automatic,
+            LibsolvCache use_cache = LibsolvCache::yes
+        );
         MRepo(MPool& pool, const PrefixData& prefix_data);
         MRepo(MPool& pool, const std::string& name, const std::vector<PackageInfo>& uris);
 
@@ -96,12 +116,11 @@ namespace mamba
 
         void add_pip_as_python_dependency();
         void clear(bool reuse_ids = true);
-        void load_file(const fs::u8path& filename);
-        void read_json(const fs::u8path& filename);
+        void load_file(const fs::u8path& filename, RepodataParser parser, LibsolvCache use_cache);
+        void libsolv_read_json(const fs::u8path& filename);
+        void mamba_read_json(const fs::u8path& filename);
         bool read_solv(const fs::u8path& filename);
         void write_solv(fs::u8path path);
-        void add_package_info(const PackageInfo& pkg_info);
-        void set_solvables_url(const std::string& repo_url);
 
         MPool m_pool;
 

diff --git a/libmamba/src/api/configuration.cpp b/libmamba/src/api/configuration.cpp
@@ -1209,6 +1209,12 @@ namespace mamba
                         under active development and not stable yet.)"))
                    .set_post_merge_hook(detail::experimental_hook));
 
+        insert(Configurable("experimental_repodata_parsing", &m_context.experimental_repodata_parsing)
+                   .group("Basic")
+                   .description("Enable experimental parsing of repodata.json using nl::json")
+                   .set_rc_configurable()
+                   .set_env_var_names());
+
         insert(Configurable("debug", &m_context.debug)
                    .group("Basic")
                    .set_env_var_names()