Merge pull request #129 from hakasapl/0.7.2test3

0.7.2 Test 3

CHANGELOG.md

@@ -19,6 +19,11 @@
 - Added ESMify option for ParallaxGen.esp
 - INI files in the data folder will be read for BSA loading now
 - Advanced is now a checkbox with persistence in the launcher GUI
+- Added critical error if outputting to MO2 mod and mod is enabled in MO2 VFS
+- Added critical error if DynDoLOD output is activated
+- Fixed failed shader upgrade applying the wrong shader
+- Shader transform errors don't post more than once now
+- Exceptions in threads will trigger exceptions in main thread now to prevent error spam
 
 ## [0.7.1] - 2024-11-18
 

CMakeLists.txt

@@ -27,7 +27,7 @@ project(${PROJECT_NAME} VERSION ${PARALLAXGEN_VERSION})
 add_compile_definitions(PARALLAXGEN_VERSION="${PARALLAXGEN_VERSION}")
 
 # Set test version (set this to 0 for prod releases)
-add_compile_definitions(PARALLAXGEN_TEST_VERSION=2)
+add_compile_definitions(PARALLAXGEN_TEST_VERSION=3)
 
 # Setup Folders
 set(EXTRN_BUILD_DIR ${CMAKE_BINARY_DIR}/external/blds)

ParallaxGen/src/main.cpp

@@ -1,6 +1,7 @@
 #include <CLI/CLI.hpp>
 
 #include <NifFile.hpp>
+#include <boost/algorithm/string/predicate.hpp>
 #include <spdlog/common.h>
 #include <spdlog/logger.h>
 #include <spdlog/sinks/rotating_file_sink.h>
@@ -194,14 +195,9 @@ void mainRunner(ParallaxGenCLIArgs &Args, const filesystem::path &ExePath) {
     exit(1);
   }
 
-  // delete existing output
-  PG.deleteOutputDir();
-
-  // Check if ParallaxGen output already exists in data directory
-  const filesystem::path PGStateFilePath = BG.getGameDataPath() / ParallaxGen::getDiffJSONName();
-  if (filesystem::exists(PGStateFilePath)) {
-    spdlog::critical("ParallaxGen meshes exist in your data directory, please delete before "
-                     "re-running.");
+  // If output dir is a subdirectory of data dir vfs issues can occur
+  if (boost::istarts_with(Params.Output.Dir.wstring(), BG.getGameDataPath().wstring() + "\\")) {
+    spdlog::critical("Output directory cannot be a subdirectory of your data folder. Exiting.");
     exit(1);
   }
 
@@ -217,12 +213,32 @@ void mainRunner(ParallaxGenCLIArgs &Args, const filesystem::path &ExePath) {
   if (Params.ModManager.Type == ModManagerDirectory::ModManagerType::ModOrganizer2 &&
       !Params.ModManager.MO2InstanceDir.empty() && !Params.ModManager.MO2Profile.empty()) {
     // MO2
-    MMD.populateModFileMapMO2(Params.ModManager.MO2InstanceDir, Params.ModManager.MO2Profile);
+    MMD.populateModFileMapMO2(Params.ModManager.MO2InstanceDir, Params.ModManager.MO2Profile, Params.Output.Dir);
   } else if (Params.ModManager.Type == ModManagerDirectory::ModManagerType::Vortex) {
     // Vortex
     MMD.populateModFileMapVortex(BG.getGameDataPath());
   }
 
+  // delete existing output
+  PG.deleteOutputDir();
+
+  // Check if ParallaxGen output already exists in data directory
+  const filesystem::path PGStateFilePath = BG.getGameDataPath() / ParallaxGen::getDiffJSONName();
+  if (filesystem::exists(PGStateFilePath)) {
+    spdlog::critical("ParallaxGen meshes exist in your data directory, please delete before "
+                     "re-running.");
+    exit(1);
+  }
+
+  // Check if dyndolod.esp exists
+  const auto ActivePlugins = BG.getActivePlugins(false, true);
+  if (find(ActivePlugins.begin(), ActivePlugins.end(), L"dyndolod.esp") != ActivePlugins.end()) {
+    spdlog::critical("DynDoLOD and TexGen outputs must be disabled prior to running ParallaxGen. It is recommended to "
+                     "generate LODs after running ParallaxGen with the ParallaxGen output enabled.");
+    exit(1);
+  }
+
+  // Init file map
   PGD.populateFileMap(Params.Processing.BSA);
 
   // Map files

ParallaxGenLib/include/BethesdaGame.hpp

@@ -63,7 +63,7 @@ class BethesdaGame {
   [[nodiscard]] auto getPluginsFile() const -> std::filesystem::path;
 
   // Get number of active plugins including Bethesda master files
-  [[nodiscard]] auto getActivePlugins(const bool &TrimExtension = false) const -> std::vector<std::wstring>;
+  [[nodiscard]] auto getActivePlugins(const bool &TrimExtension = false, const bool &Lowercase = false) const -> std::vector<std::wstring>;
 
   // Helpers
   [[nodiscard]] static auto getGameTypes() -> std::vector<GameType>;

ParallaxGenLib/include/ModManagerDirectory.hpp

@@ -8,11 +8,7 @@
 class ModManagerDirectory {
 
 public:
-  enum class ModManagerType {
-    None,
-    Vortex,
-    ModOrganizer2
-  };
+  enum class ModManagerType { None, Vortex, ModOrganizer2 };
 
 private:
   std::unordered_map<std::filesystem::path, std::wstring> ModFileMap;
@@ -30,7 +26,8 @@ class ModManagerDirectory {
 
   static auto getMO2ProfilesFromInstanceDir(const std::filesystem::path &InstanceDir) -> std::vector<std::wstring>;
 
-  void populateModFileMapMO2(const std::filesystem::path &InstanceDir, const std::wstring &Profile);
+  void populateModFileMapMO2(const std::filesystem::path &InstanceDir, const std::wstring &Profile,
+                             const std::filesystem::path &OutputDir);
   void populateModFileMapVortex(const std::filesystem::path &DeploymentDir);
 
   // Helpers

ParallaxGenLib/include/patchers/PatcherShaderTransform.hpp

@@ -1,7 +1,9 @@
 #pragma once
 
 #include <string>
+#include <unordered_set>
 
+#include "NIFUtil.hpp"
 #include "patchers/Patcher.hpp"
 #include "patchers/PatcherShader.hpp"
 
@@ -10,13 +12,34 @@
  * @brief Base class for shader transform patchers
  */
 class PatcherShaderTransform : public Patcher {
+private:
+  struct ErrorTrackerHasher {
+    auto operator()(const std::tuple<std::filesystem::path, NIFUtil::ShapeShader, NIFUtil::ShapeShader> &Key) const
+        -> std::size_t {
+      return std::hash<std::filesystem::path>{}(std::get<0>(Key)) ^
+             std::hash<NIFUtil::ShapeShader>{}(std::get<1>(Key)) ^ std::hash<NIFUtil::ShapeShader>{}(std::get<2>(Key));
+    }
+  };
+
+  static std::unordered_set<std::tuple<std::filesystem::path, NIFUtil::ShapeShader, NIFUtil::ShapeShader>,
+                            ErrorTrackerHasher>
+      ErrorTracker;                /** Tracks transforms that failed for error messages */
+  NIFUtil::ShapeShader FromShader; /** Shader to transform from */
+  NIFUtil::ShapeShader ToShader;   /** Shader to transform to */
+
+protected:
+  void postError(const std::filesystem::path &File);            /** Post error message for transform */
+  auto alreadyTried(const std::filesystem::path &File) -> bool; /** Check if transform has already been tried */
+
 public:
   // Custom type definitions
-  using PatcherShaderTransformFactory = std::function<std::unique_ptr<PatcherShaderTransform>(std::filesystem::path, nifly::NifFile*)>;
+  using PatcherShaderTransformFactory =
+      std::function<std::unique_ptr<PatcherShaderTransform>(std::filesystem::path, nifly::NifFile *)>;
   using PatcherShaderTransformObject = std::unique_ptr<PatcherShaderTransform>;
 
   // Constructors
-  PatcherShaderTransform(std::filesystem::path NIFPath, nifly::NifFile *NIF, std::string PatcherName);
+  PatcherShaderTransform(std::filesystem::path NIFPath, nifly::NifFile *NIF, std::string PatcherName,
+                         const NIFUtil::ShapeShader &From, const NIFUtil::ShapeShader &To);
   virtual ~PatcherShaderTransform() = default;
   PatcherShaderTransform(const PatcherShaderTransform &Other) = default;
   auto operator=(const PatcherShaderTransform &Other) -> PatcherShaderTransform & = default;
@@ -29,5 +52,5 @@ class PatcherShaderTransform : public Patcher {
    * @param FromMatch shader match to transform
    * @return PatcherShader::PatcherMatch transformed match
    */
-  virtual auto transform(const PatcherShader::PatcherMatch &FromMatch) -> PatcherShader::PatcherMatch = 0;
+  virtual auto transform(const PatcherShader::PatcherMatch &FromMatch, PatcherShader::PatcherMatch &Result) -> bool = 0;
 };

ParallaxGenLib/include/patchers/PatcherUpgradeParallaxToCM.hpp

@@ -48,5 +48,5 @@ class PatcherUpgradeParallaxToCM : public PatcherShaderTransform {
    * @param FromMatch Match to transform
    * @return PatcherShader::PatcherMatch Transformed match
    */
-  auto transform(const PatcherShader::PatcherMatch &FromMatch) -> PatcherShader::PatcherMatch override;
+  auto transform(const PatcherShader::PatcherMatch &FromMatch, PatcherShader::PatcherMatch &Result) -> bool override;
 };

ParallaxGenLib/src/BethesdaGame.cpp

@@ -296,7 +296,7 @@ auto BethesdaGame::getPluginsFile() const -> filesystem::path {
   return GamePluginsFile;
 }
 
-auto BethesdaGame::getActivePlugins(const bool &TrimExtension) const -> vector<wstring> {
+auto BethesdaGame::getActivePlugins(const bool &TrimExtension, const bool &Lowercase) const -> vector<wstring> {
   vector<wstring> OutputLO;
 
   // Build set of plugins that are actually active
@@ -365,7 +365,12 @@ auto BethesdaGame::getActivePlugins(const bool &TrimExtension) const -> vector<w
     }
 
     // Add to output list
-    OutputLO.push_back(ParallaxGenUtil::UTF8toUTF16(Line));
+    auto OutputLOElem = ParallaxGenUtil::UTF8toUTF16(Line);
+    if (Lowercase) {
+      OutputLOElem = ParallaxGenUtil::ToLowerASCII(OutputLOElem);
+    }
+
+    OutputLO.push_back(OutputLOElem);
   }
 
   // close file handle

ParallaxGenLib/src/ModManagerDirectory.cpp

@@ -72,15 +72,17 @@ void ModManagerDirectory::populateModFileMapVortex(const filesystem::path &Deplo
   }
 }
 
-void ModManagerDirectory::populateModFileMapMO2(const filesystem::path &InstanceDir, const wstring &Profile) {
+void ModManagerDirectory::populateModFileMapMO2(const filesystem::path &InstanceDir, const wstring &Profile,
+                                                const filesystem::path &OutputDir) {
   // required file is modlist.txt in the profile folder
 
   spdlog::info("Populating mods from Mod Organizer 2");
 
   // First read modorganizer.ini in the instance folder to get the profiles and mods folders
   filesystem::path MO2IniFile = InstanceDir / L"modorganizer.ini";
   if (!filesystem::exists(MO2IniFile)) {
-    throw runtime_error("Mod Organizer 2 ini file does not exist: " + ParallaxGenUtil::UTF16toUTF8(MO2IniFile.wstring()));
+    throw runtime_error("Mod Organizer 2 ini file does not exist: " +
+                        ParallaxGenUtil::UTF16toUTF8(MO2IniFile.wstring()));
   }
 
   // Find MO2 paths from ModOrganizer.ini
@@ -90,7 +92,7 @@ void ModManagerDirectory::populateModFileMapMO2(const filesystem::path &Instance
 
   ifstream MO2IniFileF(MO2IniFile);
   string MO2IniLine;
-  while(getline(MO2IniFileF, MO2IniLine)) {
+  while (getline(MO2IniFileF, MO2IniLine)) {
     if (MO2IniLine.starts_with("profiles_directory=")) {
       ProfileDir = InstanceDir / MO2IniLine.substr(19);
     } else if (MO2IniLine.starts_with("mod_directory=")) {
@@ -153,10 +155,17 @@ void ModManagerDirectory::populateModFileMapMO2(const filesystem::path &Instance
 
     // loop through all files in mod
     Mod.erase(0, 1); // remove +
+    const auto CurModDir = ModDir / Mod;
+
+    // check if mod dir is output dir
+    if (filesystem::equivalent(CurModDir, OutputDir)) {
+      spdlog::critical(L"If outputting to MO2 you must disable the mod {} first to prevent issues with MO2 VFS", Mod);
+      exit(1);
+    }
+
     AllMods.insert(Mod);
     InferredOrder.insert(InferredOrder.begin(), Mod);
 
-    const auto CurModDir = ModDir / Mod;
     if (!filesystem::exists(CurModDir)) {
       spdlog::warn(L"Mod directory from modlist.txt does not exist: {}", CurModDir.wstring());
       continue;
@@ -239,7 +248,7 @@ auto ModManagerDirectory::getMO2ProfilesFromInstanceDir(const filesystem::path &
 
   ifstream MO2IniFileF(MO2IniFile);
   string MO2IniLine;
-  while(getline(MO2IniFileF, MO2IniLine)) {
+  while (getline(MO2IniFileF, MO2IniLine)) {
     if (MO2IniLine.starts_with("profiles_directory=")) {
       ProfileDir = InstanceDir / MO2IniLine.substr(19);
     } else if (MO2IniLine.starts_with("base_directory=")) {
@@ -274,6 +283,4 @@ auto ModManagerDirectory::getMO2ProfilesFromInstanceDir(const filesystem::path &
   return Profiles;
 }
 
-auto ModManagerDirectory::getInferredOrder() const -> const vector<wstring> & {
-  return InferredOrder;
-}
+auto ModManagerDirectory::getInferredOrder() const -> const vector<wstring> & { return InferredOrder; }

ParallaxGenLib/src/ParallaxGen.cpp

@@ -69,18 +69,29 @@ void ParallaxGen::patchMeshes(const PatcherUtil::PatcherSet &Patchers, const uno
 
     boost::asio::thread_pool MeshPatchPool(NumThreads);
 
+    bool KillThreads = false;
+
     for (const auto &Mesh : Meshes) {
-      boost::asio::post(MeshPatchPool, [this, &TaskTracker, &DiffJSON, &Mesh, &PatchPlugin, &Patchers, &ModPriority] {
-        ParallaxGenTask::PGResult Result = ParallaxGenTask::PGResult::SUCCESS;
-        try {
-          Result = processNIF(Patchers, ModPriority, Mesh, &DiffJSON, PatchPlugin);
-        } catch (const exception &E) {
-          spdlog::error(L"Exception in thread patching NIF {}: {}", Mesh.wstring(), ASCIItoUTF16(E.what()));
-          Result = ParallaxGenTask::PGResult::FAILURE;
-        }
+      boost::asio::post(
+          MeshPatchPool, [this, &TaskTracker, &DiffJSON, &Mesh, &PatchPlugin, &Patchers, &ModPriority, &KillThreads] {
+            ParallaxGenTask::PGResult Result = ParallaxGenTask::PGResult::SUCCESS;
+            try {
+              Result = processNIF(Patchers, ModPriority, Mesh, &DiffJSON, PatchPlugin);
+            } catch (const exception &E) {
+              KillThreads = true;
+              spdlog::error(L"Exception in thread patching NIF {}: {}", Mesh.wstring(), ASCIItoUTF16(E.what()));
+              Result = ParallaxGenTask::PGResult::FAILURE;
+            }
 
-        TaskTracker.completeJob(Result);
-      });
+            TaskTracker.completeJob(Result);
+          });
+    }
+
+    while (!TaskTracker.isCompleted()) {
+      if (KillThreads) {
+        MeshPatchPool.stop();
+        throw runtime_error("Exception in thread patching NIF");
+      }
     }
 
     // verify that all threads complete (should be redundant)
@@ -125,15 +136,18 @@ auto ParallaxGen::findModConflicts(const PatcherUtil::PatcherSet &Patchers, cons
     const size_t NumThreads = boost::thread::hardware_concurrency();
 #endif
 
+    bool KillThreads = false;
+
     boost::asio::thread_pool MeshPatchPool(NumThreads);
 
     for (const auto &Mesh : Meshes) {
       boost::asio::post(MeshPatchPool, [this, &TaskTracker, &Mesh, &PatchPlugin, &ConflictMods, &ConflictModsMutex,
-                                        &Patchers] {
+                                        &Patchers, &KillThreads] {
         ParallaxGenTask::PGResult Result = ParallaxGenTask::PGResult::SUCCESS;
         try {
           Result = processNIF(Patchers, nullptr, Mesh, nullptr, PatchPlugin, true, &ConflictMods, &ConflictModsMutex);
         } catch (const exception &E) {
+          KillThreads = true;
           spdlog::error(L"Exception in thread finding mod conflicts {}: {}", Mesh.wstring(), ASCIItoUTF16(E.what()));
           Result = ParallaxGenTask::PGResult::FAILURE;
         }
@@ -142,6 +156,13 @@ auto ParallaxGen::findModConflicts(const PatcherUtil::PatcherSet &Patchers, cons
       });
     }
 
+    while (!TaskTracker.isCompleted()) {
+      if (KillThreads) {
+        MeshPatchPool.stop();
+        throw runtime_error("Exception in thread finding mod conflicts");
+      }
+    }
+
     // verify that all threads complete (should be redundant)
     MeshPatchPool.join();
 
@@ -328,8 +349,7 @@ auto ParallaxGen::processNIF(
     }
 
     // Sort ShapeTracker by new block id
-    sort(ShapeTracker.begin(), ShapeTracker.end(),
-         [](const auto &A, const auto &B) { return get<2>(A) < get<2>(B); });
+    sort(ShapeTracker.begin(), ShapeTracker.end(), [](const auto &A, const auto &B) { return get<2>(A) < get<2>(B); });
 
     // Find new 3d index for each shape
     for (int I = 0; I < ShapeTracker.size(); I++) {
@@ -451,7 +471,7 @@ auto ParallaxGen::processShape(
   unordered_set<wstring> ModSet;
   if (!CacheExists) {
     for (const auto &[Shader, Patcher] : Patchers.ShaderPatchers) {
-        // note: name is defined in source code in UTF8-encoded files
+      // note: name is defined in source code in UTF8-encoded files
       Logger::Prefix PrefixPatches(UTF8toUTF16(Patcher->getPatcherName()));
 
       // Check if shader should be applied
@@ -566,7 +586,8 @@ void ParallaxGen::addFileToZip(mz_zip_archive &Zip, const filesystem::path &File
   const string RelativeFilePathAscii = UTF16toASCII(RelativePath.wstring());
 
   // add file to Zip
-  if (mz_zip_writer_add_mem(&Zip, RelativeFilePathAscii.c_str(), Buffer.data(), Buffer.size(), MZ_NO_COMPRESSION) == 0) {
+  if (mz_zip_writer_add_mem(&Zip, RelativeFilePathAscii.c_str(), Buffer.data(), Buffer.size(), MZ_NO_COMPRESSION) ==
+      0) {
     spdlog::error(L"Error adding file to zip: {}", FilePath.wstring());
     exit(1);
   }