diff --git a/CHANGELOG.md b/CHANGELOG.md index 143fff0..b5bc8d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,11 @@ - Added ESMify option for ParallaxGen.esp - INI files in the data folder will be read for BSA loading now - Advanced is now a checkbox with persistence in the launcher GUI +- Added critical error if outputting to MO2 mod and mod is enabled in MO2 VFS +- Added critical error if DynDoLOD output is activated +- Fixed failed shader upgrade applying the wrong shader +- Shader transform errors don't post more than once now +- Exceptions in threads will trigger exceptions in main thread now to prevent error spam ## [0.7.1] - 2024-11-18 diff --git a/CMakeLists.txt b/CMakeLists.txt index 9111f2d..f73a65a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ project(${PROJECT_NAME} VERSION ${PARALLAXGEN_VERSION}) add_compile_definitions(PARALLAXGEN_VERSION="${PARALLAXGEN_VERSION}") # Set test version (set this to 0 for prod releases) -add_compile_definitions(PARALLAXGEN_TEST_VERSION=2) +add_compile_definitions(PARALLAXGEN_TEST_VERSION=3) # Setup Folders set(EXTRN_BUILD_DIR ${CMAKE_BINARY_DIR}/external/blds) diff --git a/ParallaxGen/src/main.cpp b/ParallaxGen/src/main.cpp index 69667af..2c4f491 100644 --- a/ParallaxGen/src/main.cpp +++ b/ParallaxGen/src/main.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -194,14 +195,9 @@ void mainRunner(ParallaxGenCLIArgs &Args, const filesystem::path &ExePath) { exit(1); } - // delete existing output - PG.deleteOutputDir(); - - // Check if ParallaxGen output already exists in data directory - const filesystem::path PGStateFilePath = BG.getGameDataPath() / ParallaxGen::getDiffJSONName(); - if (filesystem::exists(PGStateFilePath)) { - spdlog::critical("ParallaxGen meshes exist in your data directory, please delete before " - "re-running."); + // If output dir is a subdirectory of data dir vfs issues can occur + if (boost::istarts_with(Params.Output.Dir.wstring(), BG.getGameDataPath().wstring() + "\\")) { + spdlog::critical("Output directory cannot be a subdirectory of your data folder. Exiting."); exit(1); } @@ -217,12 +213,32 @@ void mainRunner(ParallaxGenCLIArgs &Args, const filesystem::path &ExePath) { if (Params.ModManager.Type == ModManagerDirectory::ModManagerType::ModOrganizer2 && !Params.ModManager.MO2InstanceDir.empty() && !Params.ModManager.MO2Profile.empty()) { // MO2 - MMD.populateModFileMapMO2(Params.ModManager.MO2InstanceDir, Params.ModManager.MO2Profile); + MMD.populateModFileMapMO2(Params.ModManager.MO2InstanceDir, Params.ModManager.MO2Profile, Params.Output.Dir); } else if (Params.ModManager.Type == ModManagerDirectory::ModManagerType::Vortex) { // Vortex MMD.populateModFileMapVortex(BG.getGameDataPath()); } + // delete existing output + PG.deleteOutputDir(); + + // Check if ParallaxGen output already exists in data directory + const filesystem::path PGStateFilePath = BG.getGameDataPath() / ParallaxGen::getDiffJSONName(); + if (filesystem::exists(PGStateFilePath)) { + spdlog::critical("ParallaxGen meshes exist in your data directory, please delete before " + "re-running."); + exit(1); + } + + // Check if dyndolod.esp exists + const auto ActivePlugins = BG.getActivePlugins(false, true); + if (find(ActivePlugins.begin(), ActivePlugins.end(), L"dyndolod.esp") != ActivePlugins.end()) { + spdlog::critical("DynDoLOD and TexGen outputs must be disabled prior to running ParallaxGen. It is recommended to " + "generate LODs after running ParallaxGen with the ParallaxGen output enabled."); + exit(1); + } + + // Init file map PGD.populateFileMap(Params.Processing.BSA); // Map files diff --git a/ParallaxGenLib/include/BethesdaGame.hpp b/ParallaxGenLib/include/BethesdaGame.hpp index 2a49220..eb9af37 100644 --- a/ParallaxGenLib/include/BethesdaGame.hpp +++ b/ParallaxGenLib/include/BethesdaGame.hpp @@ -63,7 +63,7 @@ class BethesdaGame { [[nodiscard]] auto getPluginsFile() const -> std::filesystem::path; // Get number of active plugins including Bethesda master files - [[nodiscard]] auto getActivePlugins(const bool &TrimExtension = false) const -> std::vector; + [[nodiscard]] auto getActivePlugins(const bool &TrimExtension = false, const bool &Lowercase = false) const -> std::vector; // Helpers [[nodiscard]] static auto getGameTypes() -> std::vector; diff --git a/ParallaxGenLib/include/ModManagerDirectory.hpp b/ParallaxGenLib/include/ModManagerDirectory.hpp index f9c85aa..2af8b94 100644 --- a/ParallaxGenLib/include/ModManagerDirectory.hpp +++ b/ParallaxGenLib/include/ModManagerDirectory.hpp @@ -8,11 +8,7 @@ class ModManagerDirectory { public: - enum class ModManagerType { - None, - Vortex, - ModOrganizer2 - }; + enum class ModManagerType { None, Vortex, ModOrganizer2 }; private: std::unordered_map ModFileMap; @@ -30,7 +26,8 @@ class ModManagerDirectory { static auto getMO2ProfilesFromInstanceDir(const std::filesystem::path &InstanceDir) -> std::vector; - void populateModFileMapMO2(const std::filesystem::path &InstanceDir, const std::wstring &Profile); + void populateModFileMapMO2(const std::filesystem::path &InstanceDir, const std::wstring &Profile, + const std::filesystem::path &OutputDir); void populateModFileMapVortex(const std::filesystem::path &DeploymentDir); // Helpers diff --git a/ParallaxGenLib/include/patchers/PatcherShaderTransform.hpp b/ParallaxGenLib/include/patchers/PatcherShaderTransform.hpp index 36cb5a6..112264b 100644 --- a/ParallaxGenLib/include/patchers/PatcherShaderTransform.hpp +++ b/ParallaxGenLib/include/patchers/PatcherShaderTransform.hpp @@ -1,7 +1,9 @@ #pragma once #include +#include +#include "NIFUtil.hpp" #include "patchers/Patcher.hpp" #include "patchers/PatcherShader.hpp" @@ -10,13 +12,34 @@ * @brief Base class for shader transform patchers */ class PatcherShaderTransform : public Patcher { +private: + struct ErrorTrackerHasher { + auto operator()(const std::tuple &Key) const + -> std::size_t { + return std::hash{}(std::get<0>(Key)) ^ + std::hash{}(std::get<1>(Key)) ^ std::hash{}(std::get<2>(Key)); + } + }; + + static std::unordered_set, + ErrorTrackerHasher> + ErrorTracker; /** Tracks transforms that failed for error messages */ + NIFUtil::ShapeShader FromShader; /** Shader to transform from */ + NIFUtil::ShapeShader ToShader; /** Shader to transform to */ + +protected: + void postError(const std::filesystem::path &File); /** Post error message for transform */ + auto alreadyTried(const std::filesystem::path &File) -> bool; /** Check if transform has already been tried */ + public: // Custom type definitions - using PatcherShaderTransformFactory = std::function(std::filesystem::path, nifly::NifFile*)>; + using PatcherShaderTransformFactory = + std::function(std::filesystem::path, nifly::NifFile *)>; using PatcherShaderTransformObject = std::unique_ptr; // Constructors - PatcherShaderTransform(std::filesystem::path NIFPath, nifly::NifFile *NIF, std::string PatcherName); + PatcherShaderTransform(std::filesystem::path NIFPath, nifly::NifFile *NIF, std::string PatcherName, + const NIFUtil::ShapeShader &From, const NIFUtil::ShapeShader &To); virtual ~PatcherShaderTransform() = default; PatcherShaderTransform(const PatcherShaderTransform &Other) = default; auto operator=(const PatcherShaderTransform &Other) -> PatcherShaderTransform & = default; @@ -29,5 +52,5 @@ class PatcherShaderTransform : public Patcher { * @param FromMatch shader match to transform * @return PatcherShader::PatcherMatch transformed match */ - virtual auto transform(const PatcherShader::PatcherMatch &FromMatch) -> PatcherShader::PatcherMatch = 0; + virtual auto transform(const PatcherShader::PatcherMatch &FromMatch, PatcherShader::PatcherMatch &Result) -> bool = 0; }; diff --git a/ParallaxGenLib/include/patchers/PatcherUpgradeParallaxToCM.hpp b/ParallaxGenLib/include/patchers/PatcherUpgradeParallaxToCM.hpp index af90c5a..79173d8 100644 --- a/ParallaxGenLib/include/patchers/PatcherUpgradeParallaxToCM.hpp +++ b/ParallaxGenLib/include/patchers/PatcherUpgradeParallaxToCM.hpp @@ -48,5 +48,5 @@ class PatcherUpgradeParallaxToCM : public PatcherShaderTransform { * @param FromMatch Match to transform * @return PatcherShader::PatcherMatch Transformed match */ - auto transform(const PatcherShader::PatcherMatch &FromMatch) -> PatcherShader::PatcherMatch override; + auto transform(const PatcherShader::PatcherMatch &FromMatch, PatcherShader::PatcherMatch &Result) -> bool override; }; diff --git a/ParallaxGenLib/src/BethesdaGame.cpp b/ParallaxGenLib/src/BethesdaGame.cpp index 55f2259..df055e9 100644 --- a/ParallaxGenLib/src/BethesdaGame.cpp +++ b/ParallaxGenLib/src/BethesdaGame.cpp @@ -296,7 +296,7 @@ auto BethesdaGame::getPluginsFile() const -> filesystem::path { return GamePluginsFile; } -auto BethesdaGame::getActivePlugins(const bool &TrimExtension) const -> vector { +auto BethesdaGame::getActivePlugins(const bool &TrimExtension, const bool &Lowercase) const -> vector { vector OutputLO; // Build set of plugins that are actually active @@ -365,7 +365,12 @@ auto BethesdaGame::getActivePlugins(const bool &TrimExtension) const -> vector(A) < get<2>(B); }); + sort(ShapeTracker.begin(), ShapeTracker.end(), [](const auto &A, const auto &B) { return get<2>(A) < get<2>(B); }); // Find new 3d index for each shape for (int I = 0; I < ShapeTracker.size(); I++) { @@ -451,7 +471,7 @@ auto ParallaxGen::processShape( unordered_set ModSet; if (!CacheExists) { for (const auto &[Shader, Patcher] : Patchers.ShaderPatchers) { - // note: name is defined in source code in UTF8-encoded files + // note: name is defined in source code in UTF8-encoded files Logger::Prefix PrefixPatches(UTF8toUTF16(Patcher->getPatcherName())); // Check if shader should be applied @@ -566,7 +586,8 @@ void ParallaxGen::addFileToZip(mz_zip_archive &Zip, const filesystem::path &File const string RelativeFilePathAscii = UTF16toASCII(RelativePath.wstring()); // add file to Zip - if (mz_zip_writer_add_mem(&Zip, RelativeFilePathAscii.c_str(), Buffer.data(), Buffer.size(), MZ_NO_COMPRESSION) == 0) { + if (mz_zip_writer_add_mem(&Zip, RelativeFilePathAscii.c_str(), Buffer.data(), Buffer.size(), MZ_NO_COMPRESSION) == + 0) { spdlog::error(L"Error adding file to zip: {}", FilePath.wstring()); exit(1); } diff --git a/ParallaxGenLib/src/ParallaxGenD3D.cpp b/ParallaxGenLib/src/ParallaxGenD3D.cpp index 7551645..c8d2e37 100644 --- a/ParallaxGenLib/src/ParallaxGenD3D.cpp +++ b/ParallaxGenLib/src/ParallaxGenD3D.cpp @@ -457,7 +457,7 @@ auto ParallaxGenD3D::upgradeToComplexMaterial(const std::filesystem::path &Paral // get parallax map DirectX::ScratchImage ParallaxMapDDS; if (ParallaxExists && getDDS(ParallaxMap, ParallaxMapDDS) != ParallaxGenTask::PGResult::SUCCESS) { - spdlog::error(L"Failed to load DDS file: {}", ParallaxMap.wstring()); + spdlog::debug(L"Failed to load DDS file: {}", ParallaxMap.wstring()); return {}; } const DirectX::TexMetadata ParallaxMeta = ParallaxMapDDS.GetMetadata(); @@ -465,7 +465,7 @@ auto ParallaxGenD3D::upgradeToComplexMaterial(const std::filesystem::path &Paral // get env map DirectX::ScratchImage EnvMapDDS; if (EnvExists && getDDS(EnvMap, EnvMapDDS) != ParallaxGenTask::PGResult::SUCCESS) { - spdlog::error(L"Failed to load DDS file: {}", ParallaxMap.wstring()); + spdlog::debug(L"Failed to load DDS file: {}", ParallaxMap.wstring()); return {}; } const DirectX::TexMetadata EnvMeta = EnvMapDDS.GetMetadata(); @@ -502,12 +502,12 @@ auto ParallaxGenD3D::upgradeToComplexMaterial(const std::filesystem::path &Paral if (ParallaxExists) { PGResult = createTexture2D(ParallaxMapDDS, ParallaxMapGPU); if (PGResult != ParallaxGenTask::PGResult::SUCCESS) { - spdlog::error(L"Failed to create GPU texture for {}", ParallaxMap.wstring()); + spdlog::debug(L"Failed to create GPU texture for {}", ParallaxMap.wstring()); return {}; } PGResult = createShaderResourceView(ParallaxMapGPU, ParallaxMapSRV); if (PGResult != ParallaxGenTask::PGResult::SUCCESS) { - spdlog::error(L"Failed to create GPU SRV for {}", ParallaxMap.wstring()); + spdlog::debug(L"Failed to create GPU SRV for {}", ParallaxMap.wstring()); return {}; } } @@ -693,7 +693,7 @@ auto ParallaxGenD3D::upgradeToComplexMaterial(const std::filesystem::path &Paral HRESULT HR = DirectX::Compress(OutputImage.GetImages(), OutputImage.GetImageCount(), OutputImage.GetMetadata(), DXGI_FORMAT_BC3_UNORM, DirectX::TEX_COMPRESS_DEFAULT, 1.0F, CompressedImage); if (FAILED(HR)) { - spdlog::error("Failed to compress output DDS file: {}", getHRESULTErrorMessage(HR)); + spdlog::debug("Failed to compress output DDS file: {}", getHRESULTErrorMessage(HR)); return {}; } @@ -1040,7 +1040,7 @@ auto ParallaxGenD3D::getDDS(const filesystem::path &DDSPath, } if (FAILED(HR)) { - spdlog::error(L"Failed to load DDS file from {}: {}", DDSPath.wstring(), ASCIItoUTF16(getHRESULTErrorMessage(HR))); + spdlog::debug(L"Failed to load DDS file from {}: {}", DDSPath.wstring(), ASCIItoUTF16(getHRESULTErrorMessage(HR))); return ParallaxGenTask::PGResult::FAILURE; } @@ -1082,7 +1082,7 @@ auto ParallaxGenD3D::getDDSMetadata(const filesystem::path &DDSPath, } if (FAILED(HR)) { - spdlog::error(L"Failed to load DDS file metadata from {}: {}", DDSPath.wstring(), + spdlog::debug(L"Failed to load DDS file metadata from {}: {}", DDSPath.wstring(), ASCIItoUTF16(getHRESULTErrorMessage(HR))); return ParallaxGenTask::PGResult::FAILURE; } diff --git a/ParallaxGenLib/src/ParallaxGenDirectory.cpp b/ParallaxGenLib/src/ParallaxGenDirectory.cpp index 82b6b9f..850fc0f 100644 --- a/ParallaxGenLib/src/ParallaxGenDirectory.cpp +++ b/ParallaxGenLib/src/ParallaxGenDirectory.cpp @@ -96,6 +96,9 @@ auto ParallaxGenDirectory::mapFiles(const vector &NIFBlocklist, const v #else const size_t NumThreads = boost::thread::hardware_concurrency(); #endif + + bool KillThreads = false; + boost::asio::thread_pool MapTextureFromMeshPool(NumThreads); // Loop through each mesh to confirm textures @@ -122,11 +125,12 @@ auto ParallaxGenDirectory::mapFiles(const vector &NIFBlocklist, const v } if (Multithreading) { - boost::asio::post(MapTextureFromMeshPool, [this, &TaskTracker, &Mesh, &CacheNIFs] { + boost::asio::post(MapTextureFromMeshPool, [this, &TaskTracker, &Mesh, &CacheNIFs, &KillThreads] { ParallaxGenTask::PGResult Result = ParallaxGenTask::PGResult::SUCCESS; try { Result = mapTexturesFromNIF(Mesh, CacheNIFs); } catch (const exception &E) { + KillThreads = true; spdlog::error(L"Exception in thread loading NIF \"{}\": {}", Mesh.wstring(), ASCIItoUTF16(E.what())); Result = ParallaxGenTask::PGResult::FAILURE; } @@ -144,6 +148,13 @@ auto ParallaxGenDirectory::mapFiles(const vector &NIFBlocklist, const v } if (Multithreading) { + while (!TaskTracker.isCompleted()) { + if (KillThreads) { + MapTextureFromMeshPool.stop(); + throw runtime_error("Exception in thread mapping textures from NIFs"); + } + } + // Wait for all threads to complete MapTextureFromMeshPool.join(); } diff --git a/ParallaxGenLib/src/patchers/PatcherShaderTransform.cpp b/ParallaxGenLib/src/patchers/PatcherShaderTransform.cpp index 4d3ce4b..16ab9c4 100644 --- a/ParallaxGenLib/src/patchers/PatcherShaderTransform.cpp +++ b/ParallaxGenLib/src/patchers/PatcherShaderTransform.cpp @@ -1,7 +1,28 @@ #include +#include "NIFUtil.hpp" +#include "ParallaxGenUtil.hpp" #include "patchers/PatcherShaderTransform.hpp" +#include + PatcherShaderTransform::PatcherShaderTransform(std::filesystem::path NIFPath, nifly::NifFile *NIF, - std::string PatcherName) - : Patcher(std::move(NIFPath), NIF, std::move(PatcherName)) {} + std::string PatcherName, const NIFUtil::ShapeShader &From, + const NIFUtil::ShapeShader &To) + : Patcher(std::move(NIFPath), NIF, std::move(PatcherName)), FromShader(From), ToShader(To) {} + +std::unordered_set, + PatcherShaderTransform::ErrorTrackerHasher> + PatcherShaderTransform::ErrorTracker; + +void PatcherShaderTransform::postError(const std::filesystem::path &File) { + if (ErrorTracker.insert({File, FromShader, ToShader}).second) { + spdlog::error(L"Failed to transform from {} to {} for {}", + ParallaxGenUtil::ASCIItoUTF16(NIFUtil::getStrFromShader(FromShader)), + ParallaxGenUtil::ASCIItoUTF16(NIFUtil::getStrFromShader(ToShader)), File.wstring()); + } +} + +auto PatcherShaderTransform::alreadyTried(const std::filesystem::path &File) -> bool { + return ErrorTracker.contains({File, FromShader, ToShader}); +} diff --git a/ParallaxGenLib/src/patchers/PatcherUpgradeParallaxToCM.cpp b/ParallaxGenLib/src/patchers/PatcherUpgradeParallaxToCM.cpp index d4d4444..4e13853 100644 --- a/ParallaxGenLib/src/patchers/PatcherUpgradeParallaxToCM.cpp +++ b/ParallaxGenLib/src/patchers/PatcherUpgradeParallaxToCM.cpp @@ -25,25 +25,32 @@ auto PatcherUpgradeParallaxToCM::getFromShader() -> NIFUtil::ShapeShader { auto PatcherUpgradeParallaxToCM::getToShader() -> NIFUtil::ShapeShader { return NIFUtil::ShapeShader::COMPLEXMATERIAL; } PatcherUpgradeParallaxToCM::PatcherUpgradeParallaxToCM(std::filesystem::path NIFPath, nifly::NifFile *NIF) - : PatcherShaderTransform(std::move(NIFPath), NIF, "UpgradeParallaxToCM") {} + : PatcherShaderTransform(std::move(NIFPath), NIF, "UpgradeParallaxToCM", NIFUtil::ShapeShader::VANILLAPARALLAX, + NIFUtil::ShapeShader::COMPLEXMATERIAL) {} -auto PatcherUpgradeParallaxToCM::transform(const PatcherShader::PatcherMatch &FromMatch) - -> PatcherShader::PatcherMatch { +auto PatcherUpgradeParallaxToCM::transform(const PatcherShader::PatcherMatch &FromMatch, + PatcherShader::PatcherMatch &Result) -> bool { lock_guard Lock(UpgradeCMMutex); const auto HeightMap = FromMatch.MatchedPath; + Result = FromMatch; + + if (alreadyTried(HeightMap)) { + // already tried this file + return false; + } + // Get texture base (remove _p.dds) const auto TexBase = NIFUtil::getTexBase(HeightMap); const filesystem::path ComplexMap = TexBase + L"_m.dds"; - PatcherShader::PatcherMatch Result = FromMatch; Result.MatchedPath = ComplexMap; if (getPGD()->isGenerated(ComplexMap)) { // this was already upgraded - return Result; + return true; } static const auto CMBaseMap = getPGD()->getTextureMapConst(NIFUtil::TextureSlots::ENVMASK); @@ -66,9 +73,10 @@ auto PatcherUpgradeParallaxToCM::transform(const PatcherShader::PatcherMatch &Fr const HRESULT HR = DirectX::SaveToDDSFile(NewComplexMap.GetImages(), NewComplexMap.GetImageCount(), NewComplexMap.GetMetadata(), DirectX::DDS_FLAGS_NONE, OutputPath.c_str()); if (FAILED(HR)) { - Logger::error(L"Unable to save complex material {}: {}", OutputPath.wstring(), + Logger::debug(L"Unable to save complex material {}: {}", OutputPath.wstring(), ParallaxGenUtil::ASCIItoUTF16(ParallaxGenD3D::getHRESULTErrorMessage(HR))); - return FromMatch; + postError(HeightMap); + return false; } // add newly created file to complexMaterialMaps for later processing @@ -81,8 +89,9 @@ auto PatcherUpgradeParallaxToCM::transform(const PatcherShader::PatcherMatch &Fr Logger::debug(L"Generated complex material map: {}", ComplexMap.wstring()); - return Result; + return true; } - return FromMatch; + postError(HeightMap); + return false; } diff --git a/ParallaxGenLib/src/patchers/PatcherUtil.cpp b/ParallaxGenLib/src/patchers/PatcherUtil.cpp index 4141362..067565f 100644 --- a/ParallaxGenLib/src/patchers/PatcherUtil.cpp +++ b/ParallaxGenLib/src/patchers/PatcherUtil.cpp @@ -56,10 +56,11 @@ auto PatcherUtil::applyTransformIfNeeded(const ShaderPatcherMatch &Match, auto *const Transform = Patchers.ShaderTransformPatchers.at(Match.Shader).at(Match.ShaderTransformTo).get(); // Transform Shader - TransformedMatch.Match = Transform->transform(Match.Match); - TransformedMatch.Shader = Match.ShaderTransformTo; + if (Transform->transform(Match.Match, TransformedMatch.Match)) { + // Reset Transform + TransformedMatch.Shader = Match.ShaderTransformTo; + } - // Reset Transform TransformedMatch.ShaderTransformTo = NIFUtil::ShapeShader::NONE; }