added --fix-complex-terrain feature, untested

CHANGELOG.md

@@ -4,6 +4,8 @@
 
 - GPU code will now verify textures are a power of 2
 - Added "skyrimgog" game type
+- Added --fix-complex-terrain CLI argument which will create normal height maps and/or complex material maps for complex terrain parallax for use in blending meshes
+- Switched to using regex for file lookups
 
 ## [0.4.6] - 2024-07-23
 

include/BethesdaDirectory/BethesdaDirectory.hpp

@@ -11,6 +11,7 @@
 #include <vector>
 #include <bsa/tes4.hpp>
 #include <boost/algorithm/string.hpp>
+#include <regex>
 
 #include "BethesdaGame/BethesdaGame.hpp"
 
@@ -73,7 +74,7 @@ class BethesdaDirectory {
 	bool isBSAFile(const std::filesystem::path rel_path) const;
 	bool isFile(const std::filesystem::path rel_path) const;
 	std::filesystem::path getFullPath(const std::filesystem::path rel_path) const;
-	std::vector<std::filesystem::path> findFilesBySuffix(const std::string_view suffix, const bool lower = false, const std::vector<std::wstring>& parent_blocklist = std::vector<std::wstring>()) const;
+	std::vector<std::filesystem::path> findFiles(const std::wregex& pattern, const bool lower = false, const std::vector<std::wstring>& component_blocklist = std::vector<std::wstring>(), const std::vector<std::wstring>& component_whitelist = std::vector<std::wstring>()) const;
 	std::filesystem::path getDataPath() const;
 
 	// BSA functions

include/ParallaxGen/ParallaxGen.hpp

@@ -50,6 +50,8 @@ class ParallaxGen
 
     // constructor
     ParallaxGen(const std::filesystem::path output_dir, ParallaxGenDirectory* pgd, ParallaxGenD3D* pgd3d, bool optimize_meshes = false, bool ignore_parallax = false, bool ignore_complex_material = false);
+	// generate regular parallax for complex terrain
+	void fixComplexTerrain();
 	// upgrades textures whenever possible
 	void upgradeShaders();
     // enables parallax on relevant meshes

include/ParallaxGenD3D/ParallaxGenD3D.hpp

@@ -25,6 +25,7 @@ class ParallaxGenD3D
     const UINT NUM_GPU_THREADS = 16;
 
     Microsoft::WRL::ComPtr<ID3D11ComputeShader> shader_MergeToComplexMaterial;
+    Microsoft::WRL::ComPtr<ID3D11ComputeShader> shader_ExtractParallaxFromAlpha;
     struct shader_MergeToComplexMaterial_params
     {
         DirectX::XMFLOAT2 EnvMapScalingFactor;
@@ -49,9 +50,14 @@ class ParallaxGenD3D
     // GPU functions
     void initGPU();
 
+    // Extract parallax map from complex terrain
+    DirectX::ScratchImage extractParallaxMapFromAlpha(const std::filesystem::path& dds) const;
+
     // Attempt to upgrade vanilla parallax to complex material
     DirectX::ScratchImage upgradeToComplexMaterial(const std::filesystem::path& parallax_map, const std::filesystem::path& env_map) const;
 
+    // Other Helpers
+    static std::string getHRESULTErrorMessage(HRESULT hr);
 private:
     // GPU functions
     void initShaders();
@@ -77,9 +83,6 @@ class ParallaxGenD3D
     bool getDDS(const std::filesystem::path& dds_path, DirectX::ScratchImage& dds) const;
     bool getDDSMetadata(const std::filesystem::path& dds_path, DirectX::TexMetadata& dds_meta) const;
     static DirectX::ScratchImage LoadRawPixelsToScratchImage(const std::vector<unsigned char> rawPixels, size_t width, size_t height, DXGI_FORMAT format, int channels);
-
-    // Other Helpers
-    static std::string getHRESULTErrorMessage(HRESULT hr);
 };
 
 #endif

include/ParallaxGenDirectory/ParallaxGenDirectory.hpp

@@ -4,6 +4,7 @@
 #include <vector>
 #include <filesystem>
 #include <array>
+#include <regex>
 
 #include "BethesdaDirectory/BethesdaDirectory.hpp"
 
@@ -25,9 +26,14 @@ class ParallaxGenDirectory : public BethesdaDirectory {
 		L"water"
 	};
 
+	static inline const std::vector<std::wstring> landscape_paths = {
+		L"landscape"
+	};
+
 	std::vector<std::filesystem::path> heightMaps;
 	std::vector<std::filesystem::path> complexMaterialMaps;
 	std::vector<std::filesystem::path> meshes;
+	std::vector<std::filesystem::path> complexTerrainParallaxMaps;
 
 public:
 	static inline const std::filesystem::path default_cubemap_path = "textures\\cubemaps\\dynamic1pxcubemap_black.dds";
@@ -41,6 +47,8 @@ class ParallaxGenDirectory : public BethesdaDirectory {
 	void findComplexMaterialMaps();
 	// searches for meshes in the data directory
 	void findMeshes();
+	// searches for complex terrain parallax files
+	void findComplexTerrainParallaxMaps();
 
 	// add methods
 	void addHeightMap(std::filesystem::path path);
@@ -58,9 +66,11 @@ class ParallaxGenDirectory : public BethesdaDirectory {
 	const std::vector<std::filesystem::path> getHeightMaps() const;
 	const std::vector<std::filesystem::path> getComplexMaterialMaps() const;
 	const std::vector<std::filesystem::path> getMeshes() const;
+	const std::vector<std::filesystem::path> getComplexTerrainParallaxMaps() const;
 
 	// helpers
-	static std::filesystem::path changeDDSSuffix(std::filesystem::path path, std::wstring suffix);
+	static std::wstring buildComponentExclusionRegex(const std::vector<std::wstring>& exclusions);
+	bool doesDDSHaveAlpha(const std::filesystem::path& path) const;
 };
 
 #endif

include/ParallaxGenUtil/ParallaxGenUtil.hpp

@@ -25,6 +25,8 @@ namespace ParallaxGenUtil
 
 	std::vector<std::byte> getFileBytes(const std::filesystem::path& file_path);
 
+	std::filesystem::path pathReplaceLast(const std::filesystem::path& path, const std::wstring& from, const std::wstring& to);
+
 	// concatenates two vectors without duplicates
 	template <typename T>
 	void concatenateVectorsWithoutDuplicates(std::vector<T>& vec1, const std::vector<T>& vec2) {

shaders/ExtractParallaxFromAlpha.hlsl

@@ -0,0 +1,14 @@
+// Global Registers
+Texture2D<float4> Input : register(t0);
+RWTexture2D<float> Output : register(u0);
+
+// Define the thread group size
+[numthreads(16, 16, 1)]
+void main(uint3 id : SV_DispatchThreadID)
+{
+    // Load alpha value
+    float input_alpha = Input.Load(uint3(id.xy, 0)).a;
+
+    // Write to the output texture
+    Output[id.xy] = input_alpha;
+}

src/BethesdaDirectory/BethesdaDirectory.cpp

@@ -142,7 +142,7 @@ fs::path BethesdaDirectory::getDataPath() const
 	return data_dir;
 }
 
-vector<fs::path> BethesdaDirectory::findFilesBySuffix(const string_view suffix, const bool lower, const vector<wstring>& path_blocklist) const
+vector<fs::path> BethesdaDirectory::findFiles(const wregex& pattern, const bool lower, const vector<wstring>& component_blocklist, const vector<wstring>& component_whitelist) const
 {
 	// find all keys in fileMap that match pattern
 	vector<fs::path> found_files;
@@ -152,19 +152,28 @@ vector<fs::path> BethesdaDirectory::findFilesBySuffix(const string_view suffix,
 		fs::path cur_file_path = value.path;
 
 		if (logging) {
-			spdlog::trace(L"Checking file for suffix {} match: {}", convertToWstring(string(suffix)), cur_file_path.wstring());
+			spdlog::trace(L"Checking file for regex match: {}", cur_file_path.wstring());
 		}
 
-        if (boost::algorithm::ends_with(key.wstring(), suffix)) {
-			// check if any component of the path is in the blocklist
-			bool blocked = checkIfAnyComponentIs(cur_file_path, path_blocklist);
-			if (blocked)
-			{
+		if (regex_search(cur_file_path.wstring(), pattern)) {
+			// Check if any component of the path is in the blocklist
+			if (!component_blocklist.empty() && checkIfAnyComponentIs(cur_file_path, component_blocklist)) {
+				if (logging) {
+					spdlog::trace(L"Matched file by regex but blocked by component blocklist: {}", cur_file_path.wstring());
+				}
+				continue;
+			}
+
+			// Check if any component of the path is in the whitelist
+			if (!component_whitelist.empty() && !checkIfAnyComponentIs(cur_file_path, component_whitelist)) {
+				if (logging) {
+					spdlog::trace(L"Matched file by regex but not in component whitelist: {}", cur_file_path.wstring());
+				}
 				continue;
 			}
 
 			if (logging) {
-				spdlog::trace(L"Matched file by suffix: {}", cur_file_path.wstring());
+				spdlog::trace(L"Matched file by regex: {}", cur_file_path.wstring());
 			}
 
 			if (lower) {

src/ParallaxGen/ParallaxGen.cpp

@@ -35,6 +35,57 @@ ParallaxGen::ParallaxGen(const fs::path output_dir, ParallaxGenDirectory* pgd, P
 	this->ignore_complex_material = ignore_complex_material;
 }
 
+void ParallaxGen::fixComplexTerrain()
+{
+	spdlog::info("Starting to create reegular parallax for complex terrain...");
+
+	//loop through height maps
+	size_t finished_task = 0;
+
+	auto ctMaps = pgd->getComplexTerrainParallaxMaps();
+	size_t num_upgrades = ctMaps.size();
+	for (fs::path ct_map : ctMaps) {
+		spdlog::trace(L"Processing complex terrain map: {}", ct_map.wstring());
+
+		if (finished_task % 10 == 0) {
+			double progress = (double)finished_task / num_upgrades * 100.0;
+			spdlog::info("Complex Terrain Maps Processed: {}/{} ({:.1f}%)", finished_task, num_upgrades, progress);
+		}
+
+		// Replace "_p" with "_m" in the stem
+		fs::path height_map_path = ParallaxGenUtil::pathReplaceLast(ct_map, L".dds", L"_p.dds");
+
+		if (pgd->isFile(height_map_path)) {
+			// already upgraded
+			finished_task++;
+			continue;
+		}
+
+		// upgrade to complex material
+		DirectX::ScratchImage new_height_map = pgd3d->extractParallaxMapFromAlpha(ct_map);
+
+		// save to file
+		if (new_height_map.GetImageCount() > 0) {
+			fs::path output_path = output_dir / height_map_path;
+			fs::create_directories(output_path.parent_path());
+
+			HRESULT hr = DirectX::SaveToDDSFile(new_height_map.GetImages(), new_height_map.GetImageCount(), new_height_map.GetMetadata(), DirectX::DDS_FLAGS_NONE, output_path.c_str());
+			if (FAILED(hr)) {
+				spdlog::error(L"Unable to save complex material {}: {}", output_path.wstring(), ParallaxGenUtil::convertToWstring(ParallaxGenD3D::getHRESULTErrorMessage(hr)));
+			}
+
+			// add newly created file to complexMaterialMaps for later processing
+			pgd->addHeightMap(height_map_path);
+
+			spdlog::debug(L"Added complex material map: {}", height_map_path.wstring());
+		} else {
+			spdlog::warn(L"Unable to upgrade height map, skipping: {}", ct_map.wstring());
+		}
+
+		finished_task++;
+	}
+}
+
 void ParallaxGen::upgradeShaders()
 {
 	spdlog::info("Starting shader upgrade process...");
@@ -53,7 +104,7 @@ void ParallaxGen::upgradeShaders()
 		}
 
 		// Replace "_p" with "_m" in the stem
-		fs::path env_map_path = pgd->changeDDSSuffix(height_map, L"_m");
+		fs::path env_map_path = ParallaxGenUtil::pathReplaceLast(height_map, L"_p.dds", L"_m.dds");
 		fs::path complex_map_path = env_map_path;
 
 		if (pgd->isFile(env_map_path) && pgd->isComplexMaterialMap(env_map_path)) {
@@ -78,7 +129,7 @@ void ParallaxGen::upgradeShaders()
 
 			HRESULT hr = DirectX::SaveToDDSFile(new_ComplexMap.GetImages(), new_ComplexMap.GetImageCount(), new_ComplexMap.GetMetadata(), DirectX::DDS_FLAGS_NONE, output_path.c_str());
 			if (FAILED(hr)) {
-				spdlog::error(L"Unable to save complex material {}: {}", env_map_path.wstring(), hr);
+				spdlog::error(L"Unable to save complex material {}: {}", env_map_path.wstring(), ParallaxGenUtil::convertToWstring(ParallaxGenD3D::getHRESULTErrorMessage(hr)));
 			}
 
 			// add newly created file to complexMaterialMaps for later processing
@@ -304,6 +355,7 @@ void ParallaxGen::processNIF(const fs::path& nif_file, vector<fs::path>& heightM
 
 				// check if this is weapon or armor
 				bool dynCubemaps = true;
+				// TODO regex instead here?
 				if (pgd->checkIfAnyComponentIs(nif_file, dynCubemap_ignore_list) || pgd->checkIfAnyComponentIs(search_prefix, dynCubemap_ignore_list)) {
 					dynCubemaps = false;
 				}

src/ParallaxGenD3D/ParallaxGenD3D.cpp

@@ -86,6 +86,12 @@ void ParallaxGenD3D::initShaders()
         spdlog::critical("Failed to create compute shader. Exiting.");
         ParallaxGenUtil::exitWithUserInput(1);
     }
+
+    // ExtractParallaxFromAlpha.hlsl
+    if (!createComputeShader(L"shaders/ExtractParallaxFromAlpha.hlsl", shader_ExtractParallaxFromAlpha)) {
+        spdlog::critical("Failed to create compute shader. Exiting.");
+        ParallaxGenUtil::exitWithUserInput(1);
+    }
 }
 
 ComPtr<ID3DBlob> ParallaxGenD3D::compileShader(const std::filesystem::path& filename) {
@@ -130,7 +136,113 @@ bool ParallaxGenD3D::createComputeShader(const wstring& shader_path, ComPtr<ID3D
     return true;
 }
 
-DirectX::ScratchImage ParallaxGenD3D::upgradeToComplexMaterial(const std::filesystem::path& parallax_map, const std::filesystem::path& env_map) const
+DirectX::ScratchImage ParallaxGenD3D::extractParallaxMapFromAlpha(const filesystem::path& dds) const
+{
+    // Load DDS file
+    DirectX::ScratchImage dds_image;
+    if (!getDDS(dds, dds_image)) {
+        spdlog::warn(L"Failed to load DDS file: {} (skipping)", dds.wstring());
+        return DirectX::ScratchImage();
+    }
+
+    // Get metadata
+    DirectX::TexMetadata dds_image_meta = dds_image.GetMetadata();
+
+    // Create GPU texture
+    ComPtr<ID3D11Texture2D> dds_image_gpu = createTexture2D(dds_image);
+    if (!dds_image_gpu) {
+        return DirectX::ScratchImage();
+    }
+
+    // Create SRV for GPU texture
+    ComPtr<ID3D11ShaderResourceView> dds_image_srv = createShaderResourceView(dds_image_gpu);
+    if (!dds_image_srv) {
+        return DirectX::ScratchImage();
+    }
+
+    // Create output texture
+    // TODO the format should really be set based on the bytes of the input
+    D3D11_TEXTURE2D_DESC output_texture_desc;
+    dds_image_gpu->GetDesc(&output_texture_desc);
+    output_texture_desc.Format = DXGI_FORMAT_R8_UNORM;
+    output_texture_desc.BindFlags = D3D11_BIND_UNORDERED_ACCESS | D3D11_BIND_SHADER_RESOURCE;
+    output_texture_desc.MipLevels = 1;
+    output_texture_desc.ArraySize = 1;
+    output_texture_desc.SampleDesc.Count = 1;
+    output_texture_desc.Usage = D3D11_USAGE_DEFAULT;
+
+    ComPtr<ID3D11Texture2D> output_texture = createTexture2D(output_texture_desc);
+    if (!output_texture) {
+        return DirectX::ScratchImage();
+    }
+
+    // Create UAV for output texture
+    ComPtr<ID3D11UnorderedAccessView> output_uav = createUnorderedAccessView(output_texture);
+    if (!output_uav) {
+        return DirectX::ScratchImage();
+    }
+
+    // Dispatch shader
+    pContext->CSSetShader(shader_ExtractParallaxFromAlpha.Get(), nullptr, 0);
+    pContext->CSSetShaderResources(0, 1, dds_image_srv.GetAddressOf());
+    pContext->CSSetUnorderedAccessViews(0, 1, output_uav.GetAddressOf(), nullptr);
+
+    UINT output_width = static_cast<UINT>(dds_image_meta.width);
+    UINT output_height = static_cast<UINT>(dds_image_meta.height);
+
+    bool result_bool = BlockingDispatch(output_width, output_height, 1);
+    if (!result_bool) {
+        return DirectX::ScratchImage();
+    }
+
+    // Clean up shader resources
+    ID3D11ShaderResourceView* null_srv[] = { nullptr, nullptr };
+    pContext->CSSetShaderResources(0, 1, null_srv);
+    ID3D11UnorderedAccessView* null_uav[] = { nullptr, nullptr };
+    pContext->CSSetUnorderedAccessViews(0, 1, null_uav, nullptr);
+    ID3D11ComputeShader* null_shader[] = { nullptr };
+    pContext->CSSetShader(null_shader[0], nullptr, 0);
+
+    // Release some objects
+    dds_image_gpu.Reset();
+    dds_image_srv.Reset();
+
+    // Read back texture (DEBUG)
+    auto output_texture_data = readBack(output_texture, 1);
+
+    // More cleaning
+    output_texture.Reset();
+    output_uav.Reset();
+
+    // Flush GPU to avoid leaks
+    pContext->Flush();
+
+    // Import into directx scratchimage
+    if (dds.filename().wstring() == L"dirt01.dds") {
+        spdlog::trace("HERE");
+    }
+    DirectX::ScratchImage output_image = LoadRawPixelsToScratchImage(output_texture_data, output_width, output_height, DXGI_FORMAT_R8_UNORM, 1);
+
+    // Compress dds
+    // BC4 works best with heightmaps
+    DirectX::ScratchImage compressed_image;
+    HRESULT hr = DirectX::Compress(
+        output_image.GetImages(),
+        output_image.GetImageCount(),
+        output_image.GetMetadata(),
+        DXGI_FORMAT_BC4_UNORM,
+        DirectX::TEX_COMPRESS_DEFAULT,
+        1.0f,
+        compressed_image);
+    if (FAILED(hr)) {
+        spdlog::debug("Failed to compress output DDS file: {}", getHRESULTErrorMessage(hr));
+        return DirectX::ScratchImage();
+    }
+
+    return compressed_image;
+}
+
+DirectX::ScratchImage ParallaxGenD3D::upgradeToComplexMaterial(const filesystem::path& parallax_map, const filesystem::path& env_map) const
 {
     bool parallax_exists = !parallax_map.empty();
     bool env_exists = !env_map.empty();