diff --git a/pyproject.toml b/pyproject.toml index 87485a3..401aea0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "construct>=2.10.70", # 2.10.70 includes a fix for compiled structs building "randovania-lupa>=2.0.1", "zstandard", + "py-tegra-swizzle>=0.1.0", ] [project.readme] diff --git a/src/mercury_engine_data_structures/exporters/dds.py b/src/mercury_engine_data_structures/exporters/dds.py new file mode 100644 index 0000000..84f7e1d --- /dev/null +++ b/src/mercury_engine_data_structures/exporters/dds.py @@ -0,0 +1,273 @@ +from __future__ import annotations + +from enum import IntEnum +from typing import TYPE_CHECKING + +from construct.core import ( + Bytes, + Const, + FlagsEnum, + GreedyBytes, + If, + Int32ul, + Struct, + this, +) +from construct.lib.containers import Container + +from mercury_engine_data_structures.formats.bctex import XTX_Tegra_Format + +if TYPE_CHECKING: + from pathlib import Path + + from mercury_engine_data_structures.exporters.raw_texture import Array, RawTexture + + +class DDPF_FLAGS(IntEnum): + DDPF_ALPHAPIXELS = 0x1 + DDPF_ALPHA = 0x2 + DDPF_FOURCC = 0x4 + DDPF_RGB = 0x40 + DDPF_YUV = 0x200 + DDPF_LUMINANCE = 0x20000 + + +DDS_PIXELFORMAT = Struct( + "dwSize" / Const(32, Int32ul), + "dwFlags" / FlagsEnum(Int32ul, DDPF_FLAGS), + "dwFourCC" / Bytes(4), + "dwRGBBitCount" / Int32ul, + "dwRBitMask" / Int32ul, + "dwGBitMask" / Int32ul, + "dwBBitMask" / Int32ul, + "dwABitMask" / Int32ul, +) + + +class DDS_FLAGS(IntEnum): + DDSD_CAPS = 0x1 + DDSD_HEIGHT = 0x2 + DDSD_WIDTH = 0x4 + DDSD_PITCH = 0x8 + DDSD_PIXELFORMAT = 0x1000 + DDSD_MIPMAPCOUNT = 0x20000 + DDSD_LINEARSIZE = 0x80000 + DDSD_DEPTH = 0x800000 + + +class CAPS_FLAGS(IntEnum): + DDSCAPS_COMPLEX = 0x8 + DDSCAPS_TEXTURE = 0x1000 + DDSCAPS_MIPMAP = 0x400000 + + +class CAPS2_FLAGS(IntEnum): + DDSCAPS2_CUBEMAP = 0x200 + DDSCAPS2_CUBEMAP_POSITIVEX = 0x400 + DDSCAPS2_CUBEMAP_NEGATIVEX = 0x800 + DDSCAPS2_CUBEMAP_POSITIVEY = 0x1000 + DDSCAPS2_CUBEMAP_NEGATIVEY = 0x2000 + DDSCAPS2_CUBEMAP_POSITIVEZ = 0x4000 + DDSCAPS2_CUBEMAP_NEGATIVEZ = 0x8000 + DDSCAPS2_VOLUME = 0x200000 + + +DDS_HEADER = Struct( + "dwSize" / Const(124, Int32ul), + "dwFlags" / FlagsEnum(Int32ul, DDS_FLAGS), + "dwHeight" / Int32ul, + "dwWidth" / Int32ul, + "dwPitchOrLinearSize" / Int32ul, + "dwDepth" / Int32ul, + "dwMipMapCount" / Int32ul, + "dwReserved1" / Int32ul[11], + "ddspf" / DDS_PIXELFORMAT, + "dwCaps" / FlagsEnum(Int32ul, CAPS_FLAGS), + "dwCaps2" / FlagsEnum(Int32ul, CAPS2_FLAGS), + "dwCaps3" / Int32ul, + "dwCaps4" / Int32ul, + "dwReserved2" / Int32ul, +) + + +class D3D10_RESOURCEs_DIMENSION(IntEnum): + D3D10_RESOURCE_DIMENSION_UNKNOWN = 0 + D3D10_RESOURCE_DIMENSION_BUFFER = 1 + D3D10_RESOURCE_DIMENSION_TEXTURE1D = 2 + D3D10_RESOURCE_DIMENSION_TEXTURE2D = 3 + D3D10_RESOURCE_DIMENSION_TEXTURE3D = 4 + + +DDS_HEADER_DX10 = Struct( + "dxgiFormat" / Const(95, Int32ul), # only cubemaps are exported as DX10 + "resourceDimension" / Const(3, Int32ul), + "miscFlag" / Const(4, Int32ul), # 0x4 = cubemap + "arraySize" / Int32ul, + "miscFlags2" / Const(0, Int32ul), +) + +DDS = Struct( + "_magic" / Const(b"DDS "), + "header" / DDS_HEADER, + "header10" / If(this.header.dwFlags.DDPF_FOURCC and this.header.ddspf.dwFourCC == b"DX10", DDS_HEADER_DX10), + "data" / GreedyBytes, +) + +_EMPTY_FOURCC = b"\x00" * 4 +DXGI_FORMATS: dict[XTX_Tegra_Format, tuple[bool, bytes]] = { + XTX_Tegra_Format.R8_UNORM: (False, _EMPTY_FOURCC), + XTX_Tegra_Format.R8G8_UNORM: (False, _EMPTY_FOURCC), + XTX_Tegra_Format.R8G8B8A8_UNORM: (False, _EMPTY_FOURCC), + XTX_Tegra_Format.BC1_UNORM: (True, b"DXT1"), + XTX_Tegra_Format.BC3_UNORM: (True, b"DXT5"), + XTX_Tegra_Format.BC5_UNORM: (True, b"BC5U"), + XTX_Tegra_Format.BC6H_UF16: (True, b"DX10"), + XTX_Tegra_Format.B8G8R8A8_UNORM: (False, _EMPTY_FOURCC), +} + + +# caps, width, height, pixelformat, mipmapcount +STANDARD_DDSD_FLAGS = 0x21007 + +# caps without mips +_CAPS_NOMIP = CAPS_FLAGS.DDSCAPS_TEXTURE +_CAPS_MIP = CAPS_FLAGS.DDSCAPS_COMPLEX | CAPS_FLAGS.DDSCAPS_TEXTURE | CAPS_FLAGS.DDSCAPS_MIPMAP +# cubemap, all faces +_CAPS2_CUBEMAP = 0xFE00 + + +class DdsExporter: + raw: RawTexture + dds_files: list[bytes] + + def __init__(self, raw: RawTexture) -> None: + self.raw = raw + self.build_dds() + + def build_dds(self): + """ + Generates a list of raw DDS files in `self.dds_files`. + Should always be one texture per BCTEX, but haven't confirmed. + """ + texture_count = len(self.raw.textures) + if texture_count == 0: + raise ValueError("Not enough textures!") + + res = [] + for arr in self.raw.textures: + dds = self._build_dds(arr) + res.append(dds) + + self.dds_files = res + + def _build_dds(self, array: Array) -> bytes: + """ + Builds an array into a dds file + + See: https://learn.microsoft.com/en-us/windows/win32/direct3ddds/dx-graphics-dds-pguide + """ + + array_size = len(array.members) + if array_size == 0: + raise ValueError("No textures in arrays!") + + is_block_compressed, dxgi_fourcc = DXGI_FORMATS[array.format] + + # handle differences between BC and uncompressed formats + pitch = array.width * array.height * array.format.bytes_per_pixel + dds_flags = STANDARD_DDSD_FLAGS | DDS_FLAGS.DDSD_LINEARSIZE + ddpf_flags = 0 + if is_block_compressed: + pitch //= 16 + ddpf_flags = 4 + + pixelformat = Container( + dwSize=32, + dwFlags=ddpf_flags, + dwFourCC=dxgi_fourcc, + dwRGBBitCount=0, + dwRBitMask=0, + dwGBitMask=0, + dwBBitMask=0, + dwABitMask=0, + ) + + if array.format == XTX_Tegra_Format.R8G8_UNORM: + pixelformat.dwFlags = 0x41 + pixelformat.dwRGBBitCount = 0x18 + pixelformat.dwRBitMask = 0xFF << 16 + pixelformat.dwGBitMask = 0xFF << 8 + pixelformat.dwBBitMask = 0xFF + elif array.format == XTX_Tegra_Format.R8G8B8A8_UNORM: + pixelformat.dwFlags = 0x41 + pixelformat.dwRGBBitCount = 0x20 + pixelformat.dwRBitMask = 0xFF + pixelformat.dwGBitMask = 0xFF << 8 + pixelformat.dwBBitMask = 0xFF << 16 + pixelformat.dwABitMask = 0xFF << 24 + + header = Container( + dwSize=124, + dwFlags=dds_flags, + dwHeight=array.height, + dwWidth=array.width, + dwPitchOrLinearSize=pitch, + dwDepth=1, + dwMipMapCount=len(array.members[0].mips), + dwReserved1=[0] * 11, + ddspf=pixelformat, + dwCaps=_CAPS_MIP if len(array.members[0].mips) > 1 else _CAPS_NOMIP, + dwCaps2=_CAPS2_CUBEMAP if array_size == 6 else 0, + dwCaps3=0, + dwCaps4=0, + dwReserved2=0, + ) + + if dxgi_fourcc == b"DX10": + header10 = Container( + dxgiFormat=95, # only cubemaps are exported as dx10 + resourceDimension=3, + miscFlag=4, + arraySize=array_size // 6, + miscFlags2=0, + ) + else: + header10 = None + + data = b"" + for tex in array.members: + for mip in tex.mips: + data += mip.data + + dds = Container(_magic=b"DDS ", header=header, header10=header10, data=data) + + res = DDS.build(dds) + return res + + def save_dds(self, folder: Path, name: str = None): + """ + Exports a .dds file to the given folder. + + If there are multiple images contained in a single BCTEX (which there aren't in vanilla), + they are placed in a folder with the given name. + + :param folder: folder to write the dds file to + :param name: name of the file. default is `{self.raw.name}.dds` + """ + + folder.mkdir(parents=True, exist_ok=True) + if not name: + name = self.raw.name + ".dds" + + if len(self.dds_files) == 1: + folder.joinpath(name).write_bytes(self.dds_files[0]) + + elif len(self.dds_files) > 1: + multi_export = folder.joinpath(name) + multi_export.mkdir(parents=True, exist_ok=True) + + for i, raw in enumerate(self.dds_files): + multi_export.joinpath(f"image_{i}.dds").write_bytes(raw) + + else: + raise ValueError("Did not find any DDS data!") diff --git a/src/mercury_engine_data_structures/exporters/raw_texture.py b/src/mercury_engine_data_structures/exporters/raw_texture.py new file mode 100644 index 0000000..ade7d31 --- /dev/null +++ b/src/mercury_engine_data_structures/exporters/raw_texture.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import dataclasses +import math +from typing import TYPE_CHECKING + +import py_tegra_swizzle + +from mercury_engine_data_structures.formats.bctex import Bctex, BlockType, XTX_Tegra_Format +from mercury_engine_data_structures.game_check import Game + +if TYPE_CHECKING: + from construct.core import Container + + +def div_round_up(n: int, d: int) -> int: + return (n + d - 1) // d + + +@dataclasses.dataclass +class Surface: + width: int + height: int + data: bytes + + +@dataclasses.dataclass +class Texture2D: + mips: list[Surface] + + def mip0(self) -> Surface: + return self.mips[0] + + +@dataclasses.dataclass +class Array: + width: int + height: int + format: XTX_Tegra_Format + members: list[Texture2D] + + +class RawTexture: + bctex: Bctex + name: str + textures: list[Array] + + def __init__(self, texture: Bctex) -> None: + if texture.target_game != Game.DREAD: + raise ValueError("Only Dread bctex can be exported!") + + self.bctex = texture + self.name = texture.raw.data.name + + self.parse() + + def parse(self): + self.textures = [] + blocks = self.bctex.raw.data.xtx.blocks + infos = [blk for blk in blocks if blk.block_type == BlockType.TEXTURE.name] + datas = [blk for blk in blocks if blk.block_type == BlockType.DATA.name] + + for info, data in zip(infos, datas): + self.textures.append(self.parse_array(info, data)) + + @dataclasses.dataclass + class Mip0Data: + # data calculated at the array level and used for texture2d/surface handling + format: XTX_Tegra_Format + width: int + height: int + block_height: int + block_dim: py_tegra_swizzle.PyBlockDim + + def parse_array(self, info: Container, data: Container) -> Array: + xtx_format = [x for x in XTX_Tegra_Format if x.name == info.data.xtx_format][0] + width = info.data.width + height = info.data.height + assert info.data.depth == 1 + block_height_mip0 = py_tegra_swizzle.block_height_mip0(div_round_up(height, xtx_format.block_height)) + block_dim = py_tegra_swizzle.PyBlockDim(xtx_format.block_width, xtx_format.block_height, xtx_format.block_depth) + + mip0_data = self.Mip0Data(xtx_format, width, height, block_height_mip0, block_dim) + + res = Array(width, height, xtx_format, []) + array_size = info.data.data_size // info.data.slice_size # usually 1, but 6 for cubemaps + for array_level in range(array_size): + mipped_surface = self.parse_texture2d(info, data, info.data.slice_size * array_level, mip0_data) + res.members.append(mipped_surface) + + return res + + def parse_texture2d(self, info: Container, data: Container, array_offset: int, mip0: Mip0Data) -> Texture2D: + mips = [] + + mip_offset = 0 + for mip_level in range(info.data.mip_count): + mip_width = max(1, mip0.width >> mip_level) + mip_height = max(1, mip0.height >> mip_level) + mip_depth = max(1, 1 >> mip_level) + + mip_height_in_blocks = div_round_up(mip_height, mip0.format.block_height) + + mip_block_height = py_tegra_swizzle.mip_block_height(mip_height_in_blocks, mip0.block_height) + block_height_log2 = math.floor(math.log2(mip_block_height)) + + mip_size = py_tegra_swizzle.get_swizzled_surface_size( + mip_width, mip_height, mip_depth, mip0.block_dim, mip0.block_height, mip0.format.bytes_per_pixel + ) + mip_start = array_offset + mip_offset + + mip_data = data.data[mip_start : mip_start + mip_size] + + deswizzled = self._deswizzle(mip_width, mip_height, mip_depth, mip0, block_height_log2, mip_data) + + if deswizzled is None: + raise ValueError(f"Deswizzle Failed, mip={mip_level}") + + mips.append(Surface(mip_width, mip_height, deswizzled)) + mip_offset += mip_size + + return Texture2D(mips) + + def _deswizzle(self, width: int, height: int, depth: int, mip0: Mip0Data, heightLog2: int, data: bytes) -> bytes: + height_mip0 = 1 << max(min(heightLog2, 5), 0) + + width_blks = div_round_up(width, mip0.format.block_width) + height_blks = div_round_up(height, mip0.format.block_height) + depth_blks = div_round_up(depth, mip0.format.block_depth) + + try: + res = py_tegra_swizzle.deswizzle_block_linear( + width_blks, height_blks, depth_blks, data, height_mip0, mip0.format.bytes_per_pixel + ) + return res + except Exception: + return None diff --git a/src/mercury_engine_data_structures/formats/bctex.py b/src/mercury_engine_data_structures/formats/bctex.py index 483b1a3..be09375 100644 --- a/src/mercury_engine_data_structures/formats/bctex.py +++ b/src/mercury_engine_data_structures/formats/bctex.py @@ -1,6 +1,6 @@ from __future__ import annotations -from enum import Enum +from enum import Enum, IntEnum import construct from construct.core import Error @@ -9,11 +9,38 @@ from mercury_engine_data_structures.common_types import StrId, UInt from mercury_engine_data_structures.game_check import Game -BlockType = construct.Enum( - UInt, - texture=2, - data=3, -) + +class BlockType(IntEnum): + TEXTURE = 2 + DATA = 3 + + +class XTX_Tegra_Format(IntEnum): + bytes_per_pixel: int + block_width: int + block_height: int + block_depth: int + + R8_UNORM = 0x1, 1, 1, 1, 1 + R8G8_UNORM = 0xD, 2, 1, 1, 1 + R8G8B8A8_UNORM = 0x25, 4, 1, 1, 1 + BC1_UNORM = 0x42, 8, 4, 4, 1 + BC3_UNORM = 0x44, 16, 4, 4, 1 + BC5_UNORM = 0x4B, 16, 4, 4, 1 + BC6H_UF16 = 0x50, 16, 4, 4, 1 + B8G8R8A8_UNORM = 0x6D, 4, 1, 1, 1 + + def __new__(cls, value: int, bpp: int, bW: int, bH: int, bD: int) -> XTX_Tegra_Format: + member = int.__new__(cls, value) + member._value_ = value + member.bytes_per_pixel = bpp + member.block_width = bW + member.block_height = bH + member.block_depth = bD + return member + + +XTXFormatConstruct = construct.Enum(UInt, XTX_Tegra_Format) XTX_TextureBlock = construct.Struct( data_size=construct.Int64ul, @@ -22,7 +49,7 @@ height=UInt, depth=UInt, target=UInt, - xtx_format=UInt, + xtx_format=XTXFormatConstruct, mip_count=UInt, slice_size=UInt, mip_offsets=UInt[17], @@ -38,7 +65,7 @@ block_size=UInt, data_size=construct.Int64ul, data_offset=construct.Int64sl, - block_type=BlockType, + block_type=construct.Enum(UInt, BlockType), global_block_index=UInt, inc_block_type_index=UInt, _data_seek=construct.Seek(construct.this._start + construct.this.data_offset), @@ -47,7 +74,7 @@ construct.Switch( construct.this.block_type, { - BlockType.texture: XTX_TextureBlock, + BlockType.TEXTURE.name: XTX_TextureBlock, }, construct.GreedyBytes, ), diff --git a/tests/formats/test_bctex.py b/tests/formats/test_bctex.py index 510cd55..992c460 100644 --- a/tests/formats/test_bctex.py +++ b/tests/formats/test_bctex.py @@ -1,9 +1,13 @@ from __future__ import annotations +import hashlib + import pytest from tests.test_lib import parse_build_compare_editor_parsed from mercury_engine_data_structures import dread_data, samus_returns_data +from mercury_engine_data_structures.exporters.dds import DdsExporter +from mercury_engine_data_structures.exporters.raw_texture import RawTexture from mercury_engine_data_structures.formats.bctex import Bctex dread_exclusions = [ @@ -1402,6 +1406,42 @@ "system/fx/textures/wind_arc.bctex", ] +# one of each XTX_Tegra_Format +dread_dds_exports = [ + ( + "textures/actors/characters/armadigger/models/textures/armadigger_at.bctex", + "9a82d53345292f5e5ca43eab5611580c5ec39ac70791df040c91fb108f02522a", + ), + ( + "textures/actors/characters/armadigger/models/textures/armadigger_nm.bctex", + "cbdc78010948e4a41fca4a75494eb12772ee61a83ea18e03d876ba9cfe2309e2", + ), + ( + "textures/actors/characters/autector/fx/textures/alarmring.bctex", + "965d5000d9c2eb97cfea9b149a60f5291704ec78e2f2f6971d7f77b1520dfb17", + ), + ( + "textures/actors/characters/chozocommanderx/models/textures/feathers_at.bctex", + "9b4b5d274eb522678f339dd82fe8a09c0d746bf63cacb15a211b724c74fdd471", + ), + ( + "textures/actors/characters/chozowarrior/fx/textures/groundshaft.bctex", + "dbb36935604b96dcec060a3f9ce005af2ac0a6ccd7abe985fd0b17da2ed08a91", + ), + ( + "textures/actors/characters/chozowarriorx/fx/textures/mudfluid04_nor.bctex", + "f408c524d58770f11de1ca3b1f0230db42523492cf5f696dec9da234f886da1c", + ), + ( + "textures/actors/characters/rinka/fx/textures/explosion01.bctex", + "0a2707955daded5c8d96075a07a2c69a2c568a82f0e4e6721b6d6a6df0b71576", + ), + ( + "textures/maps/cubemaps/airport_cubemapdiffusehdr.bctex", + "f0c8ae58d39389283fc967ba9a477c37634a21a186acae84ee9908ed90770bfa", + ), +] + @pytest.mark.parametrize( "bctex_path", dread_data.all_files_ending_with(".bctex", dread_210_ignore + dread_210_only + dread_exclusions) @@ -1418,3 +1458,14 @@ def test_compare_dread_210(dread_tree_210, bctex_path): @pytest.mark.parametrize("bctex_path", samus_returns_data.all_files_ending_with(".bctex", sr_missing)) def test_compare_bctex_sr(samus_returns_tree, bctex_path): parse_build_compare_editor_parsed(Bctex, samus_returns_tree, bctex_path) + + +@pytest.mark.parametrize(("bctex_path", "expected_hash"), dread_dds_exports) +def test_bctex_export_dread(dread_tree_100, bctex_path, expected_hash): + bctex = dread_tree_100.get_parsed_asset(bctex_path, type_hint=Bctex) + rawtex = RawTexture(bctex) + exporter = DdsExporter(rawtex) + + assert len(exporter.dds_files) == 1 + sha = hashlib.sha256(exporter.dds_files[0]) + assert sha.hexdigest() == expected_hash