From 6ad2668ac6dfd6c2e2b9871f2e87181e2e8f769f Mon Sep 17 00:00:00 2001 From: Steven Franklin Date: Thu, 3 Oct 2024 14:04:31 -0500 Subject: [PATCH 1/8] BMSSD refactor - now uses an adapter to place blocks, objects and lights in hidden keys and puts the objects in the scene group data - rebuilds all but 4 msr files with missing crc values - added function to get a block or light by name - added function to get a scene group - added function to find scene groups an item belongs to - added function to remove an item from a scene group - added function to add an item - added function to remove an item - beeg test function --- README.md | 2 +- .../formats/bmssd.py | 315 ++++++++++++------ tests/formats/test_bmssd.py | 104 +++++- 3 files changed, 308 insertions(+), 113 deletions(-) diff --git a/README.md b/README.md index 033632e..3a8dcab 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Construct type definitions for Mercury Engine | BMSNAV | ✓ | ✓ | ✓ | ✓ | Navigation Meshes | | BMSND | ✗ | ✗ | Missing | Missing | Sound (?) | | BMSSA | ✗ | ✗ | Missing | Missing | SSA (?) | -| BMSSD | ✗ | ✗ | ✓ | ✓ | Static Scenario Data (background dressing) | +| BMSSD | ✓ | ✓ | ✓ | ✓ | Scene Data (scene blocks, objects, msr lighting) | | BMSSH | Missing | Missing | ✓ | ✓ | GUI Shape | | BMSSK | Missing | Missing | ✓ | ✓ | GUI Skin | | BMSSS | Missing | Missing | ✓ | ✓ | GUI SpriteSheet | diff --git a/src/mercury_engine_data_structures/formats/bmssd.py b/src/mercury_engine_data_structures/formats/bmssd.py index 14712b9..5e10bf7 100644 --- a/src/mercury_engine_data_structures/formats/bmssd.py +++ b/src/mercury_engine_data_structures/formats/bmssd.py @@ -1,145 +1,246 @@ from __future__ import annotations +from collections import defaultdict +from enum import Enum import construct -from construct import ( +from construct.core import ( + Adapter, Byte, Const, Construct, Int8ul, Int32ul, Int64ul, + Rebuild, Struct, ) from mercury_engine_data_structures import game_check from mercury_engine_data_structures.base_resource import BaseResource -from mercury_engine_data_structures.common_types import CVector3D, StrId, VersionAdapter, make_vector -from mercury_engine_data_structures.construct_extensions.strings import StaticPaddedString +from mercury_engine_data_structures.common_types import CVector3D, StrId, VersionAdapter, make_dict, make_vector +from mercury_engine_data_structures.crc import crc32, crc64 +from mercury_engine_data_structures.base_resource import BaseResource from mercury_engine_data_structures.game_check import Game +TransformStruct = Struct("position" / CVector3D, "rotation" / CVector3D, "scale" / CVector3D) + BMSSD = Struct( - _magic=Const(b"MSSD"), - unk1=VersionAdapter(), - part_info=game_check.is_at_most( - Game.SAMUS_RETURNS, + "_magic" / Const(b"MSSD"), + "_version" + / game_check.is_sr_or_else( + VersionAdapter("1.12.0"), + VersionAdapter("1.19.0"), + ), + # static models (just bcmdl and bsmat), stored en mass in maps/levels/c10_samus//models/ + "scene_blocks" + / game_check.is_sr_or_else( make_vector( Struct( - model_name=StrId, - byte0=Byte, - byte1=Byte, - byte2=Byte, - int3=Int32ul, - int4=Int32ul, - farr4=CVector3D, - farr5=CVector3D, + "model_name" / StrId, + "byte0" / Byte, + "byte1" / Byte, + "byte2" / Byte, + "int3" / Int32ul, + "int4" / Int32ul, + "farr4" / CVector3D, + "farr5" / CVector3D, ) ), make_vector( Struct( - model_name=StrId, - byte0=Byte, - byte1=Byte, - byte2=Byte, - int3=Int32ul, - byte4=Byte, - farr4=CVector3D, - farr5=CVector3D, - farr6=CVector3D, + "model_name" / StrId, + "byte0" / Const(1, Byte), + "byte1" / Const(1, Byte), + "byte2" / Const(1, Byte), + "int3" / Const(1, Int32ul), + "byte4" / Const(1, Byte), + "transform" / TransformStruct, ) ), ), - model_info=make_vector( + # map objects (bcmdl, bsmat and bcskla), stored in the standard actor format in maps/objects/ + "objects" / make_vector(Struct("model_name" / StrId, "transforms" / make_vector(TransformStruct))), + Const(0, Int32ul), # likely unused array + "lights" + / make_vector( # only used in MSR Struct( - str1=StrId, - elems=make_vector( - Struct( - float1=CVector3D, - float2=CVector3D, - float3=CVector3D, - ) - ), + "model_name" / StrId, + "char2" / Byte, + "char3" / Byte, + "char4" / Byte, + "int5" / Int32ul, + "int6" / Int32ul, + "int7" / Int32ul, + "char8" / Byte, + "char9" / Byte, + "int10" / Int32ul, + "float13" / CVector3D, + "int11" / Int8ul, ) ), - strings_a=make_vector(StrId), - unk_structs_a=game_check.is_at_most( - Game.SAMUS_RETURNS, - make_vector( - Struct( - str1=StrId, - char2=Byte, - char3=Byte, - char4=Byte, - int5=Int32ul, - int6=Int32ul, - int7=Int32ul, - char8=Byte, - char9=Byte, - int10=Int32ul, - float13=CVector3D, - int11=Int8ul, - ) - ), - make_vector( - Struct( - str1=StrId, - char2=Byte, - char3=Byte, - char4=Byte, - int5=Int32ul, - int6=Int32ul, - int7=Int32ul, - char8=Byte, - char9=Byte, - int10=Int32ul, - str11=StaticPaddedString(16, "utf-8"), - int12=Int32ul, - float13=CVector3D, - float14=CVector3D, - float15=CVector3D, - int16=Int32ul, - float17=CVector3D, - ) - ), - ), - strings_b=make_vector( - StrId, + Const(0, Int32ul), # likely unused array + "scene_groups" + / make_vector( + Struct( + "sg_name" / StrId, + "item_count" / Rebuild(Int32ul, lambda ctx: sum([len(g) for g in ctx.item_groups.values()])), + "item_groups" / make_dict(make_vector(game_check.is_sr_or_else(Int32ul, Int64ul)), Int32ul), + ) ), - scene_groups=game_check.is_at_most( - Game.SAMUS_RETURNS, - make_vector( - Struct( - sg_name=StrId, - models_per_sg=Int32ul, - model_groups=make_vector( - Struct( - model_group=Int32ul, - models=make_vector( - Struct( - model_id=Int32ul, - ) - ), + construct.Terminated, +) + + +def crc_func(obj): + return crc32 if obj._version == "1.12.0" else crc64 + + +class BmssdAdapter(Adapter): + ItemTypes = { + 0: "scene_blocks", + 1: "objects", + 2: "lights", + } + + def _decode(self, obj, context, path): + crc = crc_func(obj) + + res = construct.Container( + _version=obj._version, + _scene_blocks={crc(blk.model_name): blk for blk in obj.scene_blocks}, + _objects=[ + construct.Container(model_name=o.model_name, transform=t) for o in obj.objects for t in o.transforms + ], + _lights={crc(lgt.model_name): lgt for lgt in obj.lights}, + scene_groups=construct.Container(), + ) + + for sg in obj.scene_groups: + res.scene_groups[sg.sg_name] = construct.Container() + + for ig_value, items in sg.item_groups.items(): + group_type = self.ItemTypes[ig_value] + res.scene_groups[sg.sg_name][group_type] = construct.ListContainer() + + # objects are indexed and not hashed + if ig_value == 1: + res.scene_groups[sg.sg_name][group_type] = construct.ListContainer( + [res._objects[block] for block in items] ) - ), - ) - ), - make_vector( - Struct( - str1=StrId, - int2=Int32ul, - struct4=make_vector( - Struct( - int1=Int32ul, - long3=make_vector(Int64ul), + else: + res.scene_groups[sg.sg_name][group_type] = construct.ListContainer( + [res[f"_{group_type}"][block] for block in items] ) - ), + + return res + + def _encode(self, obj, context, path): + def obj_to_tuple(o): + return ( + o["model_name"], + o["transform"]["position"][0], + o["transform"]["position"][1], + o["transform"]["position"][2], + o["transform"]["rotation"][0], + o["transform"]["rotation"][1], + o["transform"]["rotation"][2], + o["transform"]["scale"][0], + o["transform"]["scale"][1], + o["transform"]["scale"][2], ) - ), - ), - rest=construct.GreedyBytes, -).compile() + + objects = defaultdict(list) + for o in obj._objects: + objects[o["model_name"]].append(o) + + object_order = dict() + object_containers = construct.ListContainer() + i = 0 + for name, objs in objects.items(): + object_containers.append( + construct.Container(model_name=name, transforms=construct.ListContainer(o["transform"] for o in objs)) + ) + for o in objs: + object_order[obj_to_tuple(o)] = i + i += 1 + + crc = crc_func(obj) + + res = construct.Container( + _version=obj._version, + scene_blocks=[blk for blk in obj._scene_blocks.values()], + objects=object_containers, + lights=[lgt for lgt in obj._lights.values()], + scene_groups=construct.ListContainer(), + ) + + for sg_name, sg in obj.scene_groups.items(): + sg_cont = construct.Container(sg_name=sg_name, item_groups=construct.Container()) + + for group_type, items in sg.items(): + group_type_int = [it for it in self.ItemTypes if self.ItemTypes[it] == group_type][0] + + if group_type_int == 1: + sg_cont.item_groups[group_type_int] = [object_order[obj_to_tuple(o)] for o in items] + else: + sg_cont.item_groups[group_type_int] = [crc(o["model_name"]) for o in items] + + res.scene_groups.append(sg_cont) + + return res + + +class ItemType(Enum): + SCENE_BLOCK = 0, "scene_blocks" + OBJECT = 1, "objects" + LIGHT = 2, "lights" + + def __new__(cls, value: int, group_name: str): + member = object.__new__(cls) + member._value_ = value + member.group_name = group_name + return member class Bmssd(BaseResource): @classmethod def construct_class(cls, target_game: Game) -> Construct: - return BMSSD + return BmssdAdapter(BMSSD) + + def get_item_by_name(self, item_name: str, item_type: ItemType) -> construct.Container: + if item_type == ItemType.OBJECT: + raise ValueError("Cannot get objects by name!") + + crc = crc_func(self.raw) + return self.raw[f"_{item_type.group_name}"].get(crc(item_name), None) + + def get_scene_group(self, scene_group: str) -> construct.Container: + return self.raw.scene_groups.get(scene_group, None) + + def scene_groups_for_item(self, item: str | construct.Container, item_type: str) -> list[str]: + if isinstance(item, str): + item = self.get_item_by_name(item, item_type) + + return [sg_name for sg_name, sg_val in self.raw.scene_groups.items() if item in sg_val[item_type.group_name]] + + def add_item(self, item: construct.Container, item_type: ItemType, scene_groups: list[str] = None): + if item_type == ItemType.OBJECT: + self.raw._objects.append(item) + else: + crc = crc_func(self.raw) + self.raw[f"_{item_type.group_name}"][crc(item["model_name"])] = item + + for sg_name in scene_groups: + self.get_scene_group(sg_name)[item_type.group_name].append(item) + + def remove_item_from_group(self, item: construct.Container, item_type: ItemType, scene_group: str): + sg = self.get_scene_group(scene_group) + if sg and item_type.group_name in sg and item in sg[item_type.group_name]: + sg[item_type.group_name].remove(item) + + def remove_item(self, item: construct.Container, item_type: ItemType): + groups = self.scene_groups_for_item(item, item_type) + for sg in groups: + self.remove_item_from_group(item, item_type, sg) + + self.raw[f"_{item_type.group_name}"].remove(item) diff --git a/tests/formats/test_bmssd.py b/tests/formats/test_bmssd.py index ef5243d..d48b290 100644 --- a/tests/formats/test_bmssd.py +++ b/tests/formats/test_bmssd.py @@ -1,10 +1,12 @@ from __future__ import annotations +import contextlib import pytest -from tests.test_lib import parse_build_compare_editor +from tests.test_lib import parse_build_compare_editor_parsed from mercury_engine_data_structures import dread_data, samus_returns_data -from mercury_engine_data_structures.formats.bmssd import Bmssd +from mercury_engine_data_structures.formats.bmssd import Bmssd, ItemType +from mercury_engine_data_structures.game_check import Game bossrush_assets = [ "maps/levels/c10_samus/s201_bossrush_scorpius/s201_bossrush_scorpius.bmssd", @@ -37,17 +39,109 @@ "maps/levels/c10_samus/s920_traininggallery/s920_traininggallery.bmssd", ] +sr_xfail = [ + "maps/levels/c10_samus/s000_surface/s000_surface.bmssd", + "maps/levels/c10_samus/s020_area2/s020_area2.bmssd", + "maps/levels/c10_samus/s050_area5/s050_area5.bmssd", + "maps/levels/c10_samus/s110_surfaceb/s110_surfaceb.bmssd", +] + @pytest.mark.parametrize("bmssd_path", dread_data.all_files_ending_with(".bmssd", bossrush_assets)) def test_compare_bmssd_dread_100(dread_tree_100, bmssd_path): - parse_build_compare_editor(Bmssd, dread_tree_100, bmssd_path) + parse_build_compare_editor_parsed(Bmssd, dread_tree_100, bmssd_path) @pytest.mark.parametrize("bmssd_path", bossrush_assets) def test_compare_dread_210(dread_tree_210, bmssd_path): - parse_build_compare_editor(Bmssd, dread_tree_210, bmssd_path) + parse_build_compare_editor_parsed(Bmssd, dread_tree_210, bmssd_path) @pytest.mark.parametrize("bmssd_path", samus_returns_data.all_files_ending_with(".bmssd", sr_missing)) def test_compare_bmssd_msr(samus_returns_tree, bmssd_path): - parse_build_compare_editor(Bmssd, samus_returns_tree, bmssd_path) + if bmssd_path in sr_xfail: + expectation = pytest.raises(KeyError) + else: + expectation = contextlib.nullcontext() + with expectation: + parse_build_compare_editor_parsed(Bmssd, samus_returns_tree, bmssd_path) + + +def test_bmssd_dread_functions(dread_tree_100): + bmssd = dread_tree_100.get_parsed_asset("maps/levels/c10_samus/s090_skybase/s090_skybase.bmssd", type_hint=Bmssd) + + # PART 1: Ensure getting an item and accessing scene group for item works + + lshaft03 = bmssd.get_item_by_name("part001_jp6_lshaft03", ItemType.SCENE_BLOCK) + + # check get_item_by_name returned the correct data + assert lshaft03.transform.position[0] == 640.7750244140625 + # check the scene groups are correct + assert bmssd.scene_groups_for_item(lshaft03, ItemType.SCENE_BLOCK) == [ + f"sg_SubArea_collision_camera_00{x}" for x in [3, 2, 1] + ] + # check using the name and object to find scene groups works + assert bmssd.scene_groups_for_item("part001_jp6_lshaft03", ItemType.SCENE_BLOCK) == bmssd.scene_groups_for_item( + lshaft03, ItemType.SCENE_BLOCK + ) + + # check objects work right + cc3 = bmssd.get_scene_group("sg_SubArea_collision_camera_003") + map_object = cc3.objects[0] + assert map_object.model_name == "chozoskypathroofx1" + assert bmssd.scene_groups_for_item(map_object, ItemType.OBJECT) == [ + f"sg_SubArea_collision_camera_00{x}" for x in [3, 2] + ] + + # PART 2: Add new items to sg (created as new dicts) + + # scene_blocks + new_sb = { + "model_name": "part420_rdv_newblock", + "byte0": 1, + "byte1": 1, + "byte2": 1, + "int3": 1, + "byte4": 1, + "transform": {"position": [100.0, 200.0, 300.0], "rotation": [0.0, 0.0, 0.0], "scale": [1.0, 1.0, 1.0]}, + } + new_sb_groups = [f"sg_casca100{x}" for x in [0, 1, 2]] + bmssd.add_item(new_sb, ItemType.SCENE_BLOCK, new_sb_groups) + assert bmssd.get_item_by_name("part420_rdv_newblock", ItemType.SCENE_BLOCK) == new_sb + assert bmssd.scene_groups_for_item("part420_rdv_newblock", ItemType.SCENE_BLOCK) == new_sb_groups + + # objects + new_obj = { + "model_name": "theoreticalplandoobj", + "transform": {"position": [1000.0, 800.0, 202100.0], "rotation": [0.0, 0.0, 0.0], "scale": [0.0, 0.0, 0.0]}, + } + new_obj_groups = ["sg_SubArea_collision_camera_005"] + bmssd.add_item(new_obj, ItemType.OBJECT, new_obj_groups) + assert bmssd.scene_groups_for_item(new_obj, ItemType.OBJECT) == new_obj_groups + assert new_obj in bmssd.get_scene_group(new_obj_groups[0])["objects"] + + # PART 3: break things and ensure it acts right :) + + # can't get object by name + with pytest.raises(ValueError): + bmssd.get_item_by_name("theoreticalplandoobj", ItemType.OBJECT) + + # non-existant object + assert bmssd.get_item_by_name("isweariaddedthis", ItemType.SCENE_BLOCK) is None + + # actually we changed our mind on where the newblock goes + bmssd.remove_item_from_group(new_sb, ItemType.SCENE_BLOCK, "sg_casca1002") + assert "sg_casca1002" not in bmssd.scene_groups_for_item("part420_rdv_newblock", ItemType.SCENE_BLOCK) + + # lets get rid of a part of the roof and see what happens + roof = bmssd.get_scene_group("sg_SubArea_collision_camera_003").objects[0] + bmssd.remove_item(roof, ItemType.OBJECT) + assert bmssd.scene_groups_for_item(roof, ItemType.OBJECT) == [] + assert roof not in bmssd.get_scene_group("sg_SubArea_collision_camera_003").objects + assert roof not in bmssd.get_scene_group("sg_SubArea_collision_camera_002").objects + + # PART 4: make sure it can actually build and parse lol + con = Bmssd.construct_class(target_game=Game.DREAD) + built = con.build(bmssd.raw, target_game=Game.DREAD) + reparsed = con.parse(built, target_game=Game.DREAD) + assert reparsed == bmssd.raw From 45e635acec1618d28e7934b7712c16c6ab0a0595 Mon Sep 17 00:00:00 2001 From: Steven Franklin Date: Fri, 4 Oct 2024 14:35:55 -0500 Subject: [PATCH 2/8] BMSSD - refactored get_item to allow getting any item by crc (or index for objects) - updated test --- .../formats/bmssd.py | 26 +++++++++++++---- tests/formats/test_bmssd.py | 29 +++++++------------ 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/mercury_engine_data_structures/formats/bmssd.py b/src/mercury_engine_data_structures/formats/bmssd.py index 5e10bf7..137fcd3 100644 --- a/src/mercury_engine_data_structures/formats/bmssd.py +++ b/src/mercury_engine_data_structures/formats/bmssd.py @@ -129,7 +129,11 @@ def _decode(self, obj, context, path): ) else: res.scene_groups[sg.sg_name][group_type] = construct.ListContainer( - [res[f"_{group_type}"][block] for block in items] + [ + # use raw hash value instead of block value if it doesn't exist above + res[f"_{group_type}"][block] if res[f"_{group_type}"].get(block, None) else block + for block in items + ] ) return res @@ -183,7 +187,11 @@ def obj_to_tuple(o): if group_type_int == 1: sg_cont.item_groups[group_type_int] = [object_order[obj_to_tuple(o)] for o in items] else: - sg_cont.item_groups[group_type_int] = [crc(o["model_name"]) for o in items] + sg_cont.item_groups[group_type_int] = [ + # handle integers (unmatched crc's in decode) + o if isinstance(o, int) else crc(o["model_name"]) + for o in items + ] res.scene_groups.append(sg_cont) @@ -207,19 +215,25 @@ class Bmssd(BaseResource): def construct_class(cls, target_game: Game) -> Construct: return BmssdAdapter(BMSSD) - def get_item_by_name(self, item_name: str, item_type: ItemType) -> construct.Container: + def get_item(self, item_name_or_id: str | int, item_type: ItemType) -> construct.Container: + if isinstance(item_name_or_id, int): + if item_type == ItemType.OBJECT: + return self.raw._objects[item_name_or_id] + else: + return self.raw[f"_{item_type.group_name}"].get(item_name_or_id, None) + if item_type == ItemType.OBJECT: - raise ValueError("Cannot get objects by name!") + raise ValueError("If accessing an Object type item, must use the index!") crc = crc_func(self.raw) - return self.raw[f"_{item_type.group_name}"].get(crc(item_name), None) + return self.raw[f"_{item_type.group_name}"].get(crc(item_name_or_id), None) def get_scene_group(self, scene_group: str) -> construct.Container: return self.raw.scene_groups.get(scene_group, None) def scene_groups_for_item(self, item: str | construct.Container, item_type: str) -> list[str]: if isinstance(item, str): - item = self.get_item_by_name(item, item_type) + item = self.get_item(item, item_type) return [sg_name for sg_name, sg_val in self.raw.scene_groups.items() if item in sg_val[item_type.group_name]] diff --git a/tests/formats/test_bmssd.py b/tests/formats/test_bmssd.py index d48b290..471a727 100644 --- a/tests/formats/test_bmssd.py +++ b/tests/formats/test_bmssd.py @@ -1,5 +1,4 @@ from __future__ import annotations -import contextlib import pytest from tests.test_lib import parse_build_compare_editor_parsed @@ -39,13 +38,6 @@ "maps/levels/c10_samus/s920_traininggallery/s920_traininggallery.bmssd", ] -sr_xfail = [ - "maps/levels/c10_samus/s000_surface/s000_surface.bmssd", - "maps/levels/c10_samus/s020_area2/s020_area2.bmssd", - "maps/levels/c10_samus/s050_area5/s050_area5.bmssd", - "maps/levels/c10_samus/s110_surfaceb/s110_surfaceb.bmssd", -] - @pytest.mark.parametrize("bmssd_path", dread_data.all_files_ending_with(".bmssd", bossrush_assets)) def test_compare_bmssd_dread_100(dread_tree_100, bmssd_path): @@ -59,12 +51,7 @@ def test_compare_dread_210(dread_tree_210, bmssd_path): @pytest.mark.parametrize("bmssd_path", samus_returns_data.all_files_ending_with(".bmssd", sr_missing)) def test_compare_bmssd_msr(samus_returns_tree, bmssd_path): - if bmssd_path in sr_xfail: - expectation = pytest.raises(KeyError) - else: - expectation = contextlib.nullcontext() - with expectation: - parse_build_compare_editor_parsed(Bmssd, samus_returns_tree, bmssd_path) + parse_build_compare_editor_parsed(Bmssd, samus_returns_tree, bmssd_path) def test_bmssd_dread_functions(dread_tree_100): @@ -72,9 +59,9 @@ def test_bmssd_dread_functions(dread_tree_100): # PART 1: Ensure getting an item and accessing scene group for item works - lshaft03 = bmssd.get_item_by_name("part001_jp6_lshaft03", ItemType.SCENE_BLOCK) + lshaft03 = bmssd.get_item("part001_jp6_lshaft03", ItemType.SCENE_BLOCK) - # check get_item_by_name returned the correct data + # check get_item returned the correct data assert lshaft03.transform.position[0] == 640.7750244140625 # check the scene groups are correct assert bmssd.scene_groups_for_item(lshaft03, ItemType.SCENE_BLOCK) == [ @@ -107,7 +94,7 @@ def test_bmssd_dread_functions(dread_tree_100): } new_sb_groups = [f"sg_casca100{x}" for x in [0, 1, 2]] bmssd.add_item(new_sb, ItemType.SCENE_BLOCK, new_sb_groups) - assert bmssd.get_item_by_name("part420_rdv_newblock", ItemType.SCENE_BLOCK) == new_sb + assert bmssd.get_item("part420_rdv_newblock", ItemType.SCENE_BLOCK) == new_sb assert bmssd.scene_groups_for_item("part420_rdv_newblock", ItemType.SCENE_BLOCK) == new_sb_groups # objects @@ -124,10 +111,14 @@ def test_bmssd_dread_functions(dread_tree_100): # can't get object by name with pytest.raises(ValueError): - bmssd.get_item_by_name("theoreticalplandoobj", ItemType.OBJECT) + bmssd.get_item("theoreticalplandoobj", ItemType.OBJECT) + + # can get object by index + obj = bmssd.get_item(69, ItemType.OBJECT) + assert obj.model_name == "chozoskypathroofx1" and obj.transform.position[1] == -2650.0 # non-existant object - assert bmssd.get_item_by_name("isweariaddedthis", ItemType.SCENE_BLOCK) is None + assert bmssd.get_item("isweariaddedthis", ItemType.SCENE_BLOCK) is None # actually we changed our mind on where the newblock goes bmssd.remove_item_from_group(new_sb, ItemType.SCENE_BLOCK, "sg_casca1002") From 677edce47bbe63fbe352c1763a820fe761cdea03 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 17 Oct 2024 21:02:43 +0000 Subject: [PATCH 3/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/mercury_engine_data_structures/formats/bmssd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mercury_engine_data_structures/formats/bmssd.py b/src/mercury_engine_data_structures/formats/bmssd.py index 137fcd3..c221051 100644 --- a/src/mercury_engine_data_structures/formats/bmssd.py +++ b/src/mercury_engine_data_structures/formats/bmssd.py @@ -1,4 +1,5 @@ from __future__ import annotations + from collections import defaultdict from enum import Enum @@ -19,7 +20,6 @@ from mercury_engine_data_structures.base_resource import BaseResource from mercury_engine_data_structures.common_types import CVector3D, StrId, VersionAdapter, make_dict, make_vector from mercury_engine_data_structures.crc import crc32, crc64 -from mercury_engine_data_structures.base_resource import BaseResource from mercury_engine_data_structures.game_check import Game TransformStruct = Struct("position" / CVector3D, "rotation" / CVector3D, "scale" / CVector3D) From 4119b784090e4999abb0fea587b33edaf82e2113 Mon Sep 17 00:00:00 2001 From: Steven Franklin Date: Thu, 17 Oct 2024 16:10:53 -0500 Subject: [PATCH 4/8] adjusted tests --- tests/formats/test_bmssd.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/tests/formats/test_bmssd.py b/tests/formats/test_bmssd.py index 471a727..9f7ece5 100644 --- a/tests/formats/test_bmssd.py +++ b/tests/formats/test_bmssd.py @@ -22,22 +22,6 @@ "maps/levels/c10_samus/s212_bossrush_commander/s212_bossrush_commander.bmssd", ] -sr_missing = [ - "maps/levels/c10_samus/s901_alpha/s901_alpha.bmssd", - "maps/levels/c10_samus/s902_gamma/s902_gamma.bmssd", - "maps/levels/c10_samus/s903_zeta/s903_zeta.bmssd", - "maps/levels/c10_samus/s904_omega/s904_omega.bmssd", - "maps/levels/c10_samus/s905_arachnus/s905_arachnus.bmssd", - "maps/levels/c10_samus/s905_queen/s905_queen.bmssd", - "maps/levels/c10_samus/s906_metroid/s906_metroid.bmssd", - "maps/levels/c10_samus/s907_manicminerbot/s907_manicminerbot.bmssd", - "maps/levels/c10_samus/s908_manicminerbotrun/s908_manicminerbotrun.bmssd", - "maps/levels/c10_samus/s909_ridley/s909_ridley.bmssd", - "maps/levels/c10_samus/s910_gym/s910_gym.bmssd", - "maps/levels/c10_samus/s911_swarmgym/s911_swarmgym.bmssd", - "maps/levels/c10_samus/s920_traininggallery/s920_traininggallery.bmssd", -] - @pytest.mark.parametrize("bmssd_path", dread_data.all_files_ending_with(".bmssd", bossrush_assets)) def test_compare_bmssd_dread_100(dread_tree_100, bmssd_path): @@ -49,7 +33,7 @@ def test_compare_dread_210(dread_tree_210, bmssd_path): parse_build_compare_editor_parsed(Bmssd, dread_tree_210, bmssd_path) -@pytest.mark.parametrize("bmssd_path", samus_returns_data.all_files_ending_with(".bmssd", sr_missing)) +@pytest.mark.parametrize("bmssd_path", samus_returns_data.all_files_ending_with(".bmssd")) def test_compare_bmssd_msr(samus_returns_tree, bmssd_path): parse_build_compare_editor_parsed(Bmssd, samus_returns_tree, bmssd_path) From 97121ce6a4ee1ab8b3f47cbee45a0052a42a057e Mon Sep 17 00:00:00 2001 From: Steven Franklin Date: Sun, 20 Oct 2024 13:36:50 -0500 Subject: [PATCH 5/8] fixed ruff --- src/mercury_engine_data_structures/formats/bmssd.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/mercury_engine_data_structures/formats/bmssd.py b/src/mercury_engine_data_structures/formats/bmssd.py index c221051..698ff97 100644 --- a/src/mercury_engine_data_structures/formats/bmssd.py +++ b/src/mercury_engine_data_structures/formats/bmssd.py @@ -20,7 +20,6 @@ from mercury_engine_data_structures.base_resource import BaseResource from mercury_engine_data_structures.common_types import CVector3D, StrId, VersionAdapter, make_dict, make_vector from mercury_engine_data_structures.crc import crc32, crc64 -from mercury_engine_data_structures.game_check import Game TransformStruct = Struct("position" / CVector3D, "rotation" / CVector3D, "scale" / CVector3D) @@ -157,7 +156,7 @@ def obj_to_tuple(o): for o in obj._objects: objects[o["model_name"]].append(o) - object_order = dict() + object_order = {} object_containers = construct.ListContainer() i = 0 for name, objs in objects.items(): @@ -172,9 +171,9 @@ def obj_to_tuple(o): res = construct.Container( _version=obj._version, - scene_blocks=[blk for blk in obj._scene_blocks.values()], + scene_blocks=obj._scene_blocks.values(), objects=object_containers, - lights=[lgt for lgt in obj._lights.values()], + lights=obj._lights.values(), scene_groups=construct.ListContainer(), ) @@ -212,7 +211,7 @@ def __new__(cls, value: int, group_name: str): class Bmssd(BaseResource): @classmethod - def construct_class(cls, target_game: Game) -> Construct: + def construct_class(cls, target_game: game_check.Game) -> Construct: return BmssdAdapter(BMSSD) def get_item(self, item_name_or_id: str | int, item_type: ItemType) -> construct.Container: From b52b1b61d2cd2a7116b0fe9ab05c9d9d8f93b292 Mon Sep 17 00:00:00 2001 From: Steven Franklin Date: Sun, 20 Oct 2024 13:52:35 -0500 Subject: [PATCH 6/8] make bmssd compilable - moved scene_groups.item_count calculation into BmssdAdapter.encode - removed lambda from scene_groups.item_count and compiled BMSSD - bmssd test duration -- before: 27.34s, after: 21.36s --- src/mercury_engine_data_structures/formats/bmssd.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/mercury_engine_data_structures/formats/bmssd.py b/src/mercury_engine_data_structures/formats/bmssd.py index 698ff97..f067cf3 100644 --- a/src/mercury_engine_data_structures/formats/bmssd.py +++ b/src/mercury_engine_data_structures/formats/bmssd.py @@ -12,7 +12,6 @@ Int8ul, Int32ul, Int64ul, - Rebuild, Struct, ) @@ -82,12 +81,12 @@ / make_vector( Struct( "sg_name" / StrId, - "item_count" / Rebuild(Int32ul, lambda ctx: sum([len(g) for g in ctx.item_groups.values()])), + "item_count" / Int32ul, "item_groups" / make_dict(make_vector(game_check.is_sr_or_else(Int32ul, Int64ul)), Int32ul), ) ), construct.Terminated, -) +).compile() def crc_func(obj): @@ -179,9 +178,11 @@ def obj_to_tuple(o): for sg_name, sg in obj.scene_groups.items(): sg_cont = construct.Container(sg_name=sg_name, item_groups=construct.Container()) + item_count = 0 for group_type, items in sg.items(): group_type_int = [it for it in self.ItemTypes if self.ItemTypes[it] == group_type][0] + item_count += len(items) if group_type_int == 1: sg_cont.item_groups[group_type_int] = [object_order[obj_to_tuple(o)] for o in items] @@ -192,6 +193,7 @@ def obj_to_tuple(o): for o in items ] + sg_cont["item_count"] = item_count res.scene_groups.append(sg_cont) return res From 2d9c803ca154d2e6cb3138676d8c6c045823e5a8 Mon Sep 17 00:00:00 2001 From: Steven Franklin Date: Wed, 30 Oct 2024 14:12:40 -0500 Subject: [PATCH 7/8] dunc review - moved TransformStruct to common types as Transform3D - used ItemType enum more consistently - fixed tests to use new ItemType format --- .../common_types.py | 1 + .../formats/bmssd.py | 70 ++++++++++--------- tests/formats/test_bmssd.py | 10 +-- 3 files changed, 42 insertions(+), 39 deletions(-) diff --git a/src/mercury_engine_data_structures/common_types.py b/src/mercury_engine_data_structures/common_types.py index 8658379..e2dcb84 100644 --- a/src/mercury_engine_data_structures/common_types.py +++ b/src/mercury_engine_data_structures/common_types.py @@ -21,6 +21,7 @@ CVector2D = construct.Array(2, Float) CVector3D = construct.Array(3, Float) CVector4D = construct.Array(4, Float) +Transform3D = construct.Struct("position" / CVector3D, "rotation" / CVector3D, "scale" / CVector3D) class VersionAdapter(Adapter): diff --git a/src/mercury_engine_data_structures/formats/bmssd.py b/src/mercury_engine_data_structures/formats/bmssd.py index f067cf3..5608af9 100644 --- a/src/mercury_engine_data_structures/formats/bmssd.py +++ b/src/mercury_engine_data_structures/formats/bmssd.py @@ -17,11 +17,16 @@ from mercury_engine_data_structures import game_check from mercury_engine_data_structures.base_resource import BaseResource -from mercury_engine_data_structures.common_types import CVector3D, StrId, VersionAdapter, make_dict, make_vector +from mercury_engine_data_structures.common_types import ( + CVector3D, + StrId, + Transform3D, + VersionAdapter, + make_dict, + make_vector, +) from mercury_engine_data_structures.crc import crc32, crc64 -TransformStruct = Struct("position" / CVector3D, "rotation" / CVector3D, "scale" / CVector3D) - BMSSD = Struct( "_magic" / Const(b"MSSD"), "_version" @@ -52,12 +57,12 @@ "byte2" / Const(1, Byte), "int3" / Const(1, Int32ul), "byte4" / Const(1, Byte), - "transform" / TransformStruct, + "transform" / Transform3D, ) ), ), # map objects (bcmdl, bsmat and bcskla), stored in the standard actor format in maps/objects/ - "objects" / make_vector(Struct("model_name" / StrId, "transforms" / make_vector(TransformStruct))), + "objects" / make_vector(Struct("model_name" / StrId, "transforms" / make_vector(Transform3D))), Const(0, Int32ul), # likely unused array "lights" / make_vector( # only used in MSR @@ -93,13 +98,21 @@ def crc_func(obj): return crc32 if obj._version == "1.12.0" else crc64 -class BmssdAdapter(Adapter): - ItemTypes = { - 0: "scene_blocks", - 1: "objects", - 2: "lights", - } +class ItemType(Enum): + group_name: str + + SCENE_BLOCK = 0, "scene_blocks" + OBJECT = 1, "objects" + LIGHT = 2, "lights" + + def __new__(cls, value: int, group_name: str): + member = object.__new__(cls) + member._value_ = value + member.group_name = group_name + return member + +class BmssdAdapter(Adapter): def _decode(self, obj, context, path): crc = crc_func(obj) @@ -117,7 +130,7 @@ def _decode(self, obj, context, path): res.scene_groups[sg.sg_name] = construct.Container() for ig_value, items in sg.item_groups.items(): - group_type = self.ItemTypes[ig_value] + group_type = ItemType(ig_value) res.scene_groups[sg.sg_name][group_type] = construct.ListContainer() # objects are indexed and not hashed @@ -129,7 +142,9 @@ def _decode(self, obj, context, path): res.scene_groups[sg.sg_name][group_type] = construct.ListContainer( [ # use raw hash value instead of block value if it doesn't exist above - res[f"_{group_type}"][block] if res[f"_{group_type}"].get(block, None) else block + res[f"_{group_type.group_name}"][block] + if res[f"_{group_type.group_name}"].get(block, None) + else block for block in items ] ) @@ -181,13 +196,12 @@ def obj_to_tuple(o): item_count = 0 for group_type, items in sg.items(): - group_type_int = [it for it in self.ItemTypes if self.ItemTypes[it] == group_type][0] item_count += len(items) - if group_type_int == 1: - sg_cont.item_groups[group_type_int] = [object_order[obj_to_tuple(o)] for o in items] + if group_type == ItemType.OBJECT: + sg_cont.item_groups[group_type.value] = [object_order[obj_to_tuple(o)] for o in items] else: - sg_cont.item_groups[group_type_int] = [ + sg_cont.item_groups[group_type.value] = [ # handle integers (unmatched crc's in decode) o if isinstance(o, int) else crc(o["model_name"]) for o in items @@ -199,18 +213,6 @@ def obj_to_tuple(o): return res -class ItemType(Enum): - SCENE_BLOCK = 0, "scene_blocks" - OBJECT = 1, "objects" - LIGHT = 2, "lights" - - def __new__(cls, value: int, group_name: str): - member = object.__new__(cls) - member._value_ = value - member.group_name = group_name - return member - - class Bmssd(BaseResource): @classmethod def construct_class(cls, target_game: game_check.Game) -> Construct: @@ -232,11 +234,11 @@ def get_item(self, item_name_or_id: str | int, item_type: ItemType) -> construct def get_scene_group(self, scene_group: str) -> construct.Container: return self.raw.scene_groups.get(scene_group, None) - def scene_groups_for_item(self, item: str | construct.Container, item_type: str) -> list[str]: + def scene_groups_for_item(self, item: str | construct.Container, item_type: ItemType) -> list[str]: if isinstance(item, str): item = self.get_item(item, item_type) - return [sg_name for sg_name, sg_val in self.raw.scene_groups.items() if item in sg_val[item_type.group_name]] + return [sg_name for sg_name, sg_val in self.raw.scene_groups.items() if item in sg_val[item_type]] def add_item(self, item: construct.Container, item_type: ItemType, scene_groups: list[str] = None): if item_type == ItemType.OBJECT: @@ -246,12 +248,12 @@ def add_item(self, item: construct.Container, item_type: ItemType, scene_groups: self.raw[f"_{item_type.group_name}"][crc(item["model_name"])] = item for sg_name in scene_groups: - self.get_scene_group(sg_name)[item_type.group_name].append(item) + self.get_scene_group(sg_name)[item_type].append(item) def remove_item_from_group(self, item: construct.Container, item_type: ItemType, scene_group: str): sg = self.get_scene_group(scene_group) - if sg and item_type.group_name in sg and item in sg[item_type.group_name]: - sg[item_type.group_name].remove(item) + if sg and item_type in sg and item in sg[item_type]: + sg[item_type].remove(item) def remove_item(self, item: construct.Container, item_type: ItemType): groups = self.scene_groups_for_item(item, item_type) diff --git a/tests/formats/test_bmssd.py b/tests/formats/test_bmssd.py index 9f7ece5..360c527 100644 --- a/tests/formats/test_bmssd.py +++ b/tests/formats/test_bmssd.py @@ -58,7 +58,7 @@ def test_bmssd_dread_functions(dread_tree_100): # check objects work right cc3 = bmssd.get_scene_group("sg_SubArea_collision_camera_003") - map_object = cc3.objects[0] + map_object = cc3[ItemType.OBJECT][0] assert map_object.model_name == "chozoskypathroofx1" assert bmssd.scene_groups_for_item(map_object, ItemType.OBJECT) == [ f"sg_SubArea_collision_camera_00{x}" for x in [3, 2] @@ -89,7 +89,7 @@ def test_bmssd_dread_functions(dread_tree_100): new_obj_groups = ["sg_SubArea_collision_camera_005"] bmssd.add_item(new_obj, ItemType.OBJECT, new_obj_groups) assert bmssd.scene_groups_for_item(new_obj, ItemType.OBJECT) == new_obj_groups - assert new_obj in bmssd.get_scene_group(new_obj_groups[0])["objects"] + assert new_obj in bmssd.get_scene_group(new_obj_groups[0])[ItemType.OBJECT] # PART 3: break things and ensure it acts right :) @@ -109,11 +109,11 @@ def test_bmssd_dread_functions(dread_tree_100): assert "sg_casca1002" not in bmssd.scene_groups_for_item("part420_rdv_newblock", ItemType.SCENE_BLOCK) # lets get rid of a part of the roof and see what happens - roof = bmssd.get_scene_group("sg_SubArea_collision_camera_003").objects[0] + roof = bmssd.get_scene_group("sg_SubArea_collision_camera_003")[ItemType.OBJECT][0] bmssd.remove_item(roof, ItemType.OBJECT) assert bmssd.scene_groups_for_item(roof, ItemType.OBJECT) == [] - assert roof not in bmssd.get_scene_group("sg_SubArea_collision_camera_003").objects - assert roof not in bmssd.get_scene_group("sg_SubArea_collision_camera_002").objects + assert roof not in bmssd.get_scene_group("sg_SubArea_collision_camera_003")[ItemType.OBJECT] + assert roof not in bmssd.get_scene_group("sg_SubArea_collision_camera_002")[ItemType.OBJECT] # PART 4: make sure it can actually build and parse lol con = Bmssd.construct_class(target_game=Game.DREAD) From 31cfd22b8c0f6c6ae5d55a6311961073f037451e Mon Sep 17 00:00:00 2001 From: Steven Franklin Date: Sat, 9 Nov 2024 20:59:13 -0600 Subject: [PATCH 8/8] uses str enum - todo: just use strenum after we stop supporting 3.10 --- .../formats/bmssd.py | 47 +++++++++++-------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/src/mercury_engine_data_structures/formats/bmssd.py b/src/mercury_engine_data_structures/formats/bmssd.py index 5608af9..e171637 100644 --- a/src/mercury_engine_data_structures/formats/bmssd.py +++ b/src/mercury_engine_data_structures/formats/bmssd.py @@ -98,19 +98,29 @@ def crc_func(obj): return crc32 if obj._version == "1.12.0" else crc64 -class ItemType(Enum): - group_name: str +class ItemType(str, Enum): + group_index: int - SCENE_BLOCK = 0, "scene_blocks" - OBJECT = 1, "objects" - LIGHT = 2, "lights" + SCENE_BLOCK = "scene_blocks", 0 + OBJECT = "objects", 1 + LIGHT = "lights", 2 - def __new__(cls, value: int, group_name: str): - member = object.__new__(cls) - member._value_ = value - member.group_name = group_name + def __new__(cls, name: int, index: str): + member = str.__new__(cls, name) + member._value_ = name + member.group_index = index return member + @classmethod + def from_index(cls, val: int): + for it in cls: + if it.group_index == val: + return it + return ValueError(f"Value {val} is not a valid index of ItemType") + + def __format__(self, format_spec: str) -> str: + return self.value.__format__(format_spec) + class BmssdAdapter(Adapter): def _decode(self, obj, context, path): @@ -130,8 +140,7 @@ def _decode(self, obj, context, path): res.scene_groups[sg.sg_name] = construct.Container() for ig_value, items in sg.item_groups.items(): - group_type = ItemType(ig_value) - res.scene_groups[sg.sg_name][group_type] = construct.ListContainer() + group_type = ItemType.from_index(ig_value) # objects are indexed and not hashed if ig_value == 1: @@ -142,9 +151,7 @@ def _decode(self, obj, context, path): res.scene_groups[sg.sg_name][group_type] = construct.ListContainer( [ # use raw hash value instead of block value if it doesn't exist above - res[f"_{group_type.group_name}"][block] - if res[f"_{group_type.group_name}"].get(block, None) - else block + res[f"_{group_type}"][block] if res[f"_{group_type}"].get(block, None) else block for block in items ] ) @@ -199,9 +206,9 @@ def obj_to_tuple(o): item_count += len(items) if group_type == ItemType.OBJECT: - sg_cont.item_groups[group_type.value] = [object_order[obj_to_tuple(o)] for o in items] + sg_cont.item_groups[group_type.group_index] = [object_order[obj_to_tuple(o)] for o in items] else: - sg_cont.item_groups[group_type.value] = [ + sg_cont.item_groups[group_type.group_index] = [ # handle integers (unmatched crc's in decode) o if isinstance(o, int) else crc(o["model_name"]) for o in items @@ -223,13 +230,13 @@ def get_item(self, item_name_or_id: str | int, item_type: ItemType) -> construct if item_type == ItemType.OBJECT: return self.raw._objects[item_name_or_id] else: - return self.raw[f"_{item_type.group_name}"].get(item_name_or_id, None) + return self.raw[f"_{item_type}"].get(item_name_or_id, None) if item_type == ItemType.OBJECT: raise ValueError("If accessing an Object type item, must use the index!") crc = crc_func(self.raw) - return self.raw[f"_{item_type.group_name}"].get(crc(item_name_or_id), None) + return self.raw[f"_{item_type}"].get(crc(item_name_or_id), None) def get_scene_group(self, scene_group: str) -> construct.Container: return self.raw.scene_groups.get(scene_group, None) @@ -245,7 +252,7 @@ def add_item(self, item: construct.Container, item_type: ItemType, scene_groups: self.raw._objects.append(item) else: crc = crc_func(self.raw) - self.raw[f"_{item_type.group_name}"][crc(item["model_name"])] = item + self.raw[f"_{item_type}"][crc(item["model_name"])] = item for sg_name in scene_groups: self.get_scene_group(sg_name)[item_type].append(item) @@ -260,4 +267,4 @@ def remove_item(self, item: construct.Container, item_type: ItemType): for sg in groups: self.remove_item_from_group(item, item_type, sg) - self.raw[f"_{item_type.group_name}"].remove(item) + self.raw[f"_{item_type}"].remove(item)