From 975ae94db802a927e1fb9a99ac85d8e25ee4388d Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 <28632661+HENDRIX-ZT2@users.noreply.github.com> Date: Tue, 3 Dec 2024 15:21:16 +0100 Subject: [PATCH] ovl - fixes for PC2 saving --- __init__.py | 2 +- __version__.py | 6 +-- .../formats/base/compounds/ZStringBuffer.py | 7 +++- generated/formats/ms2/compounds/JointData.py | 2 +- generated/formats/ovl/__init__.py | 23 ++++++----- modules/formats/shared.py | 38 ++++++++++--------- pyproject.toml | 2 +- .../formats/base/compounds/ZStringBuffer.py | 7 +++- source/formats/ms2/compounds/JointData.py | 2 +- source/formats/ovl/__init__.py | 23 ++++++----- 10 files changed, 66 insertions(+), 46 deletions(-) diff --git a/__init__.py b/__init__.py index bfb6bf57b..522989490 100644 --- a/__init__.py +++ b/__init__.py @@ -3,7 +3,7 @@ "name": "Frontier's Cobra Engine Formats", "author": "Harlequinz Ego, HENDRIX et al.", "blender": (4, 0, 0), - "version": (2024, 11, 27), + "version": (2024, 11, 28), "location": "File > Import-Export", "description": "Import-Export models, skeletons and animations", "warning": "", diff --git a/__version__.py b/__version__.py index ed01a82dd..ed229c4c2 100644 --- a/__version__.py +++ b/__version__.py @@ -1,4 +1,4 @@ # this file is auto-generated by the pre-commit hook increment_version.py -VERSION = "2024.11.27" -COMMIT_HASH = "1e9d7086b" -COMMIT_TIME = "Wed Nov 27 18:54:31 2024 +0100" +VERSION = "2024.11.28" +COMMIT_HASH = "b7a27aff9" +COMMIT_TIME = "Thu Nov 28 08:46:23 2024 +0100" diff --git a/generated/formats/base/compounds/ZStringBuffer.py b/generated/formats/base/compounds/ZStringBuffer.py index d88964e1e..883a317ff 100644 --- a/generated/formats/base/compounds/ZStringBuffer.py +++ b/generated/formats/base/compounds/ZStringBuffer.py @@ -49,17 +49,20 @@ def _get_str_at(self, pos): def update_strings(self, list_of_strs): """Updates this name buffer with a list of names""" # logging.debug(f"Updating name buffer, data = {len(self.data)} bytes {self.data}") - self.strings = sorted(set(list_of_strs)) + self.strings = [] self.offset_dic = {} self.offset_2_str = {} with BytesIO() as stream: - for name in self.strings: + for name in list_of_strs: if not isinstance(name, str): logging.warning(f"Name '{name}' is not a string, skipping") continue + if name in self.offset_dic: + continue # store offset and write zstring self.offset_dic[name] = stream.tell() self.offset_2_str[stream.tell()] = name + self.strings.append(name) ZString.to_stream(name, stream, self.context) # get the actual result buffer buffer_bytes = stream.getvalue() diff --git a/generated/formats/ms2/compounds/JointData.py b/generated/formats/ms2/compounds/JointData.py index 73d7be99d..5b025e1a2 100644 --- a/generated/formats/ms2/compounds/JointData.py +++ b/generated/formats/ms2/compounds/JointData.py @@ -236,7 +236,7 @@ def write_fields(cls, stream, instance): instance.context.joint_names = instance.joint_names # if instance.context.version <= 32: # set arg = instance.joint_names - strings = list(instance.get_strings()) + strings = sorted(instance.get_strings()) instance.joint_names.update_strings(strings) # at least PC+PZ store the length without the 8 byte alignment padding at the end # however the end of ZStringBuffer is aligned to 8 and may be padded additionally diff --git a/generated/formats/ovl/__init__.py b/generated/formats/ovl/__init__.py index 35ca9b854..e44270f9f 100644 --- a/generated/formats/ovl/__init__.py +++ b/generated/formats/ovl/__init__.py @@ -328,7 +328,7 @@ def rebuild_pools(self): logging.info("Updating pool names, deleting unused pools") # map the pool types to pools pools_by_type = {} - for pool_index, pool in enumerate(self.ovl.reporter.iter_progress(self.pools, "Updating pools")): + for pool_index, pool in enumerate(self.ovl.reporter.iter_progress(self.pools, "Rebuilding pools", cond=len(self.pools) > 1)): if pool.offsets: # store pool in pool_groups map if pool.type not in pools_by_type: @@ -1097,14 +1097,15 @@ def get_dep_hash(name): if UNK_HASH in name: logging.warning(f"Won't update hash {name}") return int(name.replace(f"{UNK_HASH}_", "")) + elif UNK_HASH.lower() in name: + logging.warning(f"Won't update hash {name}") + return int(name.replace(f"{UNK_HASH.lower()}_", "")) return djb2(name) def rebuild_ovl_arrays(self): """Call this if any file names have changed and hashes or indices have to be recomputed""" # update file hashes and extend entries per loader - # self.files.sort(key=lambda x: (x.ext, x.file_hash)) - # sorted_loaders = sorted(self.loaders, key=lambda x: (x.ext, x.file_hash)) loaders_by_extension = self.get_loaders_by_ext() mimes_ext = sorted(loaders_by_extension) mimes_triplets = [self.get_mime(ext, "triplets") for ext in mimes_ext] @@ -1138,10 +1139,10 @@ def rebuild_ovl_arrays(self): deps_ext = [ext.replace(".", ":") for ext in deps_ext] aux_suffices = [aux_suffix for aux_suffix, loader in loaders_and_aux] names_list = [ + *aux_suffices, *sorted(set(deps_ext)), *ovl_includes, *mimes_name, - *aux_suffices, *sorted(loader.basename for loader in self.loaders.values())] self.names.update_strings(names_list) # create the mimes @@ -1199,7 +1200,7 @@ def rebuild_ovl_arrays(self): # update all pools before indexing anything that points into pools pools_offset = 0 self.archives.sort(key=lambda a: a.name) - for archive in self.archives: + for archive in self.reporter.iter_progress(self.archives, "Rebuilding pools"): ovs = archive.content ovs.clear_ovs_arrays() ovs.rebuild_pools() @@ -1246,7 +1247,7 @@ def rebuild_ovs_arrays(self, flat_sorted_loaders, ext_lut): logging.exception(f"Couldn't map loader {loader.name} to ovs {loader.ovs_name}") raise # remove all entries to rebuild them from the loaders - for archive in self.archives: + for archive in self.reporter.iter_progress(self.archives, "Updating headers"): ovs = archive.content loaders = archive_name_to_loaders[archive.name] archive.num_root_entries = len(loaders) @@ -1307,7 +1308,7 @@ def resolve(pool, offset): pools_byte_offset = 0 # make a temporary copy so we can delete archive if needed - for archive in tuple(self.archives): + for archive in self.reporter.iter_progress(tuple(self.archives), "Updating archives"): logging.debug(f"Sorting pools for {archive.name}") ovs = archive.content @@ -1441,8 +1442,12 @@ def save(self, filepath, use_threads=True): self.rebuild_ovl_arrays() # these need to be done after the rest self.update_stream_files() - ovs_types = {archive.name for archive in self.archives if "Textures_L" not in archive.name} - ovs_types.discard("STATIC") + ovs_types = set() + for loader in self.loaders.values(): + if loader.ovs.arg.name != "STATIC" and loader.ext not in (".tex", ".texturestream"): + ovs_types.add(loader.ovs.arg.name) + # ovs_types = {archive.name for archive in self.archives if "Textures_L" not in archive.name} + # ovs_types.discard("STATIC") self.num_ovs_types = len(ovs_types) ovl_compressed = b"" self.reset_field("archives_meta") diff --git a/modules/formats/shared.py b/modules/formats/shared.py index f3af55a90..acb3cacf8 100644 --- a/modules/formats/shared.py +++ b/modules/formats/shared.py @@ -73,25 +73,29 @@ class DummyReporter: progress_total = DummySignal() # type: ignore current_action = DummySignal() # type: ignore - def iter_progress(self, iterable, message): - self.current_action.emit(message) - self._percentage = -1 - if len(iterable) > 1: + def iter_progress(self, iterable, message, cond=True): + if cond: + self.current_action.emit(message) + self._percentage = -1 + if len(iterable) > 1: + self.progress_total.emit(100) + else: + self.progress_total.emit(0) + for i, item in enumerate(iterable): + p = round(i / len(iterable) * 100) + if p != self._percentage: + self.progress_percentage.emit(p) + self._percentage = p + yield item + # clear both to also make indeterminate processes appear complete + self.progress_percentage.emit(100) self.progress_total.emit(100) + msg = f"Finished {message}" + self.current_action.emit(msg) + # logging.success(msg) else: - self.progress_total.emit(0) - for i, item in enumerate(iterable): - p = round(i / len(iterable) * 100) - if p != self._percentage: - self.progress_percentage.emit(p) - self._percentage = p - yield item - # clear both to also make indeterminate processes appear complete - self.progress_percentage.emit(100) - self.progress_total.emit(100) - msg = f"Finished {message}" - self.current_action.emit(msg) - # logging.success(msg) + for item in iterable: + yield item @contextlib.contextmanager def report_error_files(self, operation): diff --git a/pyproject.toml b/pyproject.toml index b2edcd687..fa2bcb5a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "cobra-tools" -version = "2024.11.27" +version = "2024.11.28" readme = "README.md" license = { file = "LICENSE" } requires-python = "==3.11.*" diff --git a/source/formats/base/compounds/ZStringBuffer.py b/source/formats/base/compounds/ZStringBuffer.py index 421231eb8..7af80b9db 100644 --- a/source/formats/base/compounds/ZStringBuffer.py +++ b/source/formats/base/compounds/ZStringBuffer.py @@ -37,17 +37,20 @@ def _get_str_at(self, pos): def update_strings(self, list_of_strs): """Updates this name buffer with a list of names""" # logging.debug(f"Updating name buffer, data = {len(self.data)} bytes {self.data}") - self.strings = sorted(set(list_of_strs)) + self.strings = [] self.offset_dic = {} self.offset_2_str = {} with BytesIO() as stream: - for name in self.strings: + for name in list_of_strs: if not isinstance(name, str): logging.warning(f"Name '{name}' is not a string, skipping") continue + if name in self.offset_dic: + continue # store offset and write zstring self.offset_dic[name] = stream.tell() self.offset_2_str[stream.tell()] = name + self.strings.append(name) ZString.to_stream(name, stream, self.context) # get the actual result buffer buffer_bytes = stream.getvalue() diff --git a/source/formats/ms2/compounds/JointData.py b/source/formats/ms2/compounds/JointData.py index 0e1cc32e6..86027e75b 100644 --- a/source/formats/ms2/compounds/JointData.py +++ b/source/formats/ms2/compounds/JointData.py @@ -55,7 +55,7 @@ def write_fields(cls, stream, instance): instance.context.joint_names = instance.joint_names # if instance.context.version <= 32: # set arg = instance.joint_names - strings = list(instance.get_strings()) + strings = sorted(instance.get_strings()) instance.joint_names.update_strings(strings) # at least PC+PZ store the length without the 8 byte alignment padding at the end # however the end of ZStringBuffer is aligned to 8 and may be padded additionally diff --git a/source/formats/ovl/__init__.py b/source/formats/ovl/__init__.py index e9c6c68b5..8f9f7fc05 100644 --- a/source/formats/ovl/__init__.py +++ b/source/formats/ovl/__init__.py @@ -327,7 +327,7 @@ def rebuild_pools(self): logging.info("Updating pool names, deleting unused pools") # map the pool types to pools pools_by_type = {} - for pool_index, pool in enumerate(self.ovl.reporter.iter_progress(self.pools, "Updating pools")): + for pool_index, pool in enumerate(self.ovl.reporter.iter_progress(self.pools, "Rebuilding pools", cond=len(self.pools) > 1)): if pool.offsets: # store pool in pool_groups map if pool.type not in pools_by_type: @@ -1096,14 +1096,15 @@ def get_dep_hash(name): if UNK_HASH in name: logging.warning(f"Won't update hash {name}") return int(name.replace(f"{UNK_HASH}_", "")) + elif UNK_HASH.lower() in name: + logging.warning(f"Won't update hash {name}") + return int(name.replace(f"{UNK_HASH.lower()}_", "")) return djb2(name) def rebuild_ovl_arrays(self): """Call this if any file names have changed and hashes or indices have to be recomputed""" # update file hashes and extend entries per loader - # self.files.sort(key=lambda x: (x.ext, x.file_hash)) - # sorted_loaders = sorted(self.loaders, key=lambda x: (x.ext, x.file_hash)) loaders_by_extension = self.get_loaders_by_ext() mimes_ext = sorted(loaders_by_extension) mimes_triplets = [self.get_mime(ext, "triplets") for ext in mimes_ext] @@ -1137,10 +1138,10 @@ def rebuild_ovl_arrays(self): deps_ext = [ext.replace(".", ":") for ext in deps_ext] aux_suffices = [aux_suffix for aux_suffix, loader in loaders_and_aux] names_list = [ + *aux_suffices, *sorted(set(deps_ext)), *ovl_includes, *mimes_name, - *aux_suffices, *sorted(loader.basename for loader in self.loaders.values())] self.names.update_strings(names_list) # create the mimes @@ -1198,7 +1199,7 @@ def rebuild_ovl_arrays(self): # update all pools before indexing anything that points into pools pools_offset = 0 self.archives.sort(key=lambda a: a.name) - for archive in self.archives: + for archive in self.reporter.iter_progress(self.archives, "Rebuilding pools"): ovs = archive.content ovs.clear_ovs_arrays() ovs.rebuild_pools() @@ -1245,7 +1246,7 @@ def rebuild_ovs_arrays(self, flat_sorted_loaders, ext_lut): logging.exception(f"Couldn't map loader {loader.name} to ovs {loader.ovs_name}") raise # remove all entries to rebuild them from the loaders - for archive in self.archives: + for archive in self.reporter.iter_progress(self.archives, "Updating headers"): ovs = archive.content loaders = archive_name_to_loaders[archive.name] archive.num_root_entries = len(loaders) @@ -1306,7 +1307,7 @@ def resolve(pool, offset): pools_byte_offset = 0 # make a temporary copy so we can delete archive if needed - for archive in tuple(self.archives): + for archive in self.reporter.iter_progress(tuple(self.archives), "Updating archives"): logging.debug(f"Sorting pools for {archive.name}") ovs = archive.content @@ -1440,8 +1441,12 @@ def save(self, filepath, use_threads=True): self.rebuild_ovl_arrays() # these need to be done after the rest self.update_stream_files() - ovs_types = {archive.name for archive in self.archives if "Textures_L" not in archive.name} - ovs_types.discard("STATIC") + ovs_types = set() + for loader in self.loaders.values(): + if loader.ovs.arg.name != "STATIC" and loader.ext not in (".tex", ".texturestream"): + ovs_types.add(loader.ovs.arg.name) + # ovs_types = {archive.name for archive in self.archives if "Textures_L" not in archive.name} + # ovs_types.discard("STATIC") self.num_ovs_types = len(ovs_types) ovl_compressed = b"" self.reset_field("archives_meta")