From 2c5c527358d1be57e90e91a4ececcf6ccb8d5605 Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 <28632661+HENDRIX-ZT2@users.noreply.github.com> Date: Tue, 26 Nov 2024 19:07:13 +0100 Subject: [PATCH] tex - start of injection for pc2 --- __version__.py | 4 ++-- generated/formats/ovl/__init__.py | 17 ++++++++++------- modules/formats/DDS.py | 10 +++++++++- modules/formats/shared.py | 14 ++++++++++++++ modules/walker.py | 11 +---------- ovl_tool_gui.py | 9 ++++++--- source/formats/ovl/__init__.py | 17 ++++++++++------- 7 files changed, 52 insertions(+), 30 deletions(-) diff --git a/__version__.py b/__version__.py index 6e48aef86..31596da81 100644 --- a/__version__.py +++ b/__version__.py @@ -1,4 +1,4 @@ # this file is auto-generated by the pre-commit hook increment_version.py VERSION = "2024.11.26" -COMMIT_HASH = "81f00ddc4" -COMMIT_TIME = "Tue Nov 26 18:32:43 2024 +0100" +COMMIT_HASH = "31f55d50a" +COMMIT_TIME = "Tue Nov 26 18:33:08 2024 +0100" diff --git a/generated/formats/ovl/__init__.py b/generated/formats/ovl/__init__.py index 1f2b59743..1d2c4017d 100644 --- a/generated/formats/ovl/__init__.py +++ b/generated/formats/ovl/__init__.py @@ -19,7 +19,7 @@ from generated.formats.ovl.versions import * from generated.formats.ovl_base.enums.Compression import Compression from modules.formats.formats_dict import FormatDict -from modules.formats.shared import djb2, DummyReporter +from modules.formats.shared import djb2, DummyReporter, walk_type from ovl_util.shared import hex_dump try: @@ -716,18 +716,21 @@ def create(self, ovl_dir): logging.info(f"Creating OVL from {ovl_dir}") self.store_filepath(f"{ovl_dir}.ovl") self.create_archive(name="STATIC") - file_paths = [os.path.join(ovl_dir, file_name) for file_name in os.listdir(ovl_dir)] + # todo - resolve redundancy to walk in add_files + # also catch subfolders for ui images in newer games + file_paths = [os.path.join(ovl_dir, file_name) for file_name in walk_type(ovl_dir, "")] self.loaders = {} - self.add_files(file_paths) + self.add_files(file_paths, common_root_dir=ovl_dir) self.load_included_ovls(os.path.join(ovl_dir, "ovls.include")) - def add_files(self, file_paths): + def add_files(self, file_paths, common_root_dir=None): logging.info(f"Adding {len(file_paths)} files to OVL [{self.game}]") logging.debug(file_paths) if not file_paths: return - # file_paths must be direct children of the same folder - common_root = os.path.dirname(file_paths[0]) + if not common_root_dir: + # file_paths must be direct children of the same folder + common_root_dir = os.path.dirname(file_paths[0]) file_paths = {os.path.normpath(file_path) for file_path in file_paths} inject_paths = set() # process the children of root @@ -783,7 +786,7 @@ def add_files(self, file_paths): logging.debug(f"Ignoring {file_path} - not a cobra format") continue # make relative to the common root, use forward slash as separator - file_name = os.path.relpath(file_path, common_root).replace("\\", "/") + file_name = os.path.relpath(file_path, common_root_dir).replace("\\", "/") try: loader = self.create_file(file_path, file_name) self.register_loader(loader) diff --git a/modules/formats/DDS.py b/modules/formats/DDS.py index 6756ebd2d..a9b9e9829 100644 --- a/modules/formats/DDS.py +++ b/modules/formats/DDS.py @@ -75,6 +75,15 @@ def prepare_buffers_and_streams(self, basename, buffer_bytes, name_ext): if self.context.is_pc_2: logging.warning("super experimental") texel_loader = self.get_texel() + # todo - refactor to move actual writing of aux to disk into OvlFile.save, + # as doing it here would require ovl name + dir to be set which isn't safe + # create aux file + with open(texel_loader.get_aux_path(""), "wb") as f: + # write bytes to it + f.write(buffer_bytes[0]) + # todo update offsets in mip infos accordingly + # for buf, mip in zip(buffer_bytes, self.texbuffer.mip_maps): + # pass # todo - instead directly register/remove texel as needed; # but do not consider it for check_controlled_conflicts, so no addition to self.controlled_loaders, or flag self.extra_loaders = [texel_loader, ] @@ -400,7 +409,6 @@ def extract(self, out_dir): with texbuffer.to_xml_file(texbuffer, texbuffer_path, debug=self.ovl.do_debug) as xml_root: pass out_files.append(texbuffer_path) - # get texel file from ovl to read external image buffer from aux texel_loader = self.get_texel() # image_buffer = texel_loader.get_image_buffer(texbuffer.mip_maps[0].offset, texbuffer.buffer_size) image_buffer = texel_loader.get_mip_buffers(texbuffer.mip_maps, dds_file, texbuffer) diff --git a/modules/formats/shared.py b/modules/formats/shared.py index 35d07675b..f3af55a90 100644 --- a/modules/formats/shared.py +++ b/modules/formats/shared.py @@ -1,7 +1,9 @@ import contextlib import logging +import os import struct import time +from pathlib import Path def get_padding_size(size, alignment=16): @@ -118,3 +120,15 @@ def log_duration(self, operation): yield duration = time.time() - start_time logging.debug(f"{operation} took {duration:.2f} seconds") + + +def walk_type(start_dir, extension=".ovl"): + logging.info(f"Scanning {Path(start_dir)} for {extension} files") + ret = [] + for root, dirs, files in os.walk(start_dir, topdown=False): + for name in files: + if extension and not name.lower().endswith(extension): + continue + ret.append(os.path.join(root, name)) + logging.info(ret) + return ret diff --git a/modules/walker.py b/modules/walker.py index 661eccc1c..9779801e2 100644 --- a/modules/walker.py +++ b/modules/walker.py @@ -5,6 +5,7 @@ from pathlib import Path from modules.formats.FGM import FgmContext +from modules.formats.shared import walk_type from ovl_util.logs import ANSI from constants.converter import write_mimes_dict, write_hashes_dict from generated.array import Array @@ -21,16 +22,6 @@ shader_map = {} -def walk_type(start_dir, extension=".ovl"): - logging.info(f"Scanning {Path(start_dir)} for {extension} files") - ret = [] - for root, dirs, files in os.walk(start_dir, topdown=False): - for name in files: - if name.lower().endswith(extension): - ret.append(os.path.join(root, name)) - return ret - - def content_folder(filepath: Path): """Return the Content folder in filepath""" if filepath.parent.name in ("ovldata", "walker_export"): diff --git a/ovl_tool_gui.py b/ovl_tool_gui.py index ad3e6f7ff..49e35ec82 100644 --- a/ovl_tool_gui.py +++ b/ovl_tool_gui.py @@ -8,6 +8,7 @@ from gui import widgets, startup, GuiOptions # Import widgets before everything except built-ins! from ovl_util.logs import get_stdout_handler from modules import walker +import modules.formats.shared from generated.formats.ovl import games, OvlFile from generated.formats.ovl_base.enums.Compression import Compression from PyQt5 import QtWidgets, QtGui, QtCore @@ -295,7 +296,7 @@ def handle_path(self, save_over=True, batch=False): if batch: with self.no_popups(): with self.log_level_override("WARNING"): - for ovl_path in walker.walk_type(self.walk_root(), extension=".ovl"): + for ovl_path in modules.formats.shared.walk_type(self.walk_root(), extension=".ovl"): # open ovl file self.open(ovl_path) # todo clear logger after each file, using self.file_widget.open_file would do that @@ -483,9 +484,11 @@ def inject_ask(self): def inject_files(self, files): """Tries to inject files into self.ovl_data""" if files: - self.cfg["dir_inject"] = os.path.dirname(files[0]) + # any path added from the gui is necessarily at the same folder level, dirs need to be expanded + common_root_dir = os.path.dirname(files[0]) + self.cfg["dir_inject"] = common_root_dir self.set_dirty() - self.run_in_threadpool(self.ovl_data.add_files, (), files) + self.run_in_threadpool(self.ovl_data.add_files, (), files, common_root_dir) # the gui is updated from the signal ovl.files_list emitted from add_files def get_replace_strings(self): diff --git a/source/formats/ovl/__init__.py b/source/formats/ovl/__init__.py index 7ccdf90ed..fd1a42921 100644 --- a/source/formats/ovl/__init__.py +++ b/source/formats/ovl/__init__.py @@ -18,7 +18,7 @@ from generated.formats.ovl.versions import * from generated.formats.ovl_base.enums.Compression import Compression from modules.formats.formats_dict import FormatDict -from modules.formats.shared import djb2, DummyReporter +from modules.formats.shared import djb2, DummyReporter, walk_type from ovl_util.shared import hex_dump try: @@ -715,18 +715,21 @@ def create(self, ovl_dir): logging.info(f"Creating OVL from {ovl_dir}") self.store_filepath(f"{ovl_dir}.ovl") self.create_archive(name="STATIC") - file_paths = [os.path.join(ovl_dir, file_name) for file_name in os.listdir(ovl_dir)] + # todo - resolve redundancy to walk in add_files + # also catch subfolders for ui images in newer games + file_paths = [os.path.join(ovl_dir, file_name) for file_name in walk_type(ovl_dir, "")] self.loaders = {} - self.add_files(file_paths) + self.add_files(file_paths, common_root_dir=ovl_dir) self.load_included_ovls(os.path.join(ovl_dir, "ovls.include")) - def add_files(self, file_paths): + def add_files(self, file_paths, common_root_dir=None): logging.info(f"Adding {len(file_paths)} files to OVL [{self.game}]") logging.debug(file_paths) if not file_paths: return - # file_paths must be direct children of the same folder - common_root = os.path.dirname(file_paths[0]) + if not common_root_dir: + # file_paths must be direct children of the same folder + common_root_dir = os.path.dirname(file_paths[0]) file_paths = {os.path.normpath(file_path) for file_path in file_paths} inject_paths = set() # process the children of root @@ -782,7 +785,7 @@ def add_files(self, file_paths): logging.debug(f"Ignoring {file_path} - not a cobra format") continue # make relative to the common root, use forward slash as separator - file_name = os.path.relpath(file_path, common_root).replace("\\", "/") + file_name = os.path.relpath(file_path, common_root_dir).replace("\\", "/") try: loader = self.create_file(file_path, file_name) self.register_loader(loader)