From ca83905d9f10a50c326b4736f6995f041f2905d7 Mon Sep 17 00:00:00 2001 From: Fabian Dill Date: Mon, 15 Aug 2022 23:52:03 +0200 Subject: [PATCH] Core: allow loading worlds from zip modules (#747) * Core: allow loading worlds from zip modules RoR2: make it zipimport compatible (remove relative imports beyond local top-level) * WebHost: add support for .apworld --- FactorioClient.py | 3 +-- WebHost.py | 34 +++++++++++++++++++----- worlds/AutoWorld.py | 52 ++++++++++++++++++++++--------------- worlds/__init__.py | 57 +++++++++++++++++++++++++++++++---------- worlds/ror2/Rules.py | 2 +- worlds/ror2/__init__.py | 2 +- 6 files changed, 104 insertions(+), 46 deletions(-) diff --git a/FactorioClient.py b/FactorioClient.py index 2fa8ba9c1511..6797578a3ad1 100644 --- a/FactorioClient.py +++ b/FactorioClient.py @@ -20,8 +20,7 @@ if __name__ == "__main__": Utils.init_logging("FactorioClient", exception_logger="Client") -from CommonClient import CommonContext, server_loop, console_loop, ClientCommandProcessor, logger, gui_enabled, \ - get_base_parser +from CommonClient import CommonContext, server_loop, ClientCommandProcessor, logger, gui_enabled, get_base_parser from MultiServer import mark_raw from NetUtils import NetworkItem, ClientStatus, JSONtoTextParser, JSONMessagePart diff --git a/WebHost.py b/WebHost.py index 09f8d8235a50..3d3c8678e271 100644 --- a/WebHost.py +++ b/WebHost.py @@ -42,20 +42,40 @@ def get_app(): def create_ordered_tutorials_file() -> typing.List[typing.Dict[str, typing.Any]]: import json import shutil - from worlds.AutoWorld import AutoWorldRegister + import pathlib + import zipfile + + zfile: zipfile.ZipInfo + + from worlds.AutoWorld import AutoWorldRegister, __file__ worlds = {} data = [] for game, world in AutoWorldRegister.world_types.items(): if hasattr(world.web, 'tutorials') and (not world.hidden or game == 'Archipelago'): worlds[game] = world + + base_target_path = Utils.local_path("WebHostLib", "static", "generated", "docs") for game, world in worlds.items(): # copy files from world's docs folder to the generated folder - source_path = Utils.local_path(os.path.dirname(sys.modules[world.__module__].__file__), 'docs') - target_path = Utils.local_path("WebHostLib", "static", "generated", "docs", game) - files = os.listdir(source_path) - for file in files: - os.makedirs(os.path.dirname(Utils.local_path(target_path, file)), exist_ok=True) - shutil.copyfile(Utils.local_path(source_path, file), Utils.local_path(target_path, file)) + target_path = os.path.join(base_target_path, game) + os.makedirs(target_path, exist_ok=True) + + if world.is_zip: + zipfile_path = pathlib.Path(world.__file__).parents[1] + + assert os.path.isfile(zipfile_path), f"{zipfile_path} is not a valid file(path)." + assert zipfile.is_zipfile(zipfile_path), f"{zipfile_path} is not a valid zipfile." + + with zipfile.ZipFile(zipfile_path) as zf: + for zfile in zf.infolist(): + if not zfile.is_dir() and "/docs/" in zfile.filename: + zf.extract(zfile, target_path) + else: + source_path = Utils.local_path(os.path.dirname(world.__file__), "docs") + files = os.listdir(source_path) + for file in files: + shutil.copyfile(Utils.local_path(source_path, file), Utils.local_path(target_path, file)) + # build a json tutorial dict per game game_data = {'gameTitle': game, 'tutorials': []} for tutorial in world.web.tutorials: diff --git a/worlds/AutoWorld.py b/worlds/AutoWorld.py index 462108bb8f06..2abb9f3a7136 100644 --- a/worlds/AutoWorld.py +++ b/worlds/AutoWorld.py @@ -2,10 +2,13 @@ import logging import sys -from typing import Dict, FrozenSet, Set, Tuple, List, Optional, TextIO, Any, Callable, Union, NamedTuple +from typing import Dict, FrozenSet, Set, Tuple, List, Optional, TextIO, Any, Callable, Union, TYPE_CHECKING -from BaseClasses import MultiWorld, Item, CollectionState, Location, Tutorial from Options import Option +from BaseClasses import CollectionState + +if TYPE_CHECKING: + from BaseClasses import MultiWorld, Item, Location, Tutorial class AutoWorldRegister(type): @@ -41,8 +44,11 @@ def __new__(mcs, name: str, bases: Tuple[type, ...], dct: Dict[str, Any]) -> Aut # construct class new_class = super().__new__(mcs, name, bases, dct) if "game" in dct: + if dct["game"] in AutoWorldRegister.world_types: + raise RuntimeError(f"""Game {dct["game"]} already registered.""") AutoWorldRegister.world_types[dct["game"]] = new_class new_class.__file__ = sys.modules[new_class.__module__].__file__ + new_class.is_zip = ".apworld" in new_class.__file__ return new_class @@ -62,12 +68,12 @@ def __new__(cls, name: str, bases: Tuple[type, ...], dct: Dict[str, Any]) -> Aut return new_class -def call_single(world: MultiWorld, method_name: str, player: int, *args: Any) -> Any: +def call_single(world: "MultiWorld", method_name: str, player: int, *args: Any) -> Any: method = getattr(world.worlds[player], method_name) return method(*args) -def call_all(world: MultiWorld, method_name: str, *args: Any) -> None: +def call_all(world: "MultiWorld", method_name: str, *args: Any) -> None: world_types: Set[AutoWorldRegister] = set() for player in world.player_ids: world_types.add(world.worlds[player].__class__) @@ -79,7 +85,7 @@ def call_all(world: MultiWorld, method_name: str, *args: Any) -> None: stage_callable(world, *args) -def call_stage(world: MultiWorld, method_name: str, *args: Any) -> None: +def call_stage(world: "MultiWorld", method_name: str, *args: Any) -> None: world_types = {world.worlds[player].__class__ for player in world.player_ids} for world_type in world_types: stage_callable = getattr(world_type, f"stage_{method_name}", None) @@ -97,7 +103,7 @@ class WebWorld: # docs folder will also be scanned for tutorial guides given the relevant information in this list. Each Tutorial # class is to be used for one guide. - tutorials: List[Tutorial] + tutorials: List["Tutorial"] # Choose a theme for your /game/* pages # Available: dirt, grass, grassFlowers, ice, jungle, ocean, partyTime, stone @@ -159,8 +165,11 @@ class World(metaclass=AutoWorldRegister): # Hide World Type from various views. Does not remove functionality. hidden: bool = False + # see WebWorld for options + web: WebWorld = WebWorld() + # autoset on creation: - world: MultiWorld + world: "MultiWorld" player: int # automatically generated @@ -170,9 +179,10 @@ class World(metaclass=AutoWorldRegister): item_names: Set[str] # set of all potential item names location_names: Set[str] # set of all potential location names - web: WebWorld = WebWorld() + is_zip: bool # was loaded from a .apworld ? + __file__: str # path it was loaded from - def __init__(self, world: MultiWorld, player: int): + def __init__(self, world: "MultiWorld", player: int): self.world = world self.player = player @@ -207,12 +217,12 @@ def pre_fill(self) -> None: @classmethod def fill_hook(cls, - progitempool: List[Item], - nonexcludeditempool: List[Item], - localrestitempool: Dict[int, List[Item]], - nonlocalrestitempool: Dict[int, List[Item]], - restitempool: List[Item], - fill_locations: List[Location]) -> None: + progitempool: List["Item"], + nonexcludeditempool: List["Item"], + localrestitempool: Dict[int, List["Item"]], + nonlocalrestitempool: Dict[int, List["Item"]], + restitempool: List["Item"], + fill_locations: List["Location"]) -> None: """Special method that gets called as part of distribute_items_restrictive (main fill). This gets called once per present world type.""" pass @@ -250,7 +260,7 @@ def write_spoiler_end(self, spoiler_handle: TextIO) -> None: # end of ordered Main.py calls - def create_item(self, name: str) -> Item: + def create_item(self, name: str) -> "Item": """Create an item for this world type and player. Warning: this may be called with self.world = None, for example by MultiServer""" raise NotImplementedError @@ -261,7 +271,7 @@ def get_filler_item_name(self) -> str: return self.world.random.choice(tuple(self.item_name_to_id.keys())) # decent place to implement progressive items, in most cases can stay as-is - def collect_item(self, state: CollectionState, item: Item, remove: bool = False) -> Optional[str]: + def collect_item(self, state: "CollectionState", item: "Item", remove: bool = False) -> Optional[str]: """Collect an item name into state. For speed reasons items that aren't logically useful get skipped. Collect None to skip item. :param state: CollectionState to collect into @@ -272,18 +282,18 @@ def collect_item(self, state: CollectionState, item: Item, remove: bool = False) return None # called to create all_state, return Items that are created during pre_fill - def get_pre_fill_items(self) -> List[Item]: + def get_pre_fill_items(self) -> List["Item"]: return [] # following methods should not need to be overridden. - def collect(self, state: CollectionState, item: Item) -> bool: + def collect(self, state: "CollectionState", item: "Item") -> bool: name = self.collect_item(state, item) if name: state.prog_items[name, self.player] += 1 return True return False - def remove(self, state: CollectionState, item: Item) -> bool: + def remove(self, state: "CollectionState", item: "Item") -> bool: name = self.collect_item(state, item, True) if name: state.prog_items[name, self.player] -= 1 @@ -292,7 +302,7 @@ def remove(self, state: CollectionState, item: Item) -> bool: return True return False - def create_filler(self) -> Item: + def create_filler(self) -> "Item": return self.create_item(self.get_filler_item_name()) diff --git a/worlds/__init__.py b/worlds/__init__.py index b9270836791b..caa170d5c650 100644 --- a/worlds/__init__.py +++ b/worlds/__init__.py @@ -1,29 +1,57 @@ import importlib +import zipimport import os +import typing -__all__ = {"lookup_any_item_id_to_name", - "lookup_any_location_id_to_name", - "network_data_package", - "AutoWorldRegister"} +folder = os.path.dirname(__file__) + +__all__ = { + "lookup_any_item_id_to_name", + "lookup_any_location_id_to_name", + "network_data_package", + "AutoWorldRegister", + "world_sources", + "folder", +} + +if typing.TYPE_CHECKING: + from .AutoWorld import World + + +class WorldSource(typing.NamedTuple): + path: str # typically relative path from this module + is_zip: bool = False + + +# find potential world containers, currently folders and zip-importable .apworld's +world_sources: typing.List[WorldSource] = [] +file: os.DirEntry # for me (Berserker) at least, PyCharm doesn't seem to infer the type correctly +for file in os.scandir(folder): + if not file.name.startswith("_"): # prevent explicitly loading __pycache__ and allow _* names for non-world folders + if file.is_dir(): + world_sources.append(WorldSource(file.name)) + elif file.is_file() and file.name.endswith(".apworld"): + world_sources.append(WorldSource(file.name, is_zip=True)) # import all submodules to trigger AutoWorldRegister -world_folders = [] -for file in os.scandir(os.path.dirname(__file__)): - if file.is_dir(): - world_folders.append(file.name) -world_folders.sort() -for world in world_folders: - if not world.startswith("_"): # prevent explicitly loading __pycache__ and allow _* names for non-world folders - importlib.import_module(f".{world}", "worlds") +world_sources.sort() +for world_source in world_sources: + if world_source.is_zip: + + importer = zipimport.zipimporter(os.path.join(folder, world_source.path)) + importer.load_module(world_source.path.split(".", 1)[0]) + else: + importlib.import_module(f".{world_source.path}", "worlds") -from .AutoWorld import AutoWorldRegister lookup_any_item_id_to_name = {} lookup_any_location_id_to_name = {} games = {} +from .AutoWorld import AutoWorldRegister + for world_name, world in AutoWorldRegister.world_types.items(): games[world_name] = { - "item_name_to_id" : world.item_name_to_id, + "item_name_to_id": world.item_name_to_id, "location_name_to_id": world.location_name_to_id, "version": world.data_version, # seems clients don't actually want this. Keeping it here in case someone changes their mind. @@ -41,5 +69,6 @@ if any(not world.data_version for world in AutoWorldRegister.world_types.values()): network_data_package["version"] = 0 import logging + logging.warning(f"Datapackage is in custom mode. Custom Worlds: " f"{[world for world in AutoWorldRegister.world_types.values() if not world.data_version]}") diff --git a/worlds/ror2/Rules.py b/worlds/ror2/Rules.py index 05c08c880379..64d741f99fa5 100644 --- a/worlds/ror2/Rules.py +++ b/worlds/ror2/Rules.py @@ -1,5 +1,5 @@ from BaseClasses import MultiWorld -from ..generic.Rules import set_rule, add_rule +from worlds.generic.Rules import set_rule, add_rule def set_rules(world: MultiWorld, player: int): diff --git a/worlds/ror2/__init__.py b/worlds/ror2/__init__.py index b1f3aa93071c..9d0d693b6148 100644 --- a/worlds/ror2/__init__.py +++ b/worlds/ror2/__init__.py @@ -5,7 +5,7 @@ from BaseClasses import Region, RegionType, Entrance, Item, ItemClassification, MultiWorld, Tutorial from .Options import ror2_options -from ..AutoWorld import World, WebWorld +from worlds.AutoWorld import World, WebWorld client_version = 1