Core: allow loading worlds from zip modules (ArchipelagoMW#747)

* Core: allow loading worlds from zip modules
RoR2: make it zipimport compatible (remove relative imports beyond local top-level)

* WebHost: add support for .apworld

FactorioClient.py

@@ -20,8 +20,7 @@
 if __name__ == "__main__":
     Utils.init_logging("FactorioClient", exception_logger="Client")
 
-from CommonClient import CommonContext, server_loop, console_loop, ClientCommandProcessor, logger, gui_enabled, \
-    get_base_parser
+from CommonClient import CommonContext, server_loop, ClientCommandProcessor, logger, gui_enabled, get_base_parser
 from MultiServer import mark_raw
 from NetUtils import NetworkItem, ClientStatus, JSONtoTextParser, JSONMessagePart
 

WebHost.py

@@ -42,20 +42,40 @@ def get_app():
 def create_ordered_tutorials_file() -> typing.List[typing.Dict[str, typing.Any]]:
     import json
     import shutil
-    from worlds.AutoWorld import AutoWorldRegister
+    import pathlib
+    import zipfile
+
+    zfile: zipfile.ZipInfo
+
+    from worlds.AutoWorld import AutoWorldRegister, __file__
     worlds = {}
     data = []
     for game, world in AutoWorldRegister.world_types.items():
         if hasattr(world.web, 'tutorials') and (not world.hidden or game == 'Archipelago'):
             worlds[game] = world
+
+    base_target_path = Utils.local_path("WebHostLib", "static", "generated", "docs")
     for game, world in worlds.items():
         # copy files from world's docs folder to the generated folder
-        source_path = Utils.local_path(os.path.dirname(sys.modules[world.__module__].__file__), 'docs')
-        target_path = Utils.local_path("WebHostLib", "static", "generated", "docs", game)
-        files = os.listdir(source_path)
-        for file in files:
-            os.makedirs(os.path.dirname(Utils.local_path(target_path, file)), exist_ok=True)
-            shutil.copyfile(Utils.local_path(source_path, file), Utils.local_path(target_path, file))
+        target_path = os.path.join(base_target_path, game)
+        os.makedirs(target_path, exist_ok=True)
+
+        if world.is_zip:
+            zipfile_path = pathlib.Path(world.__file__).parents[1]
+
+            assert os.path.isfile(zipfile_path), f"{zipfile_path} is not a valid file(path)."
+            assert zipfile.is_zipfile(zipfile_path), f"{zipfile_path} is not a valid zipfile."
+
+            with zipfile.ZipFile(zipfile_path) as zf:
+                for zfile in zf.infolist():
+                    if not zfile.is_dir() and "/docs/" in zfile.filename:
+                        zf.extract(zfile, target_path)
+        else:
+            source_path = Utils.local_path(os.path.dirname(world.__file__), "docs")
+            files = os.listdir(source_path)
+            for file in files:
+                shutil.copyfile(Utils.local_path(source_path, file), Utils.local_path(target_path, file))
+
         # build a json tutorial dict per game
         game_data = {'gameTitle': game, 'tutorials': []}
         for tutorial in world.web.tutorials:

worlds/AutoWorld.py

@@ -2,10 +2,13 @@
 
 import logging
 import sys
-from typing import Dict, FrozenSet, Set, Tuple, List, Optional, TextIO, Any, Callable, Union, NamedTuple
+from typing import Dict, FrozenSet, Set, Tuple, List, Optional, TextIO, Any, Callable, Union, TYPE_CHECKING
 
-from BaseClasses import MultiWorld, Item, CollectionState, Location, Tutorial
 from Options import Option
+from BaseClasses import CollectionState
+
+if TYPE_CHECKING:
+    from BaseClasses import MultiWorld, Item, Location, Tutorial
 
 
 class AutoWorldRegister(type):
@@ -41,8 +44,11 @@ def __new__(mcs, name: str, bases: Tuple[type, ...], dct: Dict[str, Any]) -> Aut
         # construct class
         new_class = super().__new__(mcs, name, bases, dct)
         if "game" in dct:
+            if dct["game"] in AutoWorldRegister.world_types:
+                raise RuntimeError(f"""Game {dct["game"]} already registered.""")
             AutoWorldRegister.world_types[dct["game"]] = new_class
         new_class.__file__ = sys.modules[new_class.__module__].__file__
+        new_class.is_zip = ".apworld" in new_class.__file__
         return new_class
 
 
@@ -62,12 +68,12 @@ def __new__(cls, name: str, bases: Tuple[type, ...], dct: Dict[str, Any]) -> Aut
         return new_class
 
 
-def call_single(world: MultiWorld, method_name: str, player: int, *args: Any) -> Any:
+def call_single(world: "MultiWorld", method_name: str, player: int, *args: Any) -> Any:
     method = getattr(world.worlds[player], method_name)
     return method(*args)
 
 
-def call_all(world: MultiWorld, method_name: str, *args: Any) -> None:
+def call_all(world: "MultiWorld", method_name: str, *args: Any) -> None:
     world_types: Set[AutoWorldRegister] = set()
     for player in world.player_ids:
         world_types.add(world.worlds[player].__class__)
@@ -79,7 +85,7 @@ def call_all(world: MultiWorld, method_name: str, *args: Any) -> None:
             stage_callable(world, *args)
 
 
-def call_stage(world: MultiWorld, method_name: str, *args: Any) -> None:
+def call_stage(world: "MultiWorld", method_name: str, *args: Any) -> None:
     world_types = {world.worlds[player].__class__ for player in world.player_ids}
     for world_type in world_types:
         stage_callable = getattr(world_type, f"stage_{method_name}", None)
@@ -97,7 +103,7 @@ class WebWorld:
 
     # docs folder will also be scanned for tutorial guides given the relevant information in this list. Each Tutorial
     # class is to be used for one guide.
-    tutorials: List[Tutorial]
+    tutorials: List["Tutorial"]
 
     # Choose a theme for your /game/* pages
     # Available: dirt, grass, grassFlowers, ice, jungle, ocean, partyTime, stone
@@ -159,8 +165,11 @@ class World(metaclass=AutoWorldRegister):
     # Hide World Type from various views. Does not remove functionality.
     hidden: bool = False
 
+    # see WebWorld for options
+    web: WebWorld = WebWorld()
+
     # autoset on creation:
-    world: MultiWorld
+    world: "MultiWorld"
     player: int
 
     # automatically generated
@@ -170,9 +179,10 @@ class World(metaclass=AutoWorldRegister):
     item_names: Set[str]  # set of all potential item names
     location_names: Set[str]  # set of all potential location names
 
-    web: WebWorld = WebWorld()
+    is_zip: bool  # was loaded from a .apworld ?
+    __file__: str  # path it was loaded from
 
-    def __init__(self, world: MultiWorld, player: int):
+    def __init__(self, world: "MultiWorld", player: int):
         self.world = world
         self.player = player
 
@@ -207,12 +217,12 @@ def pre_fill(self) -> None:
 
     @classmethod
     def fill_hook(cls,
-                  progitempool: List[Item],
-                  nonexcludeditempool: List[Item],
-                  localrestitempool: Dict[int, List[Item]],
-                  nonlocalrestitempool: Dict[int, List[Item]],
-                  restitempool: List[Item],
-                  fill_locations: List[Location]) -> None:
+                  progitempool: List["Item"],
+                  nonexcludeditempool: List["Item"],
+                  localrestitempool: Dict[int, List["Item"]],
+                  nonlocalrestitempool: Dict[int, List["Item"]],
+                  restitempool: List["Item"],
+                  fill_locations: List["Location"]) -> None:
         """Special method that gets called as part of distribute_items_restrictive (main fill).
         This gets called once per present world type."""
         pass
@@ -250,7 +260,7 @@ def write_spoiler_end(self, spoiler_handle: TextIO) -> None:
 
     # end of ordered Main.py calls
 
-    def create_item(self, name: str) -> Item:
+    def create_item(self, name: str) -> "Item":
         """Create an item for this world type and player.
         Warning: this may be called with self.world = None, for example by MultiServer"""
         raise NotImplementedError
@@ -261,7 +271,7 @@ def get_filler_item_name(self) -> str:
         return self.world.random.choice(tuple(self.item_name_to_id.keys()))
 
     # decent place to implement progressive items, in most cases can stay as-is
-    def collect_item(self, state: CollectionState, item: Item, remove: bool = False) -> Optional[str]:
+    def collect_item(self, state: "CollectionState", item: "Item", remove: bool = False) -> Optional[str]:
         """Collect an item name into state. For speed reasons items that aren't logically useful get skipped.
         Collect None to skip item.
         :param state: CollectionState to collect into
@@ -272,18 +282,18 @@ def collect_item(self, state: CollectionState, item: Item, remove: bool = False)
         return None
 
     # called to create all_state, return Items that are created during pre_fill
-    def get_pre_fill_items(self) -> List[Item]:
+    def get_pre_fill_items(self) -> List["Item"]:
         return []
 
     # following methods should not need to be overridden.
-    def collect(self, state: CollectionState, item: Item) -> bool:
+    def collect(self, state: "CollectionState", item: "Item") -> bool:
         name = self.collect_item(state, item)
         if name:
             state.prog_items[name, self.player] += 1
             return True
         return False
 
-    def remove(self, state: CollectionState, item: Item) -> bool:
+    def remove(self, state: "CollectionState", item: "Item") -> bool:
         name = self.collect_item(state, item, True)
         if name:
             state.prog_items[name, self.player] -= 1
@@ -292,7 +302,7 @@ def remove(self, state: CollectionState, item: Item) -> bool:
             return True
         return False
 
-    def create_filler(self) -> Item:
+    def create_filler(self) -> "Item":
         return self.create_item(self.get_filler_item_name())
 
 

worlds/__init__.py

@@ -1,29 +1,57 @@
 import importlib
+import zipimport
 import os
+import typing
 
-__all__ = {"lookup_any_item_id_to_name",
-           "lookup_any_location_id_to_name",
-           "network_data_package",
-           "AutoWorldRegister"}
+folder = os.path.dirname(__file__)
+
+__all__ = {
+    "lookup_any_item_id_to_name",
+    "lookup_any_location_id_to_name",
+    "network_data_package",
+    "AutoWorldRegister",
+    "world_sources",
+    "folder",
+}
+
+if typing.TYPE_CHECKING:
+    from .AutoWorld import World
+
+
+class WorldSource(typing.NamedTuple):
+    path: str  # typically relative path from this module
+    is_zip: bool = False
+
+
+# find potential world containers, currently folders and zip-importable .apworld's
+world_sources: typing.List[WorldSource] = []
+file: os.DirEntry  # for me (Berserker) at least, PyCharm doesn't seem to infer the type correctly
+for file in os.scandir(folder):
+    if not file.name.startswith("_"):  # prevent explicitly loading __pycache__ and allow _* names for non-world folders
+        if file.is_dir():
+            world_sources.append(WorldSource(file.name))
+        elif file.is_file() and file.name.endswith(".apworld"):
+            world_sources.append(WorldSource(file.name, is_zip=True))
 
 # import all submodules to trigger AutoWorldRegister
-world_folders = []
-for file in os.scandir(os.path.dirname(__file__)):
-    if file.is_dir():
-        world_folders.append(file.name)
-world_folders.sort()
-for world in world_folders:
-    if not world.startswith("_"):  # prevent explicitly loading __pycache__ and allow _* names for non-world folders
-        importlib.import_module(f".{world}", "worlds")
+world_sources.sort()
+for world_source in world_sources:
+    if world_source.is_zip:
+
+        importer = zipimport.zipimporter(os.path.join(folder, world_source.path))
+        importer.load_module(world_source.path.split(".", 1)[0])
+    else:
+        importlib.import_module(f".{world_source.path}", "worlds")
 
-from .AutoWorld import AutoWorldRegister
 lookup_any_item_id_to_name = {}
 lookup_any_location_id_to_name = {}
 games = {}
 
+from .AutoWorld import AutoWorldRegister
+
 for world_name, world in AutoWorldRegister.world_types.items():
     games[world_name] = {
-        "item_name_to_id" : world.item_name_to_id,
+        "item_name_to_id": world.item_name_to_id,
         "location_name_to_id": world.location_name_to_id,
         "version": world.data_version,
         # seems clients don't actually want this. Keeping it here in case someone changes their mind.
@@ -41,5 +69,6 @@
 if any(not world.data_version for world in AutoWorldRegister.world_types.values()):
     network_data_package["version"] = 0
     import logging
+
     logging.warning(f"Datapackage is in custom mode. Custom Worlds: "
                     f"{[world for world in AutoWorldRegister.world_types.values() if not world.data_version]}")

worlds/ror2/Rules.py

@@ -1,5 +1,5 @@
 from BaseClasses import MultiWorld
-from ..generic.Rules import set_rule, add_rule
+from worlds.generic.Rules import set_rule, add_rule
 
 
 def set_rules(world: MultiWorld, player: int):

worlds/ror2/__init__.py

@@ -5,7 +5,7 @@
 
 from BaseClasses import Region, RegionType, Entrance, Item, ItemClassification, MultiWorld, Tutorial
 from .Options import ror2_options
-from ..AutoWorld import World, WebWorld
+from worlds.AutoWorld import World, WebWorld
 
 client_version = 1