Merge branch 'main' into component_args_1

.github/workflows/unittests.yml

@@ -37,12 +37,13 @@ jobs:
           - {version: '3.9'}
           - {version: '3.10'}
           - {version: '3.11'}
+          - {version: '3.12'}
         include:
           - python: {version: '3.8'}  # win7 compat
             os: windows-latest
-          - python: {version: '3.11'}  # current
+          - python: {version: '3.12'}  # current
             os: windows-latest
-          - python: {version: '3.11'}  # current
+          - python: {version: '3.12'}  # current
             os: macos-latest
 
     steps:
@@ -70,7 +71,7 @@ jobs:
         os:
           - ubuntu-latest
         python:
-          - {version: '3.11'}  # current
+          - {version: '3.12'}  # current
 
     steps:
     - uses: actions/checkout@v4

BaseClasses.py

@@ -692,17 +692,25 @@ def __init__(self, parent: MultiWorld):
 
     def update_reachable_regions(self, player: int):
         self.stale[player] = False
+        world: AutoWorld.World = self.multiworld.worlds[player]
         reachable_regions = self.reachable_regions[player]
-        blocked_connections = self.blocked_connections[player]
         queue = deque(self.blocked_connections[player])
-        start = self.multiworld.get_region("Menu", player)
+        start: Region = world.get_region(world.origin_region_name)
 
         # init on first call - this can't be done on construction since the regions don't exist yet
         if start not in reachable_regions:
             reachable_regions.add(start)
-            blocked_connections.update(start.exits)
+            self.blocked_connections[player].update(start.exits)
             queue.extend(start.exits)
 
+        if world.explicit_indirect_conditions:
+            self._update_reachable_regions_explicit_indirect_conditions(player, queue)
+        else:
+            self._update_reachable_regions_auto_indirect_conditions(player, queue)
+
+    def _update_reachable_regions_explicit_indirect_conditions(self, player: int, queue: deque):
+        reachable_regions = self.reachable_regions[player]
+        blocked_connections = self.blocked_connections[player]
         # run BFS on all connections, and keep track of those blocked by missing items
         while queue:
             connection = queue.popleft()
@@ -722,6 +730,29 @@ def update_reachable_regions(self, player: int):
                     if new_entrance in blocked_connections and new_entrance not in queue:
                         queue.append(new_entrance)
 
+    def _update_reachable_regions_auto_indirect_conditions(self, player: int, queue: deque):
+        reachable_regions = self.reachable_regions[player]
+        blocked_connections = self.blocked_connections[player]
+        new_connection: bool = True
+        # run BFS on all connections, and keep track of those blocked by missing items
+        while new_connection:
+            new_connection = False
+            while queue:
+                connection = queue.popleft()
+                new_region = connection.connected_region
+                if new_region in reachable_regions:
+                    blocked_connections.remove(connection)
+                elif connection.can_reach(self):
+                    assert new_region, f"tried to search through an Entrance \"{connection}\" with no Region"
+                    reachable_regions.add(new_region)
+                    blocked_connections.remove(connection)
+                    blocked_connections.update(new_region.exits)
+                    queue.extend(new_region.exits)
+                    self.path[new_region] = (new_region.name, self.path.get(connection, None))
+                    new_connection = True
+            # sweep for indirect connections, mostly Entrance.can_reach(unrelated_Region)
+            queue.extend(blocked_connections)
+
     def copy(self) -> CollectionState:
         ret = CollectionState(self.multiworld)
         ret.prog_items = {player: counter.copy() for player, counter in self.prog_items.items()}
@@ -1176,7 +1207,7 @@ class ItemClassification(IntFlag):
     filler = 0b0000  # aka trash, as in filler items like ammo, currency etc,
     progression = 0b0001  # Item that is logically relevant
     useful = 0b0010  # Item that is generally quite useful, but not required for anything logical
-    trap = 0b0100  # detrimental or entirely useless (nothing) item
+    trap = 0b0100  # detrimental item
     skip_balancing = 0b1000  # should technically never occur on its own
     # Item that is logically relevant, but progression balancing should not touch.
     # Typically currency or other counted items.

CommonClient.py

@@ -662,17 +662,19 @@ def handle_connection_loss(self, msg: str) -> None:
         logger.exception(msg, exc_info=exc_info, extra={'compact_gui': True})
         self._messagebox_connection_loss = self.gui_error(msg, exc_info[1])
 
-    def run_gui(self):
-        """Import kivy UI system and start running it as self.ui_task."""
+    def make_gui(self) -> type:
+        """To return the Kivy App class needed for run_gui so it can be overridden before being built"""
         from kvui import GameManager
 
         class TextManager(GameManager):
-            logging_pairs = [
-                ("Client", "Archipelago")
-            ]
             base_title = "Archipelago Text Client"
 
-        self.ui = TextManager(self)
+        return TextManager
+
+    def run_gui(self):
+        """Import kivy UI system from make_gui() and start running it as self.ui_task."""
+        ui_class = self.make_gui()
+        self.ui = ui_class(self)
         self.ui_task = asyncio.create_task(self.ui.async_run(), name="UI")
 
     def run_cli(self):
@@ -1035,6 +1037,7 @@ async def main(args):
     parser.add_argument("url", nargs="?", help="Archipelago connection url")
     args = parser.parse_args(args)
 
+
     if args.url:
         url = urllib.parse.urlparse(args.url)
         args.connect = url.netloc

Launcher.py

@@ -16,10 +16,11 @@
 import shlex
 import subprocess
 import sys
+import urllib.parse
 import webbrowser
 from os.path import isfile
 from shutil import which
-from typing import Callable, Sequence, Union, Optional
+from typing import Callable, Optional, Sequence, Tuple, Union
 
 import Utils
 import settings
@@ -107,7 +108,81 @@ def update_settings():
 ])
 
 
-def identify(path: Union[None, str]):
+def handle_uri(path: str, launch_args: Tuple[str, ...]) -> None:
+    url = urllib.parse.urlparse(path)
+    queries = urllib.parse.parse_qs(url.query)
+    launch_args = (path, *launch_args)
+    client_component = None
+    text_client_component = None
+    if "game" in queries:
+        game = queries["game"][0]
+    else:  # TODO around 0.6.0 - this is for pre this change webhost uri's
+        game = "Archipelago"
+    for component in components:
+        if component.supports_uri and component.game_name == game:
+            client_component = component
+        elif component.display_name == "Text Client":
+            text_client_component = component
+
+    from kvui import App, Button, BoxLayout, Label, Clock, Window
+
+    class Popup(App):
+        timer_label: Label
+        remaining_time: Optional[int]
+
+        def __init__(self):
+            self.title = "Connect to Multiworld"
+            self.icon = r"data/icon.png"
+            super().__init__()
+
+        def build(self):
+            layout = BoxLayout(orientation="vertical")
+
+            if client_component is None:
+                self.remaining_time = 7
+                label_text = (f"A game client able to parse URIs was not detected for {game}.\n"
+                              f"Launching Text Client in 7 seconds...")
+                self.timer_label = Label(text=label_text)
+                layout.add_widget(self.timer_label)
+                Clock.schedule_interval(self.update_label, 1)
+            else:
+                layout.add_widget(Label(text="Select client to open and connect with."))
+                button_row = BoxLayout(orientation="horizontal", size_hint=(1, 0.4))
+
+                text_client_button = Button(
+                    text=text_client_component.display_name,
+                    on_release=lambda *args: run_component(text_client_component, *launch_args)
+                )
+                button_row.add_widget(text_client_button)
+
+                game_client_button = Button(
+                    text=client_component.display_name,
+                    on_release=lambda *args: run_component(client_component, *launch_args)
+                )
+                button_row.add_widget(game_client_button)
+
+                layout.add_widget(button_row)
+
+            return layout
+
+        def update_label(self, dt):
+            if self.remaining_time > 1:
+                # countdown the timer and string replace the number
+                self.remaining_time -= 1
+                self.timer_label.text = self.timer_label.text.replace(
+                    str(self.remaining_time + 1), str(self.remaining_time)
+                )
+            else:
+                # our timer is finished so launch text client and close down
+                run_component(text_client_component, *launch_args)
+                Clock.unschedule(self.update_label)
+                App.get_running_app().stop()
+                Window.close()
+
+    Popup().run()
+
+
+def identify(path: Union[None, str]) -> Tuple[Union[None, str], Union[None, Component]]:
     if path is None:
         return None, None
     for component in components:
@@ -299,20 +374,24 @@ def main(args: Optional[Union[argparse.Namespace, dict]] = None):
     elif not args:
         args = {}
 
-    if args.get("Patch|Game|Component", None) is not None:
-        file, component = identify(args["Patch|Game|Component"])
+    path = args.get("Patch|Game|Component|url", None)
+    if path is not None:
+        if path.startswith("archipelago://"):
+            handle_uri(path, args.get("args", ()))
+            return
+        file, component = identify(path)
         if file:
             args['file'] = file
         if component:
             args['component'] = component
         if not component:
-            logging.warning(f"Could not identify Component responsible for {args['Patch|Game|Component']}")
+            logging.warning(f"Could not identify Component responsible for {path}")
 
     if args["update_settings"]:
         update_settings()
-    if 'file' in args:
+    if "file" in args:
         run_component(args["component"], args["file"], *args["args"])
-    elif 'component' in args:
+    elif "component" in args:
         run_component(args["component"], *args["args"])
     elif not args["update_settings"]:
         run_gui()
@@ -329,8 +408,9 @@ def main(args: Optional[Union[argparse.Namespace, dict]] = None):
     run_group = parser.add_argument_group("Run")
     run_group.add_argument("--update_settings", action="store_true",
                            help="Update host.yaml and exit.")
-    run_group.add_argument("Patch|Game|Component", type=str, nargs="?",
-                           help="Pass either a patch file, a generated game or the name of a component to run.")
+    run_group.add_argument("Patch|Game|Component|url", type=str, nargs="?",
+                           help="Pass either a patch file, a generated game, the component name to run, or a url to "
+                                "connect with.")
     run_group.add_argument("args", nargs="*",
                            help="Arguments to pass to component.")
     main(parser.parse_args())

ModuleUpdate.py

@@ -75,13 +75,13 @@ def update(yes: bool = False, force: bool = False) -> None:
     if not update_ran:
         update_ran = True
 
+        install_pkg_resources(yes=yes)
+        import pkg_resources
+
         if force:
             update_command()
             return
 
-        install_pkg_resources(yes=yes)
-        import pkg_resources
-
         prev = ""  # if a line ends in \ we store here and merge later
         for req_file in requirements_files:
             path = os.path.join(os.path.dirname(sys.argv[0]), req_file)

MultiServer.py

@@ -67,6 +67,21 @@ def update_dict(dictionary, entries):
     return dictionary
 
 
+def queue_gc():
+    import gc
+    from threading import Thread
+
+    gc_thread: typing.Optional[Thread] = getattr(queue_gc, "_thread", None)
+    def async_collect():
+        time.sleep(2)
+        setattr(queue_gc, "_thread", None)
+        gc.collect()
+    if not gc_thread:
+        gc_thread = Thread(target=async_collect)
+        setattr(queue_gc, "_thread", gc_thread)
+        gc_thread.start()
+
+
 # functions callable on storable data on the server by clients
 modify_functions = {
     # generic:
@@ -551,6 +566,9 @@ def get_datetime_second():
                         self.logger.info(f"Saving failed. Retry in {self.auto_save_interval} seconds.")
                     else:
                         self.save_dirty = False
+                if not atexit_save:  # if atexit is used, that keeps a reference anyway
+                    queue_gc()
+
             self.auto_saver_thread = threading.Thread(target=save_regularly, daemon=True)
             self.auto_saver_thread.start()
 
@@ -1203,6 +1221,10 @@ def _cmd_countdown(self, seconds: str = "10") -> bool:
             timer = int(seconds, 10)
         except ValueError:
             timer = 10
+        else:
+            if timer > 60 * 60:
+                raise ValueError(f"{timer} is invalid. Maximum is 1 hour.")
+
         async_start(countdown(self.ctx, timer))
         return True
 
@@ -2039,6 +2061,8 @@ def _cmd_send_multiple(self, amount: typing.Union[int, str], player_name: str, *
             item_name, usable, response = get_intended_text(item_name, names)
             if usable:
                 amount: int = int(amount)
+                if amount > 100:
+                    raise ValueError(f"{amount} is invalid. Maximum is 100.")
                 new_items = [NetworkItem(names[item_name], -1, 0) for _ in range(int(amount))]
                 send_items_to(self.ctx, team, slot, *new_items)
 

WebHost.py

@@ -1,3 +1,4 @@
+import argparse
 import os
 import multiprocessing
 import logging
@@ -31,6 +32,15 @@ def get_app() -> "Flask":
         import yaml
         app.config.from_file(configpath, yaml.safe_load)
         logging.info(f"Updated config from {configpath}")
+    # inside get_app() so it's usable in systems like gunicorn, which do not run WebHost.py, but import it.
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--config_override', default=None,
+                        help="Path to yaml config file that overrules config.yaml.")
+    args = parser.parse_known_args()[0]
+    if args.config_override:
+        import yaml
+        app.config.from_file(os.path.abspath(args.config_override), yaml.safe_load)
+        logging.info(f"Updated config from {args.config_override}")
     if not app.config["HOST_ADDRESS"]:
         logging.info("Getting public IP, as HOST_ADDRESS is empty.")
         app.config["HOST_ADDRESS"] = Utils.get_public_ipv4()

WebHostLib/customserver.py

@@ -72,6 +72,14 @@ def __init__(self, static_server_data: dict, logger: logging.Logger):
         self.video = {}
         self.tags = ["AP", "WebHost"]
 
+    def __del__(self):
+        try:
+            import psutil
+            from Utils import format_SI_prefix
+            self.logger.debug(f"Context destroyed, Mem: {format_SI_prefix(psutil.Process().memory_info().rss, 1024)}iB")
+        except ImportError:
+            self.logger.debug("Context destroyed")
+
     def _load_game_data(self):
         for key, value in self.static_server_data.items():
             # NOTE: attributes are mutable and shared, so they will have to be copied before being modified
@@ -249,6 +257,7 @@ async def start_room(room_id):
                 ctx = WebHostContext(static_server_data, logger)
                 ctx.load(room_id)
                 ctx.init_save()
+                assert ctx.server is None
                 try:
                     ctx.server = websockets.serve(
                         functools.partial(server, ctx=ctx), ctx.host, ctx.port, ssl=ssl_context)
@@ -279,6 +288,7 @@ async def start_room(room_id):
                     ctx.auto_shutdown = Room.get(id=room_id).timeout
                 if ctx.saving:
                     setattr(asyncio.current_task(), "save", lambda: ctx._save(True))
+                assert ctx.shutdown_task is None
                 ctx.shutdown_task = asyncio.create_task(auto_shutdown(ctx, []))
                 await ctx.shutdown_task
 
@@ -325,7 +335,7 @@ def _done(self, task: asyncio.Future):
         def run(self):
             while 1:
                 next_room = rooms_to_run.get(block=True,  timeout=None)
-                gc.collect(0)
+                gc.collect()
                 task = asyncio.run_coroutine_threadsafe(start_room(next_room), loop)
                 self._tasks.append(task)
                 task.add_done_callback(self._done)