Merge branch 'main' into factorio_update

.github/workflows/unittests.yml

@@ -89,4 +89,4 @@ jobs:
       run: |
         source venv/bin/activate
         export PYTHONPATH=$(pwd)
-        python test/hosting/__main__.py
+        timeout 600 python test/hosting/__main__.py

BaseClasses.py

@@ -1264,6 +1264,10 @@ def useful(self) -> bool:
     def trap(self) -> bool:
         return ItemClassification.trap in self.classification
 
+    @property
+    def excludable(self) -> bool:
+        return not (self.advancement or self.useful)
+
     @property
     def flags(self) -> int:
         return self.classification.as_flag()

Generate.py

@@ -110,7 +110,7 @@ def main(args=None) -> Tuple[argparse.Namespace, int]:
     player_files = {}
     for file in os.scandir(args.player_files_path):
         fname = file.name
-        if file.is_file() and not fname.startswith(".") and \
+        if file.is_file() and not fname.startswith(".") and not fname.lower().endswith(".ini") and \
                 os.path.join(args.player_files_path, fname) not in {args.meta_file_path, args.weights_file_path}:
             path = os.path.join(args.player_files_path, fname)
             try:

Launcher.py

@@ -104,6 +104,7 @@ def update_settings():
     Component("Open host.yaml", func=open_host_yaml),
     Component("Open Patch", func=open_patch),
     Component("Generate Template Options", func=generate_yamls),
+    Component("Archipelago Website", func=lambda: webbrowser.open("https://archipelago.gg/")),
     Component("Discord Server", icon="discord", func=lambda: webbrowser.open("https://discord.gg/8Z65BR2")),
     Component("Unrated/18+ Discord Server", icon="discord", func=lambda: webbrowser.open("https://discord.gg/fqvNCCRsu4")),
     Component("Browse Files", func=browse_files),
@@ -254,7 +255,7 @@ class Launcher(App):
         _client_layout: Optional[ScrollBox] = None
 
         def __init__(self, ctx=None):
-            self.title = self.base_title
+            self.title = self.base_title + " " + Utils.__version__
             self.ctx = ctx
             self.icon = r"data/icon.png"
             super().__init__()

MultiServer.py

@@ -1960,8 +1960,10 @@ def _cmd_status(self, tag: str = "") -> bool:
 
     def _cmd_exit(self) -> bool:
         """Shutdown the server"""
-        self.ctx.server.ws_server.close()
-        self.ctx.exit_event.set()
+        try:
+            self.ctx.server.ws_server.close()
+        finally:
+            self.ctx.exit_event.set()
         return True
 
     @mark_raw

SNIClient.py

@@ -633,7 +633,13 @@ async def game_watcher(ctx: SNIContext) -> None:
         if not ctx.client_handler:
             continue
 
-        rom_validated = await ctx.client_handler.validate_rom(ctx)
+        try:
+            rom_validated = await ctx.client_handler.validate_rom(ctx)
+        except Exception as e:
+            snes_logger.error(f"An error occurred, see logs for details: {e}")
+            text_file_logger = logging.getLogger()
+            text_file_logger.exception(e)
+            rom_validated = False
 
         if not rom_validated or (ctx.auth and ctx.auth != ctx.rom):
             snes_logger.warning("ROM change detected, please reconnect to the multiworld server")
@@ -649,7 +655,13 @@ async def game_watcher(ctx: SNIContext) -> None:
 
         perf_counter = time.perf_counter()
 
-        await ctx.client_handler.game_watcher(ctx)
+        try:
+            await ctx.client_handler.game_watcher(ctx)
+        except Exception as e:
+            snes_logger.error(f"An error occurred, see logs for details: {e}")
+            text_file_logger = logging.getLogger()
+            text_file_logger.exception(e)
+            await snes_disconnect(ctx)
 
 
 async def run_game(romfile: str) -> None:

Utils.py

@@ -31,6 +31,7 @@
     import tkinter
     import pathlib
     from BaseClasses import Region
+    import multiprocessing
 
 
 def tuplize_version(version: str) -> Version:
@@ -664,6 +665,19 @@ def get_input_text_from_response(text: str, command: str) -> typing.Optional[str
     return None
 
 
+def is_kivy_running() -> bool:
+    if "kivy" in sys.modules:
+        from kivy.app import App
+        return App.get_running_app() is not None
+    return False
+
+
+def _mp_open_filename(res: "multiprocessing.Queue[typing.Optional[str]]", *args: Any) -> None:
+    if is_kivy_running():
+        raise RuntimeError("kivy should not be running in multiprocess")
+    res.put(open_filename(*args))
+
+
 def open_filename(title: str, filetypes: typing.Iterable[typing.Tuple[str, typing.Iterable[str]]], suggest: str = "") \
         -> typing.Optional[str]:
     logging.info(f"Opening file input dialog for {title}.")
@@ -693,6 +707,13 @@ def run(*args: str):
                       f'This attempt was made because open_filename was used for "{title}".')
         raise e
     else:
+        if is_macos and is_kivy_running():
+            # on macOS, mixing kivy and tk does not work, so spawn a new process
+            # FIXME: performance of this is pretty bad, and we should (also) look into alternatives
+            from multiprocessing import Process, Queue
+            res: "Queue[typing.Optional[str]]" = Queue()
+            Process(target=_mp_open_filename, args=(res, title, filetypes, suggest)).start()
+            return res.get()
         try:
             root = tkinter.Tk()
         except tkinter.TclError:
@@ -702,6 +723,12 @@ def run(*args: str):
                                                   initialfile=suggest or None)
 
 
+def _mp_open_directory(res: "multiprocessing.Queue[typing.Optional[str]]", *args: Any) -> None:
+    if is_kivy_running():
+        raise RuntimeError("kivy should not be running in multiprocess")
+    res.put(open_directory(*args))
+
+
 def open_directory(title: str, suggest: str = "") -> typing.Optional[str]:
     def run(*args: str):
         return subprocess.run(args, capture_output=True, text=True).stdout.split("\n", 1)[0] or None
@@ -725,9 +752,16 @@ def run(*args: str):
         import tkinter.filedialog
     except Exception as e:
         logging.error('Could not load tkinter, which is likely not installed. '
-                      f'This attempt was made because open_filename was used for "{title}".')
+                      f'This attempt was made because open_directory was used for "{title}".')
         raise e
     else:
+        if is_macos and is_kivy_running():
+            # on macOS, mixing kivy and tk does not work, so spawn a new process
+            # FIXME: performance of this is pretty bad, and we should (also) look into alternatives
+            from multiprocessing import Process, Queue
+            res: "Queue[typing.Optional[str]]" = Queue()
+            Process(target=_mp_open_directory, args=(res, title, suggest)).start()
+            return res.get()
         try:
             root = tkinter.Tk()
         except tkinter.TclError:
@@ -740,12 +774,6 @@ def messagebox(title: str, text: str, error: bool = False) -> None:
     def run(*args: str):
         return subprocess.run(args, capture_output=True, text=True).stdout.split("\n", 1)[0] or None
 
-    def is_kivy_running():
-        if "kivy" in sys.modules:
-            from kivy.app import App
-            return App.get_running_app() is not None
-        return False
-
     if is_kivy_running():
         from kvui import MessageBox
         MessageBox(title, text, error).open()

WebHostLib/requirements.txt

@@ -1,5 +1,5 @@
 flask>=3.0.3
-werkzeug>=3.0.4
+werkzeug>=3.0.6
 pony>=0.7.19
 waitress>=3.0.0
 Flask-Caching>=2.3.0

WebHostLib/tracker.py

@@ -5,7 +5,7 @@
 from uuid import UUID
 from email.utils import parsedate_to_datetime
 
-from flask import render_template, make_response, Response, request
+from flask import make_response, render_template, request, Request, Response
 from werkzeug.exceptions import abort
 
 from MultiServer import Context, get_saving_second
@@ -298,17 +298,25 @@ def get_spheres(self) -> List[List[int]]:
         return self._multidata.get("spheres", [])
 
 
-def _process_if_request_valid(incoming_request, room: Optional[Room]) -> Optional[Response]:
+def _process_if_request_valid(incoming_request: Request, room: Optional[Room]) -> Optional[Response]:
     if not room:
         abort(404)
 
-    if_modified = incoming_request.headers.get("If-Modified-Since", None)
-    if if_modified:
-        if_modified = parsedate_to_datetime(if_modified)
+    if_modified_str: Optional[str] = incoming_request.headers.get("If-Modified-Since", None)
+    if if_modified_str:
+        if_modified = parsedate_to_datetime(if_modified_str)
+        if if_modified.tzinfo is None:
+            abort(400)  # standard requires "GMT" timezone
+        # database may use datetime.utcnow(), which is timezone-naive. convert to timezone-aware.
+        last_activity = room.last_activity
+        if last_activity.tzinfo is None:
+            last_activity = room.last_activity.replace(tzinfo=datetime.timezone.utc)
         # if_modified has less precision than last_activity, so we bring them to same precision
-        if if_modified >= room.last_activity.replace(microsecond=0):
+        if if_modified >= last_activity.replace(microsecond=0):
             return make_response("",  304)
 
+    return None
+
 
 @app.route("/tracker/<suuid:tracker>/<int:tracked_team>/<int:tracked_player>")
 def get_player_tracker(tracker: UUID, tracked_team: int, tracked_player: int, generic: bool = False) -> Response:

docs/running from source.md

@@ -85,4 +85,4 @@ PyCharm has a built-in version control integration that supports Git.
 
 ## Running tests
 
-Run `pip install pytest pytest-subtests`, then use your IDE to run tests or run `pytest` from the source folder.
+Information about running tests can be found in [tests.md](https://github.com/ArchipelagoMW/Archipelago/blob/main/docs/tests.md#running-tests)

docs/tests.md

@@ -84,7 +84,19 @@ testing portions of your code that can be tested without relying on a multiworld
 
 ## Running Tests
 
-In PyCharm, running all tests can be done by right-clicking the root `test` directory and selecting `run Python tests`.
-If you do not have pytest installed, you may get import failures. To solve this, edit the run configuration, and set the
-working directory of the run to the Archipelago directory. If you only want to run your world's defined tests, repeat
-the steps for the test directory within your world.
+#### Using Pycharm
+
+In PyCharm, running all tests can be done by right-clicking the root test directory and selecting Run 'Archipelago Unittests'. 
+Unless you configured PyCharm to use pytest as a test runner, you may get import failures. To solve this, edit the run configuration, 
+and set the working directory to the Archipelago directory which contains all the project files. 
+
+If you only want to run your world's defined tests, repeat the steps for the test directory within your world.
+Your working directory should be the directory of your world in the worlds directory and the script should be the 
+tests folder within your world.
+
+You can also find the 'Archipelago Unittests' as an option in the dropdown at the top of the window
+next to the run and debug buttons.
+
+#### Running Tests without Pycharm
+
+Run `pip install pytest pytest-subtests`, then use your IDE to run tests or run `pytest` from the source folder.

requirements.txt

@@ -1,5 +1,5 @@
 colorama>=0.4.6
-websockets>=13.0.1
+websockets>=13.0.1,<14
 PyYAML>=6.0.2
 jellyfish>=1.1.0
 jinja2>=3.1.4

setup.py

@@ -632,7 +632,7 @@ def find_lib(lib: str, arch: str, libc: str) -> Optional[str]:
             "packages": ["worlds", "kivy", "cymem", "websockets"],
             "includes": [],
             "excludes": ["numpy", "Cython", "PySide2", "PIL",
-                         "pandas"],
+                         "pandas", "zstandard"],
             "zip_include_packages": ["*"],
             "zip_exclude_packages": ["worlds", "sc2", "orjson"],  # TODO: remove orjson here once we drop py3.8 support
             "include_files": [],  # broken in cx 6.14.0, we use more special sauce now

test/general/test_fill.py

@@ -688,8 +688,8 @@ def test_non_excluded_local_items(self):
         for item in multiworld.get_items():
             item.classification = ItemClassification.useful
 
-        multiworld.local_items[player1.id].value = set(names(player1.basic_items))
-        multiworld.local_items[player2.id].value = set(names(player2.basic_items))
+        multiworld.worlds[player1.id].options.local_items.value = set(names(player1.basic_items))
+        multiworld.worlds[player2.id].options.local_items.value = set(names(player2.basic_items))
         locality_rules(multiworld)
 
         distribute_items_restrictive(multiworld)
@@ -795,8 +795,8 @@ def setUp(self) -> None:
 
     def test_balances_progression(self) -> None:
         """Tests that progression balancing moves progression items earlier"""
-        self.multiworld.progression_balancing[self.player1.id].value = 50
-        self.multiworld.progression_balancing[self.player2.id].value = 50
+        self.multiworld.worlds[self.player1.id].options.progression_balancing.value = 50
+        self.multiworld.worlds[self.player2.id].options.progression_balancing.value = 50
 
         self.assertRegionContains(
             self.player1.regions[2], self.player2.prog_items[0])
@@ -808,8 +808,8 @@ def test_balances_progression(self) -> None:
 
     def test_balances_progression_light(self) -> None:
         """Test that progression balancing still moves items earlier on minimum value"""
-        self.multiworld.progression_balancing[self.player1.id].value = 1
-        self.multiworld.progression_balancing[self.player2.id].value = 1
+        self.multiworld.worlds[self.player1.id].options.progression_balancing.value = 1
+        self.multiworld.worlds[self.player2.id].options.progression_balancing.value = 1
 
         self.assertRegionContains(
             self.player1.regions[2], self.player2.prog_items[0])
@@ -822,8 +822,8 @@ def test_balances_progression_light(self) -> None:
 
     def test_balances_progression_heavy(self) -> None:
         """Test that progression balancing moves items earlier on maximum value"""
-        self.multiworld.progression_balancing[self.player1.id].value = 99
-        self.multiworld.progression_balancing[self.player2.id].value = 99
+        self.multiworld.worlds[self.player1.id].options.progression_balancing.value = 99
+        self.multiworld.worlds[self.player2.id].options.progression_balancing.value = 99
 
         self.assertRegionContains(
             self.player1.regions[2], self.player2.prog_items[0])
@@ -836,8 +836,8 @@ def test_balances_progression_heavy(self) -> None:
 
     def test_skips_balancing_progression(self) -> None:
         """Test that progression balancing is skipped when players have it disabled"""
-        self.multiworld.progression_balancing[self.player1.id].value = 0
-        self.multiworld.progression_balancing[self.player2.id].value = 0
+        self.multiworld.worlds[self.player1.id].options.progression_balancing.value = 0
+        self.multiworld.worlds[self.player2.id].options.progression_balancing.value = 0
 
         self.assertRegionContains(
             self.player1.regions[2], self.player2.prog_items[0])
@@ -849,8 +849,8 @@ def test_skips_balancing_progression(self) -> None:
 
     def test_ignores_priority_locations(self) -> None:
         """Test that progression items on priority locations don't get moved by balancing"""
-        self.multiworld.progression_balancing[self.player1.id].value = 50
-        self.multiworld.progression_balancing[self.player2.id].value = 50
+        self.multiworld.worlds[self.player1.id].options.progression_balancing.value = 50
+        self.multiworld.worlds[self.player2.id].options.progression_balancing.value = 50
 
         self.player2.prog_items[0].location.progress_type = LocationProgressType.PRIORITY
 

test/general/test_options.py

@@ -21,6 +21,17 @@ def test_options_are_not_set_by_world(self):
                 self.assertFalse(hasattr(world_type, "options"),
                                  f"Unexpected assignment to {world_type.__name__}.options!")
 
+    def test_duplicate_options(self) -> None:
+        """Tests that a world doesn't reuse the same option class."""
+        for game_name, world_type in AutoWorldRegister.world_types.items():
+            with self.subTest(game=game_name):
+                seen_options = set()
+                for option in world_type.options_dataclass.type_hints.values():
+                    if not option.visibility:
+                        continue
+                    self.assertFalse(option in seen_options, f"{option} found in assigned options multiple times.")
+                    seen_options.add(option)
+
     def test_item_links_name_groups(self):
         """Tests that item links successfully unfold item_name_groups"""
         item_link_groups = [

test/programs/data/weights/weights.yaml

@@ -0,0 +1,10 @@
+name: Player{number}
+game: Archipelago # we only need to test options work and this "supports" all the base options
+Archipelago:
+  progression_balancing:
+    0: 50
+    50: 50
+    99: 50
+  accessibility:
+    0: 50
+    2: 50