From 6fd16ecced785b4b14b97e8ae9e952335902858e Mon Sep 17 00:00:00 2001 From: black-sliver <59490463+black-sliver@users.noreply.github.com> Date: Wed, 5 Jul 2023 10:35:03 +0200 Subject: [PATCH 01/24] MultiServer: Allow games with no locations, add checks to pure python implementation. (#1944) * Server: allow games with no locations again * Server: validate locations in pure python implementation and rework tests * Server: fix tests for py<3.11 --- NetUtils.py | 12 ++ _speedups.pyx | 14 ++- test/netutils/TestLocationStore.py | 190 ++++++++++++++++------------- 3 files changed, 124 insertions(+), 92 deletions(-) diff --git a/NetUtils.py b/NetUtils.py index b30316ca6d7b..99c37238c35a 100644 --- a/NetUtils.py +++ b/NetUtils.py @@ -347,6 +347,18 @@ def local(self): class _LocationStore(dict, typing.MutableMapping[int, typing.Dict[int, typing.Tuple[int, int, int]]]): + def __init__(self, values: typing.MutableMapping[int, typing.Dict[int, typing.Tuple[int, int, int]]]): + super().__init__(values) + + if not self: + raise ValueError(f"Rejecting game with 0 players") + + if len(self) != max(self): + raise ValueError("Player IDs not continuous") + + if len(self.get(0, {})): + raise ValueError("Invalid player id 0 for location") + def find_item(self, slots: typing.Set[int], seeked_item_id: int ) -> typing.Generator[typing.Tuple[int, int, int, int, int], None, None]: for finding_player, check_data in self.items(): diff --git a/_speedups.pyx b/_speedups.pyx index 95e837d1bba6..fc2413ceb5d1 100644 --- a/_speedups.pyx +++ b/_speedups.pyx @@ -8,6 +8,7 @@ This is deliberately .pyx because using a non-compiled "pure python" may be slow # pip install cython cymem import cython +import warnings from cpython cimport PyObject from typing import Any, Dict, Iterable, Iterator, Generator, Sequence, Tuple, TypeVar, Union, Set, List, TYPE_CHECKING from cymem.cymem cimport Pool @@ -107,13 +108,16 @@ cdef class LocationStore: count += 1 sender_count += 1 - if not count: - raise ValueError("No locations") + if not sender_count: + raise ValueError(f"Rejecting game with 0 players") if sender_count != max_sender: # we assume player 0 will never have locations raise ValueError("Player IDs not continuous") + if not count: + warnings.warn("Game has no locations") + # allocate the arrays and invalidate index (0xff...) self.entries = self._mem.alloc(count, sizeof(LocationEntry)) self.sender_index = self._mem.alloc(max_sender + 1, sizeof(IndexEntry)) @@ -140,9 +144,9 @@ cdef class LocationStore: self._proxies.append(None) # player 0 assert self.sender_index[0].count == 0 for i in range(1, max_sender + 1): - if self.sender_index[i].count == 0 and self.sender_index[i].start >= count: - self.sender_index[i].start = 0 # do not point outside valid entries - assert self.sender_index[i].start < count + assert self.sender_index[i].count == 0 or ( + self.sender_index[i].start < count and + self.sender_index[i].start + self.sender_index[i].count <= count) key = i # allocate python integer proxy = PlayerLocationProxy(self, i) self._keys.append(key) diff --git a/test/netutils/TestLocationStore.py b/test/netutils/TestLocationStore.py index 5c98437a031e..9fe904f68a16 100644 --- a/test/netutils/TestLocationStore.py +++ b/test/netutils/TestLocationStore.py @@ -1,10 +1,13 @@ # Tests for _speedups.LocationStore and NetUtils._LocationStore import typing import unittest +import warnings from NetUtils import LocationStore, _LocationStore +State = typing.Dict[typing.Tuple[int, int], typing.Set[int]] +RawLocations = typing.Dict[int, typing.Dict[int, typing.Tuple[int, int, int]]] -sample_data = { +sample_data: RawLocations = { 1: { 11: (21, 2, 7), 12: (22, 2, 0), @@ -23,28 +26,29 @@ }, } -empty_state = { +empty_state: State = { (0, slot): set() for slot in sample_data } -full_state = { +full_state: State = { (0, slot): set(locations) for (slot, locations) in sample_data.items() } -one_state = { +one_state: State = { (0, 1): {12} } class Base: class TestLocationStore(unittest.TestCase): + """Test method calls on a loaded store.""" store: typing.Union[LocationStore, _LocationStore] - def test_len(self): + def test_len(self) -> None: self.assertEqual(len(self.store), 4) self.assertEqual(len(self.store[1]), 3) - def test_key_error(self): + def test_key_error(self) -> None: with self.assertRaises(KeyError): _ = self.store[0] with self.assertRaises(KeyError): @@ -54,25 +58,25 @@ def test_key_error(self): _ = locations[7] _ = locations[11] # no Exception - def test_getitem(self): + def test_getitem(self) -> None: self.assertEqual(self.store[1][11], (21, 2, 7)) self.assertEqual(self.store[1][13], (13, 1, 0)) self.assertEqual(self.store[2][22], (12, 1, 0)) self.assertEqual(self.store[4][9], (99, 3, 0)) - def test_get(self): + def test_get(self) -> None: self.assertEqual(self.store.get(1, None), self.store[1]) self.assertEqual(self.store.get(0, None), None) self.assertEqual(self.store[1].get(11, (None, None, None)), self.store[1][11]) self.assertEqual(self.store[1].get(10, (None, None, None)), (None, None, None)) - def test_iter(self): + def test_iter(self) -> None: self.assertEqual(sorted(self.store), [1, 2, 3, 4]) self.assertEqual(len(self.store), len(sample_data)) self.assertEqual(list(self.store[1]), [11, 12, 13]) self.assertEqual(len(self.store[1]), len(sample_data[1])) - def test_items(self): + def test_items(self) -> None: self.assertEqual(sorted(p for p, _ in self.store.items()), sorted(self.store)) self.assertEqual(sorted(p for p, _ in self.store[1].items()), sorted(self.store[1])) self.assertEqual(sorted(self.store.items())[0][0], 1) @@ -80,7 +84,7 @@ def test_items(self): self.assertEqual(sorted(self.store[1].items())[0][0], 11) self.assertEqual(sorted(self.store[1].items())[0][1], self.store[1][11]) - def test_find_item(self): + def test_find_item(self) -> None: self.assertEqual(sorted(self.store.find_item(set(), 99)), []) self.assertEqual(sorted(self.store.find_item({3}, 1)), []) self.assertEqual(sorted(self.store.find_item({5}, 99)), []) @@ -89,129 +93,141 @@ def test_find_item(self): self.assertEqual(sorted(self.store.find_item({3, 4}, 99)), [(3, 9, 99, 4, 0), (4, 9, 99, 3, 0)]) - def test_get_for_player(self): + def test_get_for_player(self) -> None: self.assertEqual(self.store.get_for_player(3), {4: {9}}) self.assertEqual(self.store.get_for_player(1), {1: {13}, 2: {22, 23}}) - def get_checked(self): + def get_checked(self) -> None: self.assertEqual(self.store.get_checked(full_state, 0, 1), [11, 12, 13]) self.assertEqual(self.store.get_checked(one_state, 0, 1), [12]) self.assertEqual(self.store.get_checked(empty_state, 0, 1), []) self.assertEqual(self.store.get_checked(full_state, 0, 3), [9]) - def get_missing(self): + def get_missing(self) -> None: self.assertEqual(self.store.get_missing(full_state, 0, 1), []) self.assertEqual(self.store.get_missing(one_state, 0, 1), [11, 13]) self.assertEqual(self.store.get_missing(empty_state, 0, 1), [11, 12, 13]) self.assertEqual(self.store.get_missing(empty_state, 0, 3), [9]) - def get_remaining(self): + def get_remaining(self) -> None: self.assertEqual(self.store.get_remaining(full_state, 0, 1), []) self.assertEqual(self.store.get_remaining(one_state, 0, 1), [13, 21]) self.assertEqual(self.store.get_remaining(empty_state, 0, 1), [13, 21, 22]) self.assertEqual(self.store.get_remaining(empty_state, 0, 3), [99]) + class TestLocationStoreConstructor(unittest.TestCase): + """Test constructors for a given store type.""" + type: type + + def test_hole(self) -> None: + with self.assertRaises(Exception): + self.type({ + 1: {1: (1, 1, 1)}, + 3: {1: (1, 1, 1)}, + }) + + def test_no_slot1(self) -> None: + with self.assertRaises(Exception): + self.type({ + 2: {1: (1, 1, 1)}, + 3: {1: (1, 1, 1)}, + }) + + def test_slot0(self) -> None: + with self.assertRaises(ValueError): + self.type({ + 0: {1: (1, 1, 1)}, + 1: {1: (1, 1, 1)}, + }) + with self.assertRaises(ValueError): + self.type({ + 0: {1: (1, 1, 1)}, + 2: {1: (1, 1, 1)}, + }) + + def test_no_players(self) -> None: + with self.assertRaises(Exception): + _ = self.type({}) + + def test_no_locations(self) -> None: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + store = self.type({ + 1: {}, + }) + self.assertEqual(len(store), 1) + self.assertEqual(len(store[1]), 0) + + def test_no_locations_for_1(self) -> None: + store = self.type({ + 1: {}, + 2: {1: (1, 2, 3)}, + }) + self.assertEqual(len(store), 2) + self.assertEqual(len(store[1]), 0) + self.assertEqual(len(store[2]), 1) + + def test_no_locations_for_last(self) -> None: + store = self.type({ + 1: {1: (1, 2, 3)}, + 2: {}, + }) + self.assertEqual(len(store), 2) + self.assertEqual(len(store[1]), 1) + self.assertEqual(len(store[2]), 0) + class TestPurePythonLocationStore(Base.TestLocationStore): + """Run base method tests for pure python implementation.""" def setUp(self) -> None: self.store = _LocationStore(sample_data) super().setUp() +class TestPurePythonLocationStoreConstructor(Base.TestLocationStoreConstructor): + """Run base constructor tests for the pure python implementation.""" + def setUp(self) -> None: + self.type = _LocationStore + super().setUp() + + @unittest.skipIf(LocationStore is _LocationStore, "_speedups not available") class TestSpeedupsLocationStore(Base.TestLocationStore): + """Run base method tests for cython implementation.""" def setUp(self) -> None: self.store = LocationStore(sample_data) super().setUp() @unittest.skipIf(LocationStore is _LocationStore, "_speedups not available") -class TestSpeedupsLocationStoreConstructor(unittest.TestCase): - def test_float_key(self): +class TestSpeedupsLocationStoreConstructor(Base.TestLocationStoreConstructor): + """Run base constructor tests and tests the additional constraints for cython implementation.""" + def setUp(self) -> None: + self.type = LocationStore + super().setUp() + + def test_float_key(self) -> None: with self.assertRaises(Exception): - LocationStore({ + self.type({ 1: {1: (1, 1, 1)}, 1.1: {1: (1, 1, 1)}, 3: {1: (1, 1, 1)} }) - def test_string_key(self): + def test_string_key(self) -> None: with self.assertRaises(Exception): - LocationStore({ + self.type({ "1": {1: (1, 1, 1)}, }) - def test_hole(self): + def test_high_player_number(self) -> None: with self.assertRaises(Exception): - LocationStore({ - 1: {1: (1, 1, 1)}, - 3: {1: (1, 1, 1)}, - }) - - def test_no_slot1(self): - with self.assertRaises(Exception): - LocationStore({ - 2: {1: (1, 1, 1)}, - 3: {1: (1, 1, 1)}, - }) - - def test_slot0(self): - with self.assertRaises(Exception): - LocationStore({ - 0: {1: (1, 1, 1)}, - 1: {1: (1, 1, 1)}, - }) - with self.assertRaises(Exception): - LocationStore({ - 0: {1: (1, 1, 1)}, - 2: {1: (1, 1, 1)}, - }) - - def test_high_player_number(self): - with self.assertRaises(Exception): - LocationStore({ + self.type({ 1 << 32: {1: (1, 1, 1)}, }) - def test_no_players(self): - try: # either is fine: raise during init, or behave like {} - store = LocationStore({}) - self.assertEqual(len(store), 0) - with self.assertRaises(KeyError): - _ = store[1] - except ValueError: - pass - - def test_no_locations(self): - try: # either is fine: raise during init, or behave like {1: {}} - store = LocationStore({ - 1: {}, - }) - self.assertEqual(len(store), 1) - self.assertEqual(len(store[1]), 0) - except ValueError: - pass - - def test_no_locations_for_1(self): - store = LocationStore({ - 1: {}, - 2: {1: (1, 2, 3)}, - }) - self.assertEqual(len(store), 2) - self.assertEqual(len(store[1]), 0) - self.assertEqual(len(store[2]), 1) - - def test_no_locations_for_last(self): - store = LocationStore({ - 1: {1: (1, 2, 3)}, - 2: {}, - }) - self.assertEqual(len(store), 2) - self.assertEqual(len(store[1]), 1) - self.assertEqual(len(store[2]), 0) - - def test_not_a_tuple(self): + def test_not_a_tuple(self) -> None: with self.assertRaises(Exception): - LocationStore({ + self.type({ 1: {1: None}, }) From e920692ec3b76471088787ddcc0a67d5fc45b6bb Mon Sep 17 00:00:00 2001 From: Samuel Thayer Date: Wed, 5 Jul 2023 13:21:32 -0500 Subject: [PATCH 02/24] DS3, Docs: Add downpatching instructions to Dark Souls III setup guide (#1874) * add links to downpatching instructions * renumber properly * Update setup_en.md * Update setup_en.md * DS3, Docs: Avoid having to update the guide for steam updates --------- Co-authored-by: black-sliver <59490463+black-sliver@users.noreply.github.com> --- worlds/dark_souls_3/docs/setup_en.md | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/worlds/dark_souls_3/docs/setup_en.md b/worlds/dark_souls_3/docs/setup_en.md index 8e1af8e92d90..d9dbb2e54729 100644 --- a/worlds/dark_souls_3/docs/setup_en.md +++ b/worlds/dark_souls_3/docs/setup_en.md @@ -11,27 +11,32 @@ ## General Concept + +**This mod can ban you permanently from the FromSoftware servers if used online.** + The Dark Souls III AP Client is a dinput8.dll triggered when launching Dark Souls III. This .dll file will launch a command prompt where you can read information about your run and write any command to interact with the Archipelago server. -## Installation Procedures +This client has only been tested with the Official Steam version of the game at version 1.15. It does not matter which DLCs are installed. However, you will have to downpatch your Dark Souls III installation from current patch. - -**This mod can ban you permanently from the FromSoftware servers if used online.** - -This client has only been tested with the Official Steam version of the game (v1.15/1.35) not matter which DLCs are installed. +## Downpatching Dark Souls III + +Follow instructions from the [speedsouls wiki](https://wiki.speedsouls.com/darksouls3:Downpatching) to download version 1.15. Your download command, including the correct depot and manifest ids, will be "download_depot 374320 374321 4471176929659548333" + +## Installing the Archipelago mod Get the dinput8.dll from the [Dark Souls III AP Client](https://github.com/Marechal-L/Dark-Souls-III-Archipelago-client/releases) and add it at the root folder of your game (e.g. "SteamLibrary\steamapps\common\DARK SOULS III\Game") ## Joining a MultiWorld Game -1. Run DarkSoulsIII.exe or run the game through Steam -2. Type in "/connect {SERVER_IP}:{SERVER_PORT} {SLOT_NAME}" in the "Windows Command Prompt" that opened -3. Once connected, create a new game, choose a class and wait for the others before starting -4. You can quit and launch at anytime during a game +1. Run Steam in offline mode, both to avoid being banned and to prevent Steam from updating the game files +2. Launch Dark Souls III +3. Type in "/connect {SERVER_IP}:{SERVER_PORT} {SLOT_NAME}" in the "Windows Command Prompt" that opened +4. Once connected, create a new game, choose a class and wait for the others before starting +5. You can quit and launch at anytime during a game ## Where do I get a config file? The [Player Settings](/games/Dark%20Souls%20III/player-settings) page on the website allows you to -configure your personal settings and export them into a config file +configure your personal settings and export them into a config file. From d8a89976844d5fe3b0c45b4938fab978a4e8471d Mon Sep 17 00:00:00 2001 From: Fabian Dill Date: Wed, 5 Jul 2023 21:51:38 +0200 Subject: [PATCH 03/24] Core: remove "names" from multidata (#1928) --- Main.py | 1 - WebHostLib/tracker.py | 28 +++++++++++++++------------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/Main.py b/Main.py index 9804e6bd4582..63173601cb96 100644 --- a/Main.py +++ b/Main.py @@ -368,7 +368,6 @@ def precollect_hint(location): multidata = { "slot_data": slot_data, "slot_info": slot_info, - "names": names, # TODO: remove after 0.3.9 "connect_names": {name: (0, player) for player, name in world.player_name.items()}, "locations": locations_data, "checks_in_area": checks_in_area, diff --git a/WebHostLib/tracker.py b/WebHostLib/tracker.py index decb3dd8ceb7..987a96d02da8 100644 --- a/WebHostLib/tracker.py +++ b/WebHostLib/tracker.py @@ -1,7 +1,7 @@ import collections import datetime import typing -from typing import Counter, Optional, Dict, Any, Tuple +from typing import Counter, Optional, Dict, Any, Tuple, List from uuid import UUID from flask import render_template @@ -9,7 +9,7 @@ from werkzeug.exceptions import abort from MultiServer import Context, get_saving_second -from NetUtils import SlotType +from NetUtils import SlotType, NetworkSlot from Utils import restricted_loads from worlds import lookup_any_item_id_to_name, lookup_any_location_id_to_name, network_data_package from worlds.alttp import Items @@ -264,16 +264,17 @@ def get_static_room_data(room: Room): multidata = Context.decompress(room.seed.multidata) # in > 100 players this can take a bit of time and is the main reason for the cache locations: Dict[int, Dict[int, Tuple[int, int, int]]] = multidata['locations'] - names: Dict[int, Dict[int, str]] = multidata["names"] - games = {} + names: List[List[str]] = multidata.get("names", []) + games = multidata.get("games", {}) groups = {} custom_locations = {} custom_items = {} if "slot_info" in multidata: - games = {slot: slot_info.game for slot, slot_info in multidata["slot_info"].items()} - groups = {slot: slot_info.group_members for slot, slot_info in multidata["slot_info"].items() + slot_info_dict: Dict[int, NetworkSlot] = multidata["slot_info"] + games = {slot: slot_info.game for slot, slot_info in slot_info_dict.items()} + groups = {slot: slot_info.group_members for slot, slot_info in slot_info_dict.items() if slot_info.type == SlotType.group} - + names = [[slot_info.name for slot, slot_info in sorted(slot_info_dict.items())]] for game in games.values(): if game not in multidata["datapackage"]: continue @@ -290,8 +291,7 @@ def get_static_room_data(room: Room): {id_: name for name, id_ in game_data["location_name_to_id"].items()}) custom_items.update( {id_: name for name, id_ in game_data["item_name_to_id"].items()}) - elif "games" in multidata: - games = multidata["games"] + seed_checks_in_area = checks_in_area.copy() use_door_tracker = False @@ -341,7 +341,7 @@ def _get_player_tracker(tracker: UUID, tracked_team: int, tracked_player: int, w precollected_items, games, slot_data, groups, saving_second, custom_locations, custom_items = \ get_static_room_data(room) player_name = names[tracked_team][tracked_player - 1] - location_to_area = player_location_to_area[tracked_player] + location_to_area = player_location_to_area.get(tracked_player, {}) inventory = collections.Counter() checks_done = {loc_name: 0 for loc_name in default_locations} @@ -373,7 +373,9 @@ def _get_player_tracker(tracker: UUID, tracked_team: int, tracked_player: int, w if recipient in slots_aimed_at_player: # a check done for the tracked player attribute_item_solo(inventory, item) if ms_player == tracked_player: # a check done by the tracked player - checks_done[location_to_area[location]] += 1 + area_name = location_to_area.get(location, None) + if area_name: + checks_done[area_name] += 1 checks_done["Total"] += 1 specific_tracker = game_specific_trackers.get(games[tracked_player], None) if specific_tracker and not want_generic: @@ -1508,8 +1510,8 @@ def attribute_item(team: int, recipient: int, item: int): checks_done[team][player][player_location_to_area[player][location]] += 1 checks_done[team][player]["Total"] += 1 percent_total_checks_done[team][player] = int( - checks_done[team][player]["Total"] / seed_checks_in_area[player]["Total"] * 100) if \ - seed_checks_in_area[player]["Total"] else 100 + checks_done[team][player]["Total"] / len(player_locations) * 100) if \ + player_locations else 100 for (team, player), game_state in multisave.get("client_game_state", {}).items(): if player in groups: From 827444f5a4e065bc310c889892580eb1f97c73cb Mon Sep 17 00:00:00 2001 From: black-sliver <59490463+black-sliver@users.noreply.github.com> Date: Wed, 5 Jul 2023 22:39:35 +0200 Subject: [PATCH 04/24] Core: Add settings API ("auto settings") for host.yaml (#1871) * Add settings API ("auto settings") for host.yaml * settings: no BOM when saving * settings: fix saving / groups resetting themselves * settings: fix AutoWorldRegister import Co-authored-by: el-u <109771707+el-u@users.noreply.github.com> * Lufia2: settings: clean up imports * settings: more consistent class naming * Docs: update world api for settings api refactor * settings: fix access from World instance * settings: update migration timeline * Docs: Apply suggestions from code review Co-authored-by: Zach Parks * Settings: correctly resolve .exe in UserPath and LocalPath --------- Co-authored-by: el-u <109771707+el-u@users.noreply.github.com> Co-authored-by: Zach Parks --- .github/workflows/unittests.yml | 1 + .gitignore | 1 + Generate.py | 40 +- Launcher.py | 23 +- Main.py | 7 +- MinecraftClient.py | 2 +- OoTClient.py | 2 - Utils.py | 208 +++------ WebHost.py | 2 + docs/settings api.md | 187 ++++++++ docs/world api.md | 24 +- host.yaml | 190 -------- inno_setup.iss | 1 + settings.py | 772 ++++++++++++++++++++++++++++++++ setup.py | 10 +- test/programs/TestGenerate.py | 24 +- worlds/AutoWorld.py | 26 ++ worlds/adventure/__init__.py | 40 +- worlds/alttp/__init__.py | 26 +- worlds/dkc3/__init__.py | 12 + worlds/factorio/__init__.py | 26 ++ worlds/ff1/__init__.py | 9 + worlds/ladx/__init__.py | 15 + worlds/lufia2ac/__init__.py | 13 + worlds/minecraft/__init__.py | 19 + worlds/mmbn3/__init__.py | 13 + worlds/oot/__init__.py | 25 ++ worlds/pokemon_rb/__init__.py | 33 +- worlds/sm/__init__.py | 18 +- worlds/smw/__init__.py | 12 + worlds/soe/__init__.py | 13 + worlds/tloz/__init__.py | 29 +- worlds/wargroove/__init__.py | 16 +- worlds/zillion/__init__.py | 26 +- 34 files changed, 1454 insertions(+), 411 deletions(-) create mode 100644 docs/settings api.md delete mode 100644 host.yaml create mode 100644 settings.py diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 4358c8032bdd..8ff0f8bb44e1 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -65,6 +65,7 @@ jobs: python -m pip install --upgrade pip pip install pytest pytest-subtests python ModuleUpdate.py --yes --force --append "WebHostLib/requirements.txt" + python Launcher.py --update_settings # make sure host.yaml exists for tests - name: Unittests run: | pytest diff --git a/.gitignore b/.gitignore index 3e242d89af9f..8e4cc86657a5 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,7 @@ README.html EnemizerCLI/ /Players/ /SNI/ +/host.yaml /options.yaml /config.yaml /logs/ diff --git a/Generate.py b/Generate.py index bd265879fa31..bd1c4aa6fdea 100644 --- a/Generate.py +++ b/Generate.py @@ -14,44 +14,42 @@ ModuleUpdate.update() +import copy import Utils +import Options +from BaseClasses import seeddigits, get_seed, PlandoOptions +from Main import main as ERmain +from settings import get_settings +from Utils import parse_yamls, version_tuple, __version__, tuplize_version, user_path from worlds.alttp import Options as LttPOptions -from worlds.generic import PlandoConnection -from Utils import parse_yamls, version_tuple, __version__, tuplize_version, get_options, user_path from worlds.alttp.EntranceRandomizer import parse_arguments -from Main import main as ERmain -from BaseClasses import seeddigits, get_seed, PlandoOptions -import Options from worlds.alttp.Text import TextTable from worlds.AutoWorld import AutoWorldRegister -import copy +from worlds.generic import PlandoConnection def mystery_argparse(): - options = get_options() - defaults = options["generator"] - - def resolve_path(path: str, resolver: Callable[[str], str]) -> str: - return path if os.path.isabs(path) else resolver(path) + options = get_settings() + defaults = options.generator parser = argparse.ArgumentParser(description="CMD Generation Interface, defaults come from host.yaml.") - parser.add_argument('--weights_file_path', default=defaults["weights_file_path"], + parser.add_argument('--weights_file_path', default=defaults.weights_file_path, help='Path to the weights file to use for rolling game settings, urls are also valid') parser.add_argument('--samesettings', help='Rolls settings per weights file rather than per player', action='store_true') - parser.add_argument('--player_files_path', default=resolve_path(defaults["player_files_path"], user_path), + parser.add_argument('--player_files_path', default=defaults.player_files_path, help="Input directory for player files.") parser.add_argument('--seed', help='Define seed number to generate.', type=int) - parser.add_argument('--multi', default=defaults["players"], type=lambda value: max(int(value), 1)) - parser.add_argument('--spoiler', type=int, default=defaults["spoiler"]) - parser.add_argument('--outputpath', default=resolve_path(options["general_options"]["output_path"], user_path), + parser.add_argument('--multi', default=defaults.players, type=lambda value: max(int(value), 1)) + parser.add_argument('--spoiler', type=int, default=defaults.spoiler) + parser.add_argument('--outputpath', default=options.general_options.output_path, help="Path to output folder. Absolute or relative to cwd.") # absolute or relative to cwd - parser.add_argument('--race', action='store_true', default=defaults["race"]) - parser.add_argument('--meta_file_path', default=defaults["meta_file_path"]) + parser.add_argument('--race', action='store_true', default=defaults.race) + parser.add_argument('--meta_file_path', default=defaults.meta_file_path) parser.add_argument('--log_level', default='info', help='Sets log level') parser.add_argument('--yaml_output', default=0, type=lambda value: max(int(value), 0), help='Output rolled mystery results to yaml up to specified number (made for async multiworld)') - parser.add_argument('--plando', default=defaults["plando_options"], + parser.add_argument('--plando', default=defaults.plando_options, help='List of options that can be set manually. Can be combined, for example "bosses, items"') parser.add_argument("--skip_prog_balancing", action="store_true", help="Skip progression balancing step during generation.") @@ -71,6 +69,8 @@ def get_seed_name(random_source) -> str: def main(args=None, callback=ERmain): if not args: args, options = mystery_argparse() + else: + options = get_settings() seed = get_seed(args.seed) Utils.init_logging(f"Generate_{seed}", loglevel=args.log_level) @@ -137,7 +137,7 @@ def main(args=None, callback=ERmain): erargs = parse_arguments(['--multi', str(args.multi)]) erargs.seed = seed erargs.plando_options = args.plando - erargs.glitch_triforce = options["generator"]["glitch_triforce_room"] + erargs.glitch_triforce = options.generator.glitch_triforce_room erargs.spoiler = args.spoiler erargs.race = args.race erargs.outputname = seed_name diff --git a/Launcher.py b/Launcher.py index 84bdeeb72508..a1548d594ce8 100644 --- a/Launcher.py +++ b/Launcher.py @@ -22,6 +22,7 @@ from typing import Sequence, Union, Optional import Utils +import settings from worlds.LauncherComponents import Component, components, Type, SuffixIdentifier, icon_paths if __name__ == "__main__": @@ -33,7 +34,8 @@ def open_host_yaml(): - file = user_path('host.yaml') + file = settings.get_settings().filename + assert file, "host.yaml missing" if is_linux: exe = which('sensible-editor') or which('gedit') or \ which('xdg-open') or which('gnome-open') or which('kde-open') @@ -84,6 +86,11 @@ def open_folder(folder_path): webbrowser.open(folder_path) +def update_settings(): + from settings import get_settings + get_settings().save() + + components.extend([ # Functions Component("Open host.yaml", func=open_host_yaml), @@ -256,11 +263,13 @@ def main(args: Optional[Union[argparse.Namespace, dict]] = None): if not component: logging.warning(f"Could not identify Component responsible for {args['Patch|Game|Component']}") + if args["update_settings"]: + update_settings() if 'file' in args: run_component(args["component"], args["file"], *args["args"]) elif 'component' in args: run_component(args["component"], *args["args"]) - else: + elif not args["update_settings"]: run_gui() @@ -269,9 +278,13 @@ def main(args: Optional[Union[argparse.Namespace, dict]] = None): Utils.freeze_support() multiprocessing.set_start_method("spawn") # if launched process uses kivy, fork won't work parser = argparse.ArgumentParser(description='Archipelago Launcher') - parser.add_argument('Patch|Game|Component', type=str, nargs='?', - help="Pass either a patch file, a generated game or the name of a component to run.") - parser.add_argument('args', nargs="*", help="Arguments to pass to component.") + run_group = parser.add_argument_group("Run") + run_group.add_argument("--update_settings", action="store_true", + help="Update host.yaml and exit.") + run_group.add_argument("Patch|Game|Component", type=str, nargs="?", + help="Pass either a patch file, a generated game or the name of a component to run.") + run_group.add_argument("args", nargs="*", + help="Arguments to pass to component.") main(parser.parse_args()) from worlds.LauncherComponents import processes diff --git a/Main.py b/Main.py index 63173601cb96..6dfd61149b4f 100644 --- a/Main.py +++ b/Main.py @@ -13,7 +13,8 @@ from BaseClasses import CollectionState, Item, Location, LocationProgressType, MultiWorld, Region from Fill import balance_multiworld_progression, distribute_items_restrictive, distribute_planned, flood_items from Options import StartInventoryPool -from Utils import __version__, get_options, output_path, version_tuple +from settings import get_settings +from Utils import __version__, output_path, version_tuple from worlds import AutoWorld from worlds.generic.Rules import exclusion_rules, locality_rules @@ -22,7 +23,7 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = None): if not baked_server_options: - baked_server_options = get_options()["server_options"] + baked_server_options = get_settings().server_options if args.outputpath: os.makedirs(args.outputpath, exist_ok=True) output_path.cached_path = args.outputpath @@ -371,7 +372,7 @@ def precollect_hint(location): "connect_names": {name: (0, player) for player, name in world.player_name.items()}, "locations": locations_data, "checks_in_area": checks_in_area, - "server_options": baked_server_options, + "server_options": baked_server_options.as_dict(), "er_hint_data": er_hint_data, "precollected_items": precollected_items, "precollected_hints": precollected_hints, diff --git a/MinecraftClient.py b/MinecraftClient.py index dd7a5cfd3efb..93385ec5385e 100644 --- a/MinecraftClient.py +++ b/MinecraftClient.py @@ -299,7 +299,7 @@ def is_correct_forge(forge_dir) -> bool: versions = get_minecraft_versions(data_version, channel) - forge_dir = Utils.user_path(options["minecraft_options"]["forge_directory"]) + forge_dir = options["minecraft_options"]["forge_directory"] max_heap = options["minecraft_options"]["max_heap_size"] forge_version = args.forge or versions["forge"] java_version = args.java or versions["java"] diff --git a/OoTClient.py b/OoTClient.py index fd93c09338ac..115490417334 100644 --- a/OoTClient.py +++ b/OoTClient.py @@ -296,8 +296,6 @@ async def patch_and_run_game(apz5_file): comp_path = base_name + '.z64' # Load vanilla ROM, patch file, compress ROM rom_file_name = Utils.get_options()["oot_options"]["rom_file"] - if not os.path.exists(rom_file_name): - rom_file_name = Utils.user_path(rom_file_name) rom = Rom(rom_file_name) sub_file = None diff --git a/Utils.py b/Utils.py index 1acd56514f86..f3e748d1cc09 100644 --- a/Utils.py +++ b/Utils.py @@ -13,8 +13,9 @@ import collections import importlib import logging -from typing import BinaryIO, Coroutine, Optional, Set, Dict, Any, Union +from settings import Settings, get_settings +from typing import BinaryIO, Coroutine, Optional, Set, Dict, Any, Union from yaml import load, load_all, dump, SafeLoader try: @@ -138,13 +139,16 @@ def user_path(*path: str) -> str: user_path.cached_path = local_path() else: user_path.cached_path = home_path() - # populate home from local - TODO: upgrade feature - if user_path.cached_path != local_path() and not os.path.exists(user_path("host.yaml")): - import shutil - for dn in ("Players", "data/sprites"): - shutil.copytree(local_path(dn), user_path(dn), dirs_exist_ok=True) - for fn in ("manifest.json", "host.yaml"): - shutil.copy2(local_path(fn), user_path(fn)) + # populate home from local + if user_path.cached_path != local_path(): + import filecmp + if not os.path.exists(user_path("manifest.json")) or \ + not filecmp.cmp(local_path("manifest.json"), user_path("manifest.json"), shallow=True): + import shutil + for dn in ("Players", "data/sprites"): + shutil.copytree(local_path(dn), user_path(dn), dirs_exist_ok=True) + for fn in ("manifest.json",): + shutil.copy2(local_path(fn), user_path(fn)) return os.path.join(user_path.cached_path, *path) @@ -238,155 +242,15 @@ def get_public_ipv6() -> str: return ip -OptionsType = typing.Dict[str, typing.Dict[str, typing.Any]] +OptionsType = Settings # TODO: remove ~2 versions after 0.4.1 @cache_argsless -def get_default_options() -> OptionsType: - # Refer to host.yaml for comments as to what all these options mean. - options = { - "general_options": { - "output_path": "output", - }, - "factorio_options": { - "executable": os.path.join("factorio", "bin", "x64", "factorio"), - "filter_item_sends": False, - "bridge_chat_out": True, - }, - "sni_options": { - "sni_path": "SNI", - "snes_rom_start": True, - }, - "sm_options": { - "rom_file": "Super Metroid (JU).sfc", - }, - "soe_options": { - "rom_file": "Secret of Evermore (USA).sfc", - }, - "lttp_options": { - "rom_file": "Zelda no Densetsu - Kamigami no Triforce (Japan).sfc", - }, - "ladx_options": { - "rom_file": "Legend of Zelda, The - Link's Awakening DX (USA, Europe) (SGB Enhanced).gbc", - }, - "server_options": { - "host": None, - "port": 38281, - "password": None, - "multidata": None, - "savefile": None, - "disable_save": False, - "loglevel": "info", - "server_password": None, - "disable_item_cheat": False, - "location_check_points": 1, - "hint_cost": 10, - "release_mode": "goal", - "collect_mode": "disabled", - "remaining_mode": "goal", - "auto_shutdown": 0, - "compatibility": 2, - "log_network": 0 - }, - "generator": { - "enemizer_path": os.path.join("EnemizerCLI", "EnemizerCLI.Core"), - "player_files_path": "Players", - "players": 0, - "weights_file_path": "weights.yaml", - "meta_file_path": "meta.yaml", - "spoiler": 3, - "glitch_triforce_room": 1, - "race": 0, - "plando_options": "bosses", - }, - "minecraft_options": { - "forge_directory": "Minecraft Forge server", - "max_heap_size": "2G", - "release_channel": "release" - }, - "oot_options": { - "rom_file": "The Legend of Zelda - Ocarina of Time.z64", - "rom_start": True - }, - "dkc3_options": { - "rom_file": "Donkey Kong Country 3 - Dixie Kong's Double Trouble! (USA) (En,Fr).sfc", - }, - "smw_options": { - "rom_file": "Super Mario World (USA).sfc", - }, - "zillion_options": { - "rom_file": "Zillion (UE) [!].sms", - # RetroArch doesn't make it easy to launch a game from the command line. - # You have to know the path to the emulator core library on the user's computer. - "rom_start": "retroarch", - }, - "pokemon_rb_options": { - "red_rom_file": "Pokemon Red (UE) [S][!].gb", - "blue_rom_file": "Pokemon Blue (UE) [S][!].gb", - "rom_start": True - }, - "ffr_options": { - "display_msgs": True, - }, - "lufia2ac_options": { - "rom_file": "Lufia II - Rise of the Sinistrals (USA).sfc", - }, - "tloz_options": { - "rom_file": "Legend of Zelda, The (U) (PRG0) [!].nes", - "rom_start": True, - "display_msgs": True, - }, - "wargroove_options": { - "root_directory": "C:/Program Files (x86)/Steam/steamapps/common/Wargroove" - }, - "mmbn3_options": { - "rom_file": "Mega Man Battle Network 3 - Blue Version (USA).gba", - "rom_start": True - }, - "adventure_options": { - "rom_file": "ADVNTURE.BIN", - "display_msgs": True, - "rom_start": True, - "rom_args": "" - }, - } - return options - - -def update_options(src: dict, dest: dict, filename: str, keys: list) -> OptionsType: - for key, value in src.items(): - new_keys = keys.copy() - new_keys.append(key) - option_name = '.'.join(new_keys) - if key not in dest: - dest[key] = value - if filename.endswith("options.yaml"): - logging.info(f"Warning: {filename} is missing {option_name}") - elif isinstance(value, dict): - if not isinstance(dest.get(key, None), dict): - if filename.endswith("options.yaml"): - logging.info(f"Warning: {filename} has {option_name}, but it is not a dictionary. overwriting.") - dest[key] = value - else: - dest[key] = update_options(value, dest[key], filename, new_keys) - return dest +def get_default_options() -> Settings: # TODO: remove ~2 versions after 0.4.1 + return Settings(None) -@cache_argsless -def get_options() -> OptionsType: - filenames = ("options.yaml", "host.yaml") - locations: typing.List[str] = [] - if os.path.join(os.getcwd()) != local_path(): - locations += filenames # use files from cwd only if it's not the local_path - locations += [user_path(filename) for filename in filenames] - - for location in locations: - if os.path.exists(location): - with open(location) as f: - options = parse_yaml(f.read()) - return update_options(get_default_options(), options, location, list()) - - raise FileNotFoundError(f"Could not find {filenames[1]} to load options.") +get_options = get_settings # TODO: add a warning ~2 versions after 0.4.1 and remove once all games are ported def persistent_store(category: str, key: typing.Any, value: typing.Any): @@ -677,7 +541,7 @@ def get_fuzzy_ratio(word1: str, word2: str) -> float: ) -def open_filename(title: str, filetypes: typing.Sequence[typing.Tuple[str, typing.Sequence[str]]]) \ +def open_filename(title: str, filetypes: typing.Sequence[typing.Tuple[str, typing.Sequence[str]]], suggest: str = "") \ -> typing.Optional[str]: def run(*args: str): return subprocess.run(args, capture_output=True, text=True).stdout.split("\n", 1)[0] or None @@ -688,11 +552,43 @@ def run(*args: str): kdialog = which("kdialog") if kdialog: k_filters = '|'.join((f'{text} (*{" *".join(ext)})' for (text, ext) in filetypes)) - return run(kdialog, f"--title={title}", "--getopenfilename", ".", k_filters) + return run(kdialog, f"--title={title}", "--getopenfilename", suggest or ".", k_filters) zenity = which("zenity") if zenity: z_filters = (f'--file-filter={text} ({", ".join(ext)}) | *{" *".join(ext)}' for (text, ext) in filetypes) - return run(zenity, f"--title={title}", "--file-selection", *z_filters) + selection = (f'--filename="{suggest}',) if suggest else () + return run(zenity, f"--title={title}", "--file-selection", *z_filters, *selection) + + # fall back to tk + try: + import tkinter + import tkinter.filedialog + except Exception as e: + logging.error('Could not load tkinter, which is likely not installed. ' + f'This attempt was made because open_filename was used for "{title}".') + raise e + else: + root = tkinter.Tk() + root.withdraw() + return tkinter.filedialog.askopenfilename(title=title, filetypes=((t[0], ' '.join(t[1])) for t in filetypes), + initialfile=suggest or None) + + +def open_directory(title: str, suggest: str = "") -> typing.Optional[str]: + def run(*args: str): + return subprocess.run(args, capture_output=True, text=True).stdout.split("\n", 1)[0] or None + + if is_linux: + # prefer native dialog + from shutil import which + kdialog = None#which("kdialog") + if kdialog: + return run(kdialog, f"--title={title}", "--getexistingdirectory", suggest or ".") + zenity = None#which("zenity") + if zenity: + z_filters = ("--directory",) + selection = (f'--filename="{suggest}',) if suggest else () + return run(zenity, f"--title={title}", "--file-selection", *z_filters, *selection) # fall back to tk try: @@ -705,7 +601,7 @@ def run(*args: str): else: root = tkinter.Tk() root.withdraw() - return tkinter.filedialog.askopenfilename(title=title, filetypes=((t[0], ' '.join(t[1])) for t in filetypes)) + return tkinter.filedialog.askdirectory(title=title, mustexist=True, initialdir=suggest or None) def messagebox(title: str, text: str, error: bool = False) -> None: diff --git a/WebHost.py b/WebHost.py index 40d366a02f9e..eb859ec5852b 100644 --- a/WebHost.py +++ b/WebHost.py @@ -10,6 +10,7 @@ # in case app gets imported by something like gunicorn import Utils +import settings Utils.local_path.cached_path = os.path.dirname(__file__) or "." # py3.8 is not abs. remove "." when dropping 3.8 @@ -21,6 +22,7 @@ from WebHostLib.lttpsprites import update_sprites_lttp from WebHostLib.options import create as create_options_files +settings.no_gui = True configpath = os.path.abspath("config.yaml") if not os.path.exists(configpath): # fall back to config.yaml in home configpath = os.path.abspath(Utils.user_path('config.yaml')) diff --git a/docs/settings api.md b/docs/settings api.md new file mode 100644 index 000000000000..f9cbe5e021cc --- /dev/null +++ b/docs/settings api.md @@ -0,0 +1,187 @@ +# Archipelago Settings API + +The settings API describes how to use installation-wide config and let the user configure them, like paths, etc. using +host.yaml. For the player settings / player yamls see [options api.md](options api.md). + +The settings API replaces `Utils.get_options()` and `Utils.get_default_options()` +as well as the predefined `host.yaml` in the repository. + +For backwards compatibility with APWorlds, some interfaces are kept for now and will produce a warning when being used. + + +## Config File + +Settings use options.yaml (manual override), if that exists, or host.yaml (the default) otherwise. +The files are searched for in the current working directory, if different from install directory, and in `user_path`, +which either points to the installation directory, if writable, or to %home%/Archipelago otherwise. + +**Examples:** +* C:\Program Data\Archipelago\options.yaml +* C:\Program Data\Archipelago\host.yaml +* path\to\code\repository\host.yaml +* ~/Archipelago/host.yaml + +Using the settings API, AP can update the config file or create a new one with default values and comments, +if it does not exist. + + +## Global Settings + +All non-world-specific settings are defined directly in settings.py. +Each value needs to have a default. If the default should be `None`, define it as `typing.Optional` and assign `None`. + +To access a "global" config value, with correct typing, use one of +```python +from settings import get_settings, GeneralOptions, FolderPath +from typing import cast + +x = get_settings().general_options.output_path +y = cast(GeneralOptions, get_settings()["general_options"]).output_path +z = cast(FolderPath, get_settings()["general_options"]["output_path"]) +``` + + +## World Settings + +Worlds can define the top level key to use by defining `settings_key: ClassVar[str]` in their World class. +It defaults to `{folder_name}_options` if undefined, i.e. `worlds/factorio/...` defaults to `factorio_options`. + +Worlds define the layout of their config section using type annotation of the variable `settings` in the class. +The type has to inherit from `settings.Group`. Each value in the config can have a comment by subclassing a built-in +type. Some helper types are defined in `settings.py`, see [Types](#Types) for a list.``` + +Inside the class code, you can then simply use `self.settings.rom_file` to get the value. +In case of paths they will automatically be read as absolute file paths. No need to use user_path or local_path. + +```python +import settings +from worlds.AutoWorld import World + + +class MyGameSettings(settings.Group): + class RomFile(settings.SNESRomPath): + """Description that is put into host.yaml""" + description = "My Game US v1.0 ROM File" # displayed in the file browser + copy_to = "MyGame.sfc" # instead of storing the path, copy to AP dir + md5s = ["..."] + + rom_file: RomFile = RomFile("MyGame.sfc") # definition and default value + + +class MyGameWorld(World): + ... + settings: MyGameSettings + ... + + def something(self): + pass # use self.settings.rom_file here +``` + + +## Types + +When writing the host.yaml, the code will down cast the values to builtins. +When reading the host.yaml, the code will upcast the values to what is defined in the type annotations. +E.g. an IntEnum becomes int when saving and will construct the IntEnum when loading. + +Types that can not be down cast to / up cast from a builtin can not be used except for Group, which will be converted +to/from a dict. +`bool` is a special case, see settings.py: ServerOptions.disable_item_cheat for an example. + +Below are some predefined types that can be used if they match your requirements: + + +### Group + +A section / dict in the config file. Behaves similar to a dataclass. +Type annotation and default assignment define how loading, saving and default values behave. +It can be accessed using attributes or as a dict: `group["a"]` is equivalent to `group.a`. + +In worlds, this should only be used for the top level to avoid issues when upgrading/migrating. + + +### Bool + +Since `bool` can not be subclassed, use the `settings.Bool` helper in a `typing.Union` to get a comment in host.yaml. + +```python +import settings +import typing + +class MySettings(settings.Group): + class MyBool(settings.Bool): + """Doc string""" + + my_value: typing.Union[MyBool, bool] = True +``` + +### UserFilePath + +Path to a single file. Automatically resolves as user_path: +Source folder or AP install path on Windows. ~/Archipelago for the AppImage. +Will open a file browser if the file is missing when in GUI mode. + +#### class method validate(cls, path: str) + +Override this and raise ValueError if validation fails. +Checks the file against [md5s](#md5s) by default. + +#### is_exe: bool + +Resolves to an executable (varying file extension based on platform) + +#### description: Optional\[str\] + +Human-readable name to use in file browser + +#### copy_to: Optional\[str\] + +Instead of storing the path, copy the file. + +#### md5s: List[Union[str, bytes]] + +Provide md5 hashes as hex digests or raw bytes for automatic validation. + + +### UserFolderPath + +Same as [UserFilePath](#UserFilePath), but for a folder instead of a file. + + +### LocalFilePath + +Same as [UserFilePath](#UserFilePath), but resolves as local_path: +path inside the AP dir or Appimage even if read-only. + + +### LocalFolderPath + +Same as [LocalFilePath](#LocalFilePath), but for a folder instead of a file. + + +### OptionalUserFilePath, OptionalUserFolderPath, OptionalLocalFilePath, OptionalLocalFolderPath + +Same as UserFilePath, UserFolderPath, LocalFilePath, LocalFolderPath but does not open a file browser if missing. + + +### SNESRomPath + +Specialized [UserFilePath](#UserFilePath) that ignores an optional 512 byte header when validating. + + +## Caveats + +### Circular Imports + +Because the settings are defined on import, code that runs on import can not use settings since that would result in +circular / partial imports. Instead, the code should fetch from settings on demand during generation. + +"Global" settings are populated immediately, while worlds settings are lazy loaded, so if really necessary, +"global" settings could be used in global scope of worlds. + + +### APWorld Backwards Compatibility + +APWorlds that want to be compatible with both stable and dev versions, have two options: +1. use the old Utils.get_options() API until Archipelago 0.4.2 is out +2. add some sort of compatibility code to your world that mimics the new API diff --git a/docs/world api.md b/docs/world api.md index cc764ba3a6e0..b866549a85fd 100644 --- a/docs/world api.md +++ b/docs/world api.md @@ -91,10 +91,13 @@ added to the `World` object for easy access. ### World Options -Any AP installation can provide settings for a world, for example a ROM file, -accessible through `Utils.get_options()['_options']['