diff --git a/.gitignore b/.gitignore index 4a9f3402a964..f0df6e6f5675 100644 --- a/.gitignore +++ b/.gitignore @@ -52,6 +52,7 @@ Output Logs/ /setup.ini /installdelete.iss /data/user.kv +/datapackage # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/BaseClasses.py b/BaseClasses.py index 221675bfd43c..bfc1c69d27d4 100644 --- a/BaseClasses.py +++ b/BaseClasses.py @@ -336,7 +336,7 @@ def get_player_name(self, player: int) -> str: return self.player_name[player] def get_file_safe_player_name(self, player: int) -> str: - return ''.join(c for c in self.get_player_name(player) if c not in '<>:"/\\|?*') + return Utils.get_file_safe_name(self.get_player_name(player)) def get_out_file_name_base(self, player: int) -> str: """ the base name (without file extension) for each player's output file for a seed """ diff --git a/CommonClient.py b/CommonClient.py index 02dd55da9859..afce98d8ca36 100644 --- a/CommonClient.py +++ b/CommonClient.py @@ -136,7 +136,7 @@ class CommonContext: items_handling: typing.Optional[int] = None want_slot_data: bool = True # should slot_data be retrieved via Connect - # datapackage + # data package # Contents in flux until connection to server is made, to download correct data for this multiworld. item_names: typing.Dict[int, str] = Utils.KeyedDefaultDict(lambda code: f'Unknown item (ID:{code})') location_names: typing.Dict[int, str] = Utils.KeyedDefaultDict(lambda code: f'Unknown location (ID:{code})') @@ -223,7 +223,7 @@ def __init__(self, server_address: typing.Optional[str], password: typing.Option self.watcher_event = asyncio.Event() self.jsontotextparser = JSONtoTextParser(self) - self.update_datapackage(network_data_package) + self.update_data_package(network_data_package) # execution self.keep_alive_task = asyncio.create_task(keep_alive(self), name="Bouncy") @@ -399,32 +399,40 @@ async def shutdown(self): self.input_task.cancel() # DataPackage - async def prepare_datapackage(self, relevant_games: typing.Set[str], - remote_datepackage_versions: typing.Dict[str, int]): + async def prepare_data_package(self, relevant_games: typing.Set[str], + remote_date_package_versions: typing.Dict[str, int], + remote_data_package_checksums: typing.Dict[str, str]): """Validate that all data is present for the current multiworld. Download, assimilate and cache missing data from the server.""" # by documentation any game can use Archipelago locations/items -> always relevant relevant_games.add("Archipelago") - cache_package = Utils.persistent_load().get("datapackage", {}).get("games", {}) needed_updates: typing.Set[str] = set() for game in relevant_games: - if game not in remote_datepackage_versions: + if game not in remote_date_package_versions and game not in remote_data_package_checksums: continue - remote_version: int = remote_datepackage_versions[game] - if remote_version == 0: # custom datapackage for this game + remote_version: int = remote_date_package_versions.get(game, 0) + remote_checksum: typing.Optional[str] = remote_data_package_checksums.get(game) + + if remote_version == 0 and not remote_checksum: # custom data package and no checksum for this game needed_updates.add(game) continue + local_version: int = network_data_package["games"].get(game, {}).get("version", 0) + local_checksum: typing.Optional[str] = network_data_package["games"].get(game, {}).get("checksum") # no action required if local version is new enough - if remote_version > local_version: - cache_version: int = cache_package.get(game, {}).get("version", 0) + if (not remote_checksum and (remote_version > local_version or remote_version == 0)) \ + or remote_checksum != local_checksum: + cached_game = Utils.load_data_package_for_checksum(game, remote_checksum) + cache_version: int = cached_game.get("version", 0) + cache_checksum: typing.Optional[str] = cached_game.get("checksum") # download remote version if cache is not new enough - if remote_version > cache_version: + if (not remote_checksum and (remote_version > cache_version or remote_version == 0)) \ + or remote_checksum != cache_checksum: needed_updates.add(game) else: - self.update_game(cache_package[game]) + self.update_game(cached_game) if needed_updates: await self.send_msgs([{"cmd": "GetDataPackage", "games": list(needed_updates)}]) @@ -434,15 +442,17 @@ def update_game(self, game_package: dict): for location_name, location_id in game_package["location_name_to_id"].items(): self.location_names[location_id] = location_name - def update_datapackage(self, data_package: dict): - for game, gamedata in data_package["games"].items(): - self.update_game(gamedata) + def update_data_package(self, data_package: dict): + for game, game_data in data_package["games"].items(): + self.update_game(game_data) - def consume_network_datapackage(self, data_package: dict): - self.update_datapackage(data_package) + def consume_network_data_package(self, data_package: dict): + self.update_data_package(data_package) current_cache = Utils.persistent_load().get("datapackage", {}).get("games", {}) current_cache.update(data_package["games"]) Utils.persistent_store("datapackage", "games", current_cache) + for game, game_data in data_package["games"].items(): + Utils.store_data_package_for_checksum(game, game_data) # DeathLink hooks @@ -661,14 +671,16 @@ async def process_server_cmd(ctx: CommonContext, args: dict): current_team = network_player.team logger.info(' %s (Player %d)' % (network_player.alias, network_player.slot)) - # update datapackage - await ctx.prepare_datapackage(set(args["games"]), args["datapackage_versions"]) + # update data package + data_package_versions = args.get("datapackage_versions", {}) + data_package_checksums = args.get("datapackage_checksums", {}) + await ctx.prepare_data_package(set(args["games"]), data_package_versions, data_package_checksums) await ctx.server_auth(args['password']) elif cmd == 'DataPackage': logger.info("Got new ID/Name DataPackage") - ctx.consume_network_datapackage(args['data']) + ctx.consume_network_data_package(args['data']) elif cmd == 'ConnectionRefused': errors = args["errors"] diff --git a/Main.py b/Main.py index 372cadc5fd88..03b2e1b155b4 100644 --- a/Main.py +++ b/Main.py @@ -355,13 +355,11 @@ def precollect_hint(location): for player in world.groups.get(location.item.player, {}).get("players", [])]): precollect_hint(location) - # custom datapackage - datapackage = {} - for game_world in world.worlds.values(): - if game_world.data_version == 0 and game_world.game not in datapackage: - datapackage[game_world.game] = worlds.network_data_package["games"][game_world.game] - datapackage[game_world.game]["item_name_groups"] = game_world.item_name_groups - datapackage[game_world.game]["location_name_groups"] = game_world.location_name_groups + # embedded data package + data_package = { + game_world.game: worlds.network_data_package["games"][game_world.game] + for game_world in world.worlds.values() + } multidata = { "slot_data": slot_data, @@ -378,7 +376,7 @@ def precollect_hint(location): "tags": ["AP"], "minimum_versions": minimum_versions, "seed_name": world.seed_name, - "datapackage": datapackage, + "datapackage": data_package, } AutoWorld.call_all(world, "modify_multidata", multidata) diff --git a/MultiServer.py b/MultiServer.py index 6c3106a93d9d..4eadbb7998d4 100644 --- a/MultiServer.py +++ b/MultiServer.py @@ -7,17 +7,20 @@ import logging import zlib import collections -import typing -import inspect -import weakref import datetime -import threading -import random -import pickle +import functools +import hashlib +import inspect import itertools -import time +import logging import operator -import hashlib +import pickle +import random +import threading +import time +import typing +import weakref +import zlib import ModuleUpdate @@ -160,6 +163,7 @@ class Context: stored_data_notification_clients: typing.Dict[str, typing.Set[Client]] slot_info: typing.Dict[int, NetworkSlot] + checksums: typing.Dict[str, str] item_names: typing.Dict[int, str] = Utils.KeyedDefaultDict(lambda code: f'Unknown item (ID:{code})') item_name_groups: typing.Dict[str, typing.Dict[str, typing.Set[str]]] location_names: typing.Dict[int, str] = Utils.KeyedDefaultDict(lambda code: f'Unknown location (ID:{code})') @@ -233,6 +237,7 @@ def __init__(self, host: str, port: int, server_password: str, password: str, lo # init empty to satisfy linter, I suppose self.gamespackage = {} + self.checksums = {} self.item_name_groups = {} self.location_name_groups = {} self.all_item_and_group_names = {} @@ -241,7 +246,7 @@ def __init__(self, host: str, port: int, server_password: str, password: str, lo self._load_game_data() - # Datapackage retrieval + # Data package retrieval def _load_game_data(self): import worlds self.gamespackage = worlds.network_data_package["games"] @@ -255,6 +260,7 @@ def _load_game_data(self): def _init_game_data(self): for game_name, game_package in self.gamespackage.items(): + self.checksums[game_name] = game_package["checksum"] for item_name, item_id in game_package["item_name_to_id"].items(): self.item_names[item_id] = item_name for location_name, location_id in game_package["location_name_to_id"].items(): @@ -350,6 +356,7 @@ def notify_client_multiple(self, client: Client, texts: typing.List[str], additi [{"cmd": "PrintJSON", "data": [{ "text": text }], **additional_arguments} for text in texts])) + # loading def load(self, multidatapath: str, use_embedded_server_options: bool = False): @@ -366,7 +373,7 @@ def load(self, multidatapath: str, use_embedded_server_options: bool = False): with open(multidatapath, 'rb') as f: data = f.read() - self._load(self.decompress(data), use_embedded_server_options) + self._load(self.decompress(data), {}, use_embedded_server_options) self.data_filename = multidatapath @staticmethod @@ -376,7 +383,8 @@ def decompress(data: bytes) -> dict: raise Utils.VersionException("Incompatible multidata.") return restricted_loads(zlib.decompress(data[1:])) - def _load(self, decoded_obj: dict, use_embedded_server_options: bool): + def _load(self, decoded_obj: dict, game_data_packages: typing.Dict[str, typing.Any], + use_embedded_server_options: bool): self.read_data = {} mdata_ver = decoded_obj["minimum_versions"]["server"] if mdata_ver > Utils.version_tuple: @@ -431,13 +439,15 @@ def _load(self, decoded_obj: dict, use_embedded_server_options: bool): server_options = decoded_obj.get("server_options", {}) self._set_options(server_options) - # custom datapackage + # embedded data package for game_name, data in decoded_obj.get("datapackage", {}).items(): - logging.info(f"Loading custom datapackage for game {game_name}") + if game_name in game_data_packages: + data = game_data_packages[game_name] + logging.info(f"Loading embedded data package for game {game_name}") self.gamespackage[game_name] = data self.item_name_groups[game_name] = data["item_name_groups"] self.location_name_groups[game_name] = data["location_name_groups"] - del data["item_name_groups"] # remove from datapackage, but keep in self.item_name_groups + del data["item_name_groups"] # remove from data package, but keep in self.item_name_groups del data["location_name_groups"] self._init_game_data() for game_name, data in self.item_name_groups.items(): @@ -735,10 +745,11 @@ async def on_client_connected(ctx: Context, client: Client): NetworkPlayer(team, slot, ctx.name_aliases.get((team, slot), name), name) ) + games = {ctx.games[x] for x in range(1, len(ctx.games) + 1)} await ctx.send_msgs(client, [{ 'cmd': 'RoomInfo', 'password': bool(ctx.password), - 'games': {ctx.games[x] for x in range(1, len(ctx.games) + 1)}, + 'games': games, # tags are for additional features in the communication. # Name them by feature or fork, as you feel is appropriate. 'tags': ctx.tags, @@ -747,7 +758,9 @@ async def on_client_connected(ctx: Context, client: Client): 'hint_cost': ctx.hint_cost, 'location_check_points': ctx.location_check_points, 'datapackage_versions': {game: game_data["version"] for game, game_data - in ctx.gamespackage.items()}, + in ctx.gamespackage.items() if game in games}, + 'datapackage_checksums': {game: game_data["checksum"] for game, game_data + in ctx.gamespackage.items() if game in games}, 'seed_name': ctx.seed_name, 'time': time.time(), }]) diff --git a/Utils.py b/Utils.py index 059168c85725..f7305a1e2a14 100644 --- a/Utils.py +++ b/Utils.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import json import typing import builtins import os @@ -142,6 +143,17 @@ def user_path(*path: str) -> str: return os.path.join(user_path.cached_path, *path) +def cache_path(*path: str) -> str: + """Returns path to a file in the user's Archipelago cache directory.""" + if hasattr(cache_path, "cached_path"): + pass + else: + import appdirs + cache_path.cached_path = appdirs.user_cache_dir("Archipelago", False) + + return os.path.join(cache_path.cached_path, *path) + + def output_path(*path: str) -> str: if hasattr(output_path, 'cached_path'): return os.path.join(output_path.cached_path, *path) @@ -385,6 +397,45 @@ def persistent_load() -> typing.Dict[str, dict]: return storage +def get_file_safe_name(name: str) -> str: + return "".join(c for c in name if c not in '<>:"/\\|?*') + + +def load_data_package_for_checksum(game: str, checksum: typing.Optional[str]) -> Dict[str, Any]: + if checksum and game: + if checksum != get_file_safe_name(checksum): + raise ValueError(f"Bad symbols in checksum: {checksum}") + path = cache_path("datapackage", get_file_safe_name(game), f"{checksum}.json") + if os.path.exists(path): + try: + with open(path, "r", encoding="utf-8-sig") as f: + return json.load(f) + except Exception as e: + logging.debug(f"Could not load data package: {e}") + + # fall back to old cache + cache = persistent_load().get("datapackage", {}).get("games", {}).get(game, {}) + if cache.get("checksum") == checksum: + return cache + + # cache does not match + return {} + + +def store_data_package_for_checksum(game: str, data: typing.Dict[str, Any]) -> None: + checksum = data.get("checksum") + if checksum and game: + if checksum != get_file_safe_name(checksum): + raise ValueError(f"Bad symbols in checksum: {checksum}") + game_folder = cache_path("datapackage", get_file_safe_name(game)) + os.makedirs(game_folder, exist_ok=True) + try: + with open(os.path.join(game_folder, f"{checksum}.json"), "w", encoding="utf-8-sig") as f: + json.dump(data, f, ensure_ascii=False, separators=(",", ":")) + except Exception as e: + logging.debug(f"Could not store data package: {e}") + + def get_adjuster_settings(game_name: str) -> typing.Dict[str, typing.Any]: adjuster_settings = persistent_load().get("adjuster", {}).get(game_name, {}) return adjuster_settings diff --git a/WebHostLib/api/__init__.py b/WebHostLib/api/__init__.py index eac19d84563b..102c3a49f6aa 100644 --- a/WebHostLib/api/__init__.py +++ b/WebHostLib/api/__init__.py @@ -39,12 +39,21 @@ def get_datapackage(): @api_endpoints.route('/datapackage_version') @cache.cached() - def get_datapackage_versions(): - from worlds import network_data_package, AutoWorldRegister + from worlds import AutoWorldRegister version_package = {game: world.data_version for game, world in AutoWorldRegister.world_types.items()} return version_package +@api_endpoints.route('/datapackage_checksum') +@cache.cached() +def get_datapackage_checksums(): + from worlds import network_data_package + version_package = { + game: game_data["checksum"] for game, game_data in network_data_package["games"].items() + } + return version_package + + from . import generate, user # trigger registration diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 584ca9fecab1..6e7da4073790 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -19,7 +19,7 @@ from MultiServer import Context, server, auto_shutdown, ServerCommandProcessor, ClientMessageProcessor, load_server_cert from Utils import get_public_ipv4, get_public_ipv6, restricted_loads, cache_argsless -from .models import Room, Command, db +from .models import Command, GameDataPackage, Room, db class CustomClientMessageProcessor(ClientMessageProcessor): @@ -92,7 +92,20 @@ def load(self, room_id: int): else: self.port = get_random_port() - return self._load(self.decompress(room.seed.multidata), True) + multidata = self.decompress(room.seed.multidata) + game_data_packages = {} + for game in list(multidata["datapackage"]): + game_data = multidata["datapackage"][game] + if "checksum" in game_data: + if self.gamespackage.get(game, {}).get("checksum") == game_data["checksum"]: + # non-custom. remove from multidata + # games package could be dropped from static data once all rooms embed data package + del multidata["datapackage"][game] + else: + data = Utils.restricted_loads(GameDataPackage.get(checksum=game_data["checksum"]).data) + game_data_packages[game] = data + + return self._load(multidata, game_data_packages, True) @db_session def init_save(self, enabled: bool = True): diff --git a/WebHostLib/models.py b/WebHostLib/models.py index dbd03b166c9a..eba5c4eb4da2 100644 --- a/WebHostLib/models.py +++ b/WebHostLib/models.py @@ -56,3 +56,8 @@ class Generation(db.Entity): options = Required(buffer, lazy=True) meta = Required(LongStr, default=lambda: "{\"race\": false}") state = Required(int, default=0, index=True) + + +class GameDataPackage(db.Entity): + checksum = PrimaryKey(str) + data = Required(bytes) diff --git a/WebHostLib/tracker.py b/WebHostLib/tracker.py index 8d9311fc8f64..d4260bc85aa3 100644 --- a/WebHostLib/tracker.py +++ b/WebHostLib/tracker.py @@ -11,10 +11,10 @@ from MultiServer import Context, get_saving_second from NetUtils import SlotType from Utils import restricted_loads -from worlds import lookup_any_item_id_to_name, lookup_any_location_id_to_name +from worlds import lookup_any_item_id_to_name, lookup_any_location_id_to_name, network_data_package from worlds.alttp import Items from . import app, cache -from .models import Room +from .models import GameDataPackage, Room alttp_icons = { "Blue Shield": r"https://www.zeldadungeon.net/wiki/images/8/85/Fighters-Shield.png", @@ -229,14 +229,15 @@ def render_timedelta(delta: datetime.timedelta): @pass_context def get_location_name(context: runtime.Context, loc: int) -> str: + # once all rooms embed data package, the chain lookup can be dropped context_locations = context.get("custom_locations", {}) - return collections.ChainMap(lookup_any_location_id_to_name, context_locations).get(loc, loc) + return collections.ChainMap(context_locations, lookup_any_location_id_to_name).get(loc, loc) @pass_context def get_item_name(context: runtime.Context, item: int) -> str: context_items = context.get("custom_items", {}) - return collections.ChainMap(lookup_any_item_id_to_name, context_items).get(item, item) + return collections.ChainMap(context_items, lookup_any_item_id_to_name).get(item, item) app.jinja_env.filters["location_name"] = get_location_name @@ -274,11 +275,21 @@ def get_static_room_data(room: Room): if slot_info.type == SlotType.group} for game in games.values(): - if game in multidata["datapackage"]: - custom_locations.update( - {id: name for name, id in multidata["datapackage"][game]["location_name_to_id"].items()}) - custom_items.update( - {id: name for name, id in multidata["datapackage"][game]["item_name_to_id"].items()}) + if game not in multidata["datapackage"]: + continue + game_data = multidata["datapackage"][game] + if "checksum" in game_data: + if network_data_package["games"].get(game, {}).get("checksum") == game_data["checksum"]: + # non-custom. remove from multidata + # network_data_package import could be skipped once all rooms embed data package + del multidata["datapackage"][game] + continue + else: + game_data = restricted_loads(GameDataPackage.get(checksum=game_data["checksum"]).data) + custom_locations.update( + {id_: name for name, id_ in game_data["location_name_to_id"].items()}) + custom_items.update( + {id_: name for name, id_ in game_data["item_name_to_id"].items()}) elif "games" in multidata: games = multidata["games"] seed_checks_in_area = checks_in_area.copy() diff --git a/WebHostLib/upload.py b/WebHostLib/upload.py index dd0d218ed2ce..0314d64ab12d 100644 --- a/WebHostLib/upload.py +++ b/WebHostLib/upload.py @@ -1,19 +1,22 @@ import base64 import json +import pickle import typing import uuid import zipfile -from io import BytesIO +import zlib +from io import BytesIO from flask import request, flash, redirect, url_for, session, render_template, Markup -from pony.orm import flush, select +from pony.orm import commit, flush, select, rollback +from pony.orm.core import TransactionIntegrityError import MultiServer from NetUtils import NetworkSlot, SlotType from Utils import VersionException, __version__ from worlds.Files import AutoPatchRegister from . import app -from .models import Seed, Room, Slot +from .models import Seed, Room, Slot, GameDataPackage banned_zip_contents = (".sfc", ".z64", ".n64", ".sms", ".gb") @@ -78,6 +81,27 @@ def upload_zip_to_db(zfile: zipfile.ZipFile, owner=None, meta={"race": False}, s # Load multi data. if multidata: decompressed_multidata = MultiServer.Context.decompress(multidata) + recompress = False + + if "datapackage" in decompressed_multidata: + # strip datapackage from multidata, leaving only the checksums + game_data_packages: typing.List[GameDataPackage] = [] + for game, game_data in decompressed_multidata["datapackage"].items(): + if game_data.get("checksum"): + game_data_package = GameDataPackage(checksum=game_data["checksum"], + data=pickle.dumps(game_data)) + decompressed_multidata["datapackage"][game] = { + "version": game_data.get("version", 0), + "checksum": game_data["checksum"] + } + recompress = True + try: + commit() # commit game data package + game_data_packages.append(game_data_package) + except TransactionIntegrityError: + del game_data_package + rollback() + if "slot_info" in decompressed_multidata: for slot, slot_info in decompressed_multidata["slot_info"].items(): # Ignore Player Groups (e.g. item links) @@ -90,6 +114,9 @@ def upload_zip_to_db(zfile: zipfile.ZipFile, owner=None, meta={"race": False}, s flush() # commit slots + if recompress: + multidata = multidata[0:1] + zlib.compress(pickle.dumps(decompressed_multidata), 9) + seed = Seed(multidata=multidata, spoiler=spoiler, slots=slots, owner=owner, meta=json.dumps(meta), id=sid if sid else uuid.uuid4()) flush() # create seed diff --git a/docs/network protocol.md b/docs/network protocol.md index bfffcc580a54..c320934bd15a 100644 --- a/docs/network protocol.md +++ b/docs/network protocol.md @@ -64,18 +64,19 @@ These packets are are sent from the multiworld server to the client. They are no ### RoomInfo Sent to clients when they connect to an Archipelago server. #### Arguments -| Name | Type | Notes | -| ---- | ---- | ----- | -| version | [NetworkVersion](#NetworkVersion) | Object denoting the version of Archipelago which the server is running. | -| tags | list\[str\] | Denotes special features or capabilities that the sender is capable of. Example: `WebHost` | -| password | bool | Denoted whether a password is required to join this room.| -| permissions | dict\[str, [Permission](#Permission)\[int\]\] | Mapping of permission name to [Permission](#Permission), keys are: "release", "collect" and "remaining". | -| hint_cost | int | The amount of points it costs to receive a hint from the server. | -| location_check_points | int | The amount of hint points you receive per item/location check completed. || -| games | list\[str\] | List of games present in this multiworld. | -| datapackage_versions | dict\[str, int\] | Data versions of the individual games' data packages the server will send. Used to decide which games' caches are outdated. See [Data Package Contents](#Data-Package-Contents). | -| seed_name | str | uniquely identifying name of this generation | -| time | float | Unix time stamp of "now". Send for time synchronization if wanted for things like the DeathLink Bounce. | +| Name | Type | Notes | +|-----------------------|-----------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| version | [NetworkVersion](#NetworkVersion) | Object denoting the version of Archipelago which the server is running. | +| tags | list\[str\] | Denotes special features or capabilities that the sender is capable of. Example: `WebHost` | +| password | bool | Denoted whether a password is required to join this room. | +| permissions | dict\[str, [Permission](#Permission)\[int\]\] | Mapping of permission name to [Permission](#Permission), keys are: "release", "collect" and "remaining". | +| hint_cost | int | The amount of points it costs to receive a hint from the server. | +| location_check_points | int | The amount of hint points you receive per item/location check completed. | +| games | list\[str\] | List of games present in this multiworld. | +| datapackage_versions | dict\[str, int\] | Data versions of the individual games' data packages the server will send. Used to decide which games' caches are outdated. See [Data Package Contents](#Data-Package-Contents). **Deprecated. Use `datapackage_checksums` instead.** | +| datapackage_checksums | dict[str, str] | Checksum hash of the individual games' data packages the server will send. Used by newer clients to decide which games' caches are outdated. See [Data Package Contents](#Data-Package-Contents) for more information. | +| seed_name | str | Uniquely identifying name of this generation | +| time | float | Unix time stamp of "now". Send for time synchronization if wanted for things like the DeathLink Bounce. | #### release Dictates what is allowed when it comes to a player releasing their run. A release is an action which distributes the rest of the items in a player's run to those other players awaiting them. @@ -106,8 +107,8 @@ Dictates what is allowed when it comes to a player querying the items remaining ### ConnectionRefused Sent to clients when the server refuses connection. This is sent during the initial connection handshake. #### Arguments -| Name | Type | Notes | -| ---- | ---- | ----- | +| Name | Type | Notes | +|--------|-------------|--------------------------------------------------------------------------------------------------------------------------------------------------------| | errors | list\[str\] | Optional. When provided, should contain any one of: `InvalidSlot`, `InvalidGame`, `IncompatibleVersion`, `InvalidPassword`, or `InvalidItemsHandling`. | InvalidSlot indicates that the sent 'name' field did not match any auth entry on the server. @@ -644,11 +645,12 @@ Note: #### GameData GameData is a **dict** but contains these keys and values. It's broken out into another "type" for ease of documentation. -| Name | Type | Notes | -| ---- | ---- | ----- | -| item_name_to_id | dict[str, int] | Mapping of all item names to their respective ID. | -| location_name_to_id | dict[str, int] | Mapping of all location names to their respective ID. | -| version | int | Version number of this game's data | +| Name | Type | Notes | +|---------------------|----------------|-------------------------------------------------------------------------------------------------------------------------------| +| item_name_to_id | dict[str, int] | Mapping of all item names to their respective ID. | +| location_name_to_id | dict[str, int] | Mapping of all location names to their respective ID. | +| version | int | Version number of this game's data. Deprecated. Used by older clients to request an updated datapackage if cache is outdated. | +| checksum | str | A checksum hash of this game's data. | ### Tags Tags are represented as a list of strings, the common Client tags follow: diff --git a/docs/world api.md b/docs/world api.md index 922674fd29f6..66a639f1b802 100644 --- a/docs/world api.md +++ b/docs/world api.md @@ -364,14 +364,9 @@ class MyGameLocation(Location): # or from Locations import MyGameLocation class MyGameWorld(World): """Insert description of the world/game here.""" - game: str = "My Game" # name of the game/world + game = "My Game" # name of the game/world option_definitions = mygame_options # options the player can set - topology_present: bool = True # show path to required location checks in spoiler - - # data_version is used to signal that items, locations or their names - # changed. Set this to 0 during development so other games' clients do not - # cache any texts, then increase by 1 for each release that makes changes. - data_version = 0 + topology_present = True # show path to required location checks in spoiler # ID of first item and location, could be hard-coded but code may be easier # to read with this as a propery. diff --git a/requirements.txt b/requirements.txt index 6c9e3b9d2da6..8c73949dd475 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ jellyfish>=0.9.0 jinja2>=3.1.2 schema>=0.7.5 kivy>=2.1.0 -bsdiff4>=1.2.2 \ No newline at end of file +bsdiff4>=1.2.2 +appdirs>=1.4.4 diff --git a/worlds/AutoWorld.py b/worlds/AutoWorld.py index 3fb705bdf343..55a6b9219ac1 100644 --- a/worlds/AutoWorld.py +++ b/worlds/AutoWorld.py @@ -1,20 +1,24 @@ from __future__ import annotations +import hashlib import logging -import sys import pathlib -from typing import Dict, FrozenSet, Set, Tuple, List, Optional, TextIO, Any, Callable, Type, Union, TYPE_CHECKING, \ - ClassVar +import sys +from typing import Any, Callable, ClassVar, Dict, FrozenSet, List, Optional, Set, TYPE_CHECKING, TextIO, Tuple, Type, \ + Union -from Options import AssembleOptions from BaseClasses import CollectionState +from Options import AssembleOptions if TYPE_CHECKING: from BaseClasses import MultiWorld, Item, Location, Tutorial + from . import GamesPackage class AutoWorldRegister(type): world_types: Dict[str, Type[World]] = {} + __file__: str + zip_path: Optional[str] def __new__(mcs, name: str, bases: Tuple[type, ...], dct: Dict[str, Any]) -> AutoWorldRegister: if "web" in dct: @@ -154,9 +158,14 @@ class World(metaclass=AutoWorldRegister): data_version: ClassVar[int] = 1 """ - increment this every time something in your world's names/id mappings changes. - While this is set to 0, this world's DataPackage is considered in testing mode and will be inserted to the multidata - and retrieved by clients on every connection. + Increment this every time something in your world's names/id mappings changes. + + When this is set to 0, that world's DataPackage is considered in "testing mode", which signals to servers/clients + that it should not be cached, and clients should request that world's DataPackage every connection. Not + recommended for production-ready worlds. + + Deprecated. Clients should utilize `checksum` to determine if DataPackage has changed since last connection and + request a new DataPackage, if necessary. """ required_client_version: Tuple[int, int, int] = (0, 1, 6) @@ -343,8 +352,35 @@ def remove(self, state: "CollectionState", item: "Item") -> bool: def create_filler(self) -> "Item": return self.create_item(self.get_filler_item_name()) + @classmethod + def get_data_package_data(cls) -> "GamesPackage": + sorted_item_name_groups = { + name: sorted(cls.item_name_groups[name]) for name in sorted(cls.item_name_groups) + } + sorted_location_name_groups = { + name: sorted(cls.location_name_groups[name]) for name in sorted(cls.location_name_groups) + } + res: "GamesPackage" = { + # sorted alphabetically + "item_name_groups": sorted_item_name_groups, + "item_name_to_id": cls.item_name_to_id, + "location_name_groups": sorted_location_name_groups, + "location_name_to_id": cls.location_name_to_id, + "version": cls.data_version, + } + res["checksum"] = data_package_checksum(res) + return res + # any methods attached to this can be used as part of CollectionState, # please use a prefix as all of them get clobbered together class LogicMixin(metaclass=AutoLogicRegister): pass + + +def data_package_checksum(data: "GamesPackage") -> str: + """Calculates the data package checksum for a game from a dict""" + assert "checksum" not in data, "Checksum already in data" + assert sorted(data) == list(data), "Data not ordered" + from NetUtils import encode + return hashlib.sha1(encode(data).encode()).hexdigest() diff --git a/worlds/__init__.py b/worlds/__init__.py index 34dece0e40f8..3470c1a35338 100644 --- a/worlds/__init__.py +++ b/worlds/__init__.py @@ -20,14 +20,19 @@ from .AutoWorld import World -class GamesPackage(typing.TypedDict): +class GamesData(typing.TypedDict): + item_name_groups: typing.Dict[str, typing.List[str]] item_name_to_id: typing.Dict[str, int] + location_name_groups: typing.Dict[str, typing.List[str]] location_name_to_id: typing.Dict[str, int] version: int +class GamesPackage(GamesData, total=False): + checksum: str + + class DataPackage(typing.TypedDict): - version: int games: typing.Dict[str, GamesPackage] @@ -75,14 +80,9 @@ class WorldSource(typing.NamedTuple): from .AutoWorld import AutoWorldRegister +# Build the data package for each game. for world_name, world in AutoWorldRegister.world_types.items(): - games[world_name] = { - "item_name_to_id": world.item_name_to_id, - "location_name_to_id": world.location_name_to_id, - "version": world.data_version, - # seems clients don't actually want this. Keeping it here in case someone changes their mind. - # "item_name_groups": {name: tuple(items) for name, items in world.item_name_groups.items()} - } + games[world_name] = world.get_data_package_data() lookup_any_item_id_to_name.update(world.item_id_to_name) lookup_any_location_id_to_name.update(world.location_id_to_name) diff --git a/worlds/bk_sudoku/__init__.py b/worlds/bk_sudoku/__init__.py index 4b0f0fb40858..f914baf066aa 100644 --- a/worlds/bk_sudoku/__init__.py +++ b/worlds/bk_sudoku/__init__.py @@ -1,7 +1,8 @@ -from BaseClasses import Tutorial -from ..AutoWorld import World, WebWorld from typing import Dict +from BaseClasses import Tutorial +from ..AutoWorld import WebWorld, World + class Bk_SudokuWebWorld(WebWorld): settings_page = "games/Sudoku/info/en" @@ -24,6 +25,7 @@ class Bk_SudokuWorld(World): """ game = "Sudoku" web = Bk_SudokuWebWorld() + data_version = 1 item_name_to_id: Dict[str, int] = {} location_name_to_id: Dict[str, int] = {} diff --git a/worlds/generic/__init__.py b/worlds/generic/__init__.py index 4c7c14c48f06..732dc51196bb 100644 --- a/worlds/generic/__init__.py +++ b/worlds/generic/__init__.py @@ -42,6 +42,7 @@ class GenericWorld(World): } hidden = True web = GenericWeb() + data_version = 1 def generate_early(self): self.multiworld.player_types[self.player] = SlotType.spectator # mark as spectator diff --git a/worlds/hylics2/__init__.py b/worlds/hylics2/__init__.py index b5fede32c382..b12beaaa3a1d 100644 --- a/worlds/hylics2/__init__.py +++ b/worlds/hylics2/__init__.py @@ -2,8 +2,8 @@ from typing import Dict, Any from BaseClasses import Region, Entrance, Location, Item, Tutorial, ItemClassification from worlds.generic.Rules import set_rule -from ..AutoWorld import World, WebWorld -from . import Items, Locations, Options, Rules, Exits +from . import Exits, Items, Locations, Options, Rules +from ..AutoWorld import WebWorld, World class Hylics2Web(WebWorld): @@ -20,13 +20,13 @@ class Hylics2Web(WebWorld): class Hylics2World(World): """ - Hylics 2 is a surreal and unusual RPG, with a bizarre yet unique visual style. Play as Wayne, + Hylics 2 is a surreal and unusual RPG, with a bizarre yet unique visual style. Play as Wayne, travel the world, and gather your allies to defeat the nefarious Gibby in his Hylemxylem! """ game: str = "Hylics 2" web = Hylics2Web() - all_items = {**Items.item_table, **Items.gesture_item_table, **Items.party_item_table, + all_items = {**Items.item_table, **Items.gesture_item_table, **Items.party_item_table, **Items.medallion_item_table} all_locations = {**Locations.location_table, **Locations.tv_location_table, **Locations.party_location_table, **Locations.medallion_location_table} @@ -37,7 +37,7 @@ class Hylics2World(World): topology_present: bool = True - data_version: 1 + data_version = 1 start_location = "Waynehouse" @@ -59,7 +59,7 @@ def add_item(self, name: str, classification: ItemClassification, code: int) -> def create_event(self, event: str): return Hylics2Item(event, ItemClassification.progression_skip_balancing, None, self.player) - + # set random starting location if option is enabled def generate_early(self): if self.multiworld.random_start[self.player]: @@ -76,7 +76,7 @@ def generate_early(self): def generate_basic(self): # create item pool pool = [] - + # add regular items for i, data in Items.item_table.items(): if data["count"] > 0: @@ -114,7 +114,7 @@ def generate_basic(self): gestures = list(Items.gesture_item_table.items()) tvs = list(Locations.tv_location_table.items()) - # if Extra Items in Logic is enabled place CHARGE UP first and make sure it doesn't get + # if Extra Items in Logic is enabled place CHARGE UP first and make sure it doesn't get # placed at Sage Airship: TV if self.multiworld.extra_items_in_logic[self.player]: tv = self.multiworld.random.choice(tvs) @@ -122,7 +122,7 @@ def generate_basic(self): while tv[1]["name"] == "Sage Airship: TV": tv = self.multiworld.random.choice(tvs) self.multiworld.get_location(tv[1]["name"], self.player)\ - .place_locked_item(self.add_item(gestures[gest][1]["name"], gestures[gest][1]["classification"], + .place_locked_item(self.add_item(gestures[gest][1]["name"], gestures[gest][1]["classification"], gestures[gest])) gestures.remove(gestures[gest]) tvs.remove(tv) @@ -182,7 +182,7 @@ def create_regions(self) -> None: 16: Region("Sage Airship", self.player, self.multiworld), 17: Region("Hylemxylem", self.player, self.multiworld) } - + # create regions from table for i, reg in region_table.items(): self.multiworld.regions.append(reg) @@ -214,7 +214,7 @@ def create_regions(self) -> None: for i, data in Locations.tv_location_table.items(): region_table[data["region"]].locations\ .append(Hylics2Location(self.player, data["name"], i, region_table[data["region"]])) - + # add party member locations if option is enabled if self.multiworld.party_shuffle[self.player]: for i, data in Locations.party_location_table.items(): @@ -241,4 +241,4 @@ class Hylics2Location(Location): class Hylics2Item(Item): - game: str = "Hylics 2" \ No newline at end of file + game: str = "Hylics 2" diff --git a/worlds/oribf/__init__.py b/worlds/oribf/__init__.py index 02350917a3b3..854025a8ed51 100644 --- a/worlds/oribf/__init__.py +++ b/worlds/oribf/__init__.py @@ -13,6 +13,7 @@ class OriBlindForest(World): game: str = "Ori and the Blind Forest" topology_present = True + data_version = 1 item_name_to_id = item_table location_name_to_id = lookup_name_to_id