monorail
/
Archipelago
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Deprecate `data_version` and introduce `checksum` for DataPackages. (#684) 

Co-authored-by: black-sliver <59490463+black-sliver@users.noreply.github.com>
This commit is contained in:
Zach Parks 2023-03-20 11:01:08 -05:00 committed by GitHub
parent d825576f12
commit ff9f563d4a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 297 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -52,6 +52,7 @@ Output Logs/
/setup.ini
/installdelete.iss
/data/user.kv
/datapackage
# Byte-compiled / optimized / DLL files
__pycache__/

2
BaseClasses.py
View File

@ -336,7 +336,7 @@ class MultiWorld():
return self.player_name[player]
def get_file_safe_player_name(self, player: int) -> str:
return ''.join(c for c in self.get_player_name(player) if c not in '<>:"/\\|?*')
return Utils.get_file_safe_name(self.get_player_name(player))
def get_out_file_name_base(self, player: int) -> str:
""" the base name (without file extension) for each player's output file for a seed """

52
CommonClient.py
View File

@ -136,7 +136,7 @@ class CommonContext:
items_handling: typing.Optional[int] = None
want_slot_data: bool = True # should slot_data be retrieved via Connect
# datapackage
# data package
# Contents in flux until connection to server is made, to download correct data for this multiworld.
item_names: typing.Dict[int, str] = Utils.KeyedDefaultDict(lambda code: f'Unknown item (ID:{code})')
location_names: typing.Dict[int, str] = Utils.KeyedDefaultDict(lambda code: f'Unknown location (ID:{code})')
@ -223,7 +223,7 @@ class CommonContext:
self.watcher_event = asyncio.Event()
self.jsontotextparser = JSONtoTextParser(self)
self.update_datapackage(network_data_package)
self.update_data_package(network_data_package)
# execution
self.keep_alive_task = asyncio.create_task(keep_alive(self), name="Bouncy")
@ -399,32 +399,40 @@ class CommonContext:
self.input_task.cancel()
# DataPackage
async def prepare_datapackage(self, relevant_games: typing.Set[str],
remote_datepackage_versions: typing.Dict[str, int]):
async def prepare_data_package(self, relevant_games: typing.Set[str],
remote_date_package_versions: typing.Dict[str, int],
remote_data_package_checksums: typing.Dict[str, str]):
"""Validate that all data is present for the current multiworld.
Download, assimilate and cache missing data from the server."""
# by documentation any game can use Archipelago locations/items -> always relevant
relevant_games.add("Archipelago")
cache_package = Utils.persistent_load().get("datapackage", {}).get("games", {})
needed_updates: typing.Set[str] = set()
for game in relevant_games:
if game not in remote_datepackage_versions:
if game not in remote_date_package_versions and game not in remote_data_package_checksums:
continue
remote_version: int = remote_datepackage_versions[game]
if remote_version == 0: # custom datapackage for this game
remote_version: int = remote_date_package_versions.get(game, 0)
remote_checksum: typing.Optional[str] = remote_data_package_checksums.get(game)
if remote_version == 0 and not remote_checksum: # custom data package and no checksum for this game
needed_updates.add(game)
continue
local_version: int = network_data_package["games"].get(game, {}).get("version", 0)
local_checksum: typing.Optional[str] = network_data_package["games"].get(game, {}).get("checksum")
# no action required if local version is new enough
if remote_version > local_version:
cache_version: int = cache_package.get(game, {}).get("version", 0)
if (not remote_checksum and (remote_version > local_version or remote_version == 0)) \
or remote_checksum != local_checksum:
cached_game = Utils.load_data_package_for_checksum(game, remote_checksum)
cache_version: int = cached_game.get("version", 0)
cache_checksum: typing.Optional[str] = cached_game.get("checksum")
# download remote version if cache is not new enough
if remote_version > cache_version:
if (not remote_checksum and (remote_version > cache_version or remote_version == 0)) \
or remote_checksum != cache_checksum:
needed_updates.add(game)
else:
self.update_game(cache_package[game])
self.update_game(cached_game)
if needed_updates:
await self.send_msgs([{"cmd": "GetDataPackage", "games": list(needed_updates)}])
@ -434,15 +442,17 @@ class CommonContext:
for location_name, location_id in game_package["location_name_to_id"].items():
self.location_names[location_id] = location_name
def update_datapackage(self, data_package: dict):
for game, gamedata in data_package["games"].items():
self.update_game(gamedata)
def update_data_package(self, data_package: dict):
for game, game_data in data_package["games"].items():
self.update_game(game_data)
def consume_network_datapackage(self, data_package: dict):
self.update_datapackage(data_package)
def consume_network_data_package(self, data_package: dict):
self.update_data_package(data_package)
current_cache = Utils.persistent_load().get("datapackage", {}).get("games", {})
current_cache.update(data_package["games"])
Utils.persistent_store("datapackage", "games", current_cache)
for game, game_data in data_package["games"].items():
Utils.store_data_package_for_checksum(game, game_data)
# DeathLink hooks
@ -661,14 +671,16 @@ async def process_server_cmd(ctx: CommonContext, args: dict):
current_team = network_player.team
logger.info(' %s (Player %d)' % (network_player.alias, network_player.slot))
# update datapackage
await ctx.prepare_datapackage(set(args["games"]), args["datapackage_versions"])
# update data package
data_package_versions = args.get("datapackage_versions", {})
data_package_checksums = args.get("datapackage_checksums", {})
await ctx.prepare_data_package(set(args["games"]), data_package_versions, data_package_checksums)
await ctx.server_auth(args['password'])
elif cmd == 'DataPackage':
logger.info("Got new ID/Name DataPackage")
ctx.consume_network_datapackage(args['data'])
ctx.consume_network_data_package(args['data'])
elif cmd == 'ConnectionRefused':
errors = args["errors"]

14
Main.py
View File

@ -355,13 +355,11 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
for player in world.groups.get(location.item.player, {}).get("players", [])]):
precollect_hint(location)
# custom datapackage
datapackage = {}
for game_world in world.worlds.values():
if game_world.data_version == 0 and game_world.game not in datapackage:
datapackage[game_world.game] = worlds.network_data_package["games"][game_world.game]
datapackage[game_world.game]["item_name_groups"] = game_world.item_name_groups
datapackage[game_world.game]["location_name_groups"] = game_world.location_name_groups
# embedded data package
data_package = {
game_world.game: worlds.network_data_package["games"][game_world.game]
for game_world in world.worlds.values()
}
multidata = {
"slot_data": slot_data,
@ -378,7 +376,7 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
"tags": ["AP"],
"minimum_versions": minimum_versions,
"seed_name": world.seed_name,
"datapackage": datapackage,
"datapackage": data_package,
}
AutoWorld.call_all(world, "modify_multidata", multidata)

47
MultiServer.py
View File

@ -7,17 +7,20 @@ import functools
import logging
import zlib
import collections
import typing
import inspect
import weakref
import datetime
import threading
import random
import pickle
import itertools
import time
import operator
import functools
import hashlib
import inspect
import itertools
import logging
import operator
import pickle
import random
import threading
import time
import typing
import weakref
import zlib
import ModuleUpdate
@ -160,6 +163,7 @@ class Context:
stored_data_notification_clients: typing.Dict[str, typing.Set[Client]]
slot_info: typing.Dict[int, NetworkSlot]
checksums: typing.Dict[str, str]
item_names: typing.Dict[int, str] = Utils.KeyedDefaultDict(lambda code: f'Unknown item (ID:{code})')
item_name_groups: typing.Dict[str, typing.Dict[str, typing.Set[str]]]
location_names: typing.Dict[int, str] = Utils.KeyedDefaultDict(lambda code: f'Unknown location (ID:{code})')
@ -233,6 +237,7 @@ class Context:
# init empty to satisfy linter, I suppose
self.gamespackage = {}
self.checksums = {}
self.item_name_groups = {}
self.location_name_groups = {}
self.all_item_and_group_names = {}
@ -241,7 +246,7 @@ class Context:
self._load_game_data()
# Datapackage retrieval
# Data package retrieval
def _load_game_data(self):
import worlds
self.gamespackage = worlds.network_data_package["games"]
@ -255,6 +260,7 @@ class Context:
def _init_game_data(self):
for game_name, game_package in self.gamespackage.items():
self.checksums[game_name] = game_package["checksum"]
for item_name, item_id in game_package["item_name_to_id"].items():
self.item_names[item_id] = item_name
for location_name, location_id in game_package["location_name_to_id"].items():
@ -350,6 +356,7 @@ class Context:
[{"cmd": "PrintJSON", "data": [{ "text": text }], **additional_arguments}
for text in texts]))
# loading
def load(self, multidatapath: str, use_embedded_server_options: bool = False):
@ -366,7 +373,7 @@ class Context:
with open(multidatapath, 'rb') as f:
data = f.read()
self._load(self.decompress(data), use_embedded_server_options)
self._load(self.decompress(data), {}, use_embedded_server_options)
self.data_filename = multidatapath
@staticmethod
@ -376,7 +383,8 @@ class Context:
raise Utils.VersionException("Incompatible multidata.")
return restricted_loads(zlib.decompress(data[1:]))
def _load(self, decoded_obj: dict, use_embedded_server_options: bool):
def _load(self, decoded_obj: dict, game_data_packages: typing.Dict[str, typing.Any],
use_embedded_server_options: bool):
self.read_data = {}
mdata_ver = decoded_obj["minimum_versions"]["server"]
if mdata_ver > Utils.version_tuple:
@ -431,13 +439,15 @@ class Context:
server_options = decoded_obj.get("server_options", {})
self._set_options(server_options)
# custom datapackage
# embedded data package
for game_name, data in decoded_obj.get("datapackage", {}).items():
logging.info(f"Loading custom datapackage for game {game_name}")
if game_name in game_data_packages:
data = game_data_packages[game_name]
logging.info(f"Loading embedded data package for game {game_name}")
self.gamespackage[game_name] = data
self.item_name_groups[game_name] = data["item_name_groups"]
self.location_name_groups[game_name] = data["location_name_groups"]
del data["item_name_groups"] # remove from datapackage, but keep in self.item_name_groups
del data["item_name_groups"] # remove from data package, but keep in self.item_name_groups
del data["location_name_groups"]
self._init_game_data()
for game_name, data in self.item_name_groups.items():
@ -735,10 +745,11 @@ async def on_client_connected(ctx: Context, client: Client):
NetworkPlayer(team, slot,
ctx.name_aliases.get((team, slot), name), name)
)
games = {ctx.games[x] for x in range(1, len(ctx.games) + 1)}
await ctx.send_msgs(client, [{
'cmd': 'RoomInfo',
'password': bool(ctx.password),
'games': {ctx.games[x] for x in range(1, len(ctx.games) + 1)},
'games': games,
# tags are for additional features in the communication.
# Name them by feature or fork, as you feel is appropriate.
'tags': ctx.tags,
@ -747,7 +758,9 @@ async def on_client_connected(ctx: Context, client: Client):
'hint_cost': ctx.hint_cost,
'location_check_points': ctx.location_check_points,
'datapackage_versions': {game: game_data["version"] for game, game_data
in ctx.gamespackage.items()},
in ctx.gamespackage.items() if game in games},
'datapackage_checksums': {game: game_data["checksum"] for game, game_data
in ctx.gamespackage.items() if game in games},
'seed_name': ctx.seed_name,
'time': time.time(),
}])

51
Utils.py
View File

@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
import json
import typing
import builtins
import os
@ -142,6 +143,17 @@ def user_path(*path: str) -> str:
return os.path.join(user_path.cached_path, *path)
def cache_path(*path: str) -> str:
"""Returns path to a file in the user's Archipelago cache directory."""
if hasattr(cache_path, "cached_path"):
pass
else:
import appdirs
cache_path.cached_path = appdirs.user_cache_dir("Archipelago", False)
return os.path.join(cache_path.cached_path, *path)
def output_path(*path: str) -> str:
if hasattr(output_path, 'cached_path'):
return os.path.join(output_path.cached_path, *path)
@ -385,6 +397,45 @@ def persistent_load() -> typing.Dict[str, dict]:
return storage
def get_file_safe_name(name: str) -> str:
return "".join(c for c in name if c not in '<>:"/\\|?*')
def load_data_package_for_checksum(game: str, checksum: typing.Optional[str]) -> Dict[str, Any]:
if checksum and game:
if checksum != get_file_safe_name(checksum):
raise ValueError(f"Bad symbols in checksum: {checksum}")
path = cache_path("datapackage", get_file_safe_name(game), f"{checksum}.json")
if os.path.exists(path):
try:
with open(path, "r", encoding="utf-8-sig") as f:
return json.load(f)
except Exception as e:
logging.debug(f"Could not load data package: {e}")
# fall back to old cache
cache = persistent_load().get("datapackage", {}).get("games", {}).get(game, {})
if cache.get("checksum") == checksum:
return cache
# cache does not match
return {}
def store_data_package_for_checksum(game: str, data: typing.Dict[str, Any]) -> None:
checksum = data.get("checksum")
if checksum and game:
if checksum != get_file_safe_name(checksum):
raise ValueError(f"Bad symbols in checksum: {checksum}")
game_folder = cache_path("datapackage", get_file_safe_name(game))
os.makedirs(game_folder, exist_ok=True)
try:
with open(os.path.join(game_folder, f"{checksum}.json"), "w", encoding="utf-8-sig") as f:
json.dump(data, f, ensure_ascii=False, separators=(",", ":"))
except Exception as e:
logging.debug(f"Could not store data package: {e}")
def get_adjuster_settings(game_name: str) -> typing.Dict[str, typing.Any]:
adjuster_settings = persistent_load().get("adjuster", {}).get(game_name, {})
return adjuster_settings

13
WebHostLib/api/__init__.py
View File

@ -39,12 +39,21 @@ def get_datapackage():
@api_endpoints.route('/datapackage_version')
@cache.cached()
def get_datapackage_versions():
from worlds import network_data_package, AutoWorldRegister
from worlds import AutoWorldRegister
version_package = {game: world.data_version for game, world in AutoWorldRegister.world_types.items()}
return version_package
@api_endpoints.route('/datapackage_checksum')
@cache.cached()
def get_datapackage_checksums():
from worlds import network_data_package
version_package = {
game: game_data["checksum"] for game, game_data in network_data_package["games"].items()
}
return version_package
from . import generate, user # trigger registration

17
WebHostLib/customserver.py
View File

@ -19,7 +19,7 @@ import Utils
from MultiServer import Context, server, auto_shutdown, ServerCommandProcessor, ClientMessageProcessor, load_server_cert
from Utils import get_public_ipv4, get_public_ipv6, restricted_loads, cache_argsless
from .models import Room, Command, db
from .models import Command, GameDataPackage, Room, db
class CustomClientMessageProcessor(ClientMessageProcessor):
@ -92,7 +92,20 @@ class WebHostContext(Context):
else:
self.port = get_random_port()
return self._load(self.decompress(room.seed.multidata), True)
multidata = self.decompress(room.seed.multidata)
game_data_packages = {}
for game in list(multidata["datapackage"]):
game_data = multidata["datapackage"][game]
if "checksum" in game_data:
if self.gamespackage.get(game, {}).get("checksum") == game_data["checksum"]:
# non-custom. remove from multidata
# games package could be dropped from static data once all rooms embed data package
del multidata["datapackage"][game]
else:
data = Utils.restricted_loads(GameDataPackage.get(checksum=game_data["checksum"]).data)
game_data_packages[game] = data
return self._load(multidata, game_data_packages, True)
@db_session
def init_save(self, enabled: bool = True):

5
WebHostLib/models.py
View File

@ -56,3 +56,8 @@ class Generation(db.Entity):
options = Required(buffer, lazy=True)
meta = Required(LongStr, default=lambda: "{\"race\": false}")
state = Required(int, default=0, index=True)
class GameDataPackage(db.Entity):
checksum = PrimaryKey(str)
data = Required(bytes)

29
WebHostLib/tracker.py
View File

@ -11,10 +11,10 @@ from werkzeug.exceptions import abort
from MultiServer import Context, get_saving_second
from NetUtils import SlotType
from Utils import restricted_loads
from worlds import lookup_any_item_id_to_name, lookup_any_location_id_to_name
from worlds import lookup_any_item_id_to_name, lookup_any_location_id_to_name, network_data_package
from worlds.alttp import Items
from . import app, cache
from .models import Room
from .models import GameDataPackage, Room
alttp_icons = {
"Blue Shield": r"https://www.zeldadungeon.net/wiki/images/8/85/Fighters-Shield.png",
@ -229,14 +229,15 @@ def render_timedelta(delta: datetime.timedelta):
@pass_context
def get_location_name(context: runtime.Context, loc: int) -> str:
# once all rooms embed data package, the chain lookup can be dropped
context_locations = context.get("custom_locations", {})
return collections.ChainMap(lookup_any_location_id_to_name, context_locations).get(loc, loc)
return collections.ChainMap(context_locations, lookup_any_location_id_to_name).get(loc, loc)
@pass_context
def get_item_name(context: runtime.Context, item: int) -> str:
context_items = context.get("custom_items", {})
return collections.ChainMap(lookup_any_item_id_to_name, context_items).get(item, item)
return collections.ChainMap(context_items, lookup_any_item_id_to_name).get(item, item)
app.jinja_env.filters["location_name"] = get_location_name
@ -274,11 +275,21 @@ def get_static_room_data(room: Room):
if slot_info.type == SlotType.group}
for game in games.values():
if game in multidata["datapackage"]:
custom_locations.update(
{id: name for name, id in multidata["datapackage"][game]["location_name_to_id"].items()})
custom_items.update(
{id: name for name, id in multidata["datapackage"][game]["item_name_to_id"].items()})
if game not in multidata["datapackage"]:
continue
game_data = multidata["datapackage"][game]
if "checksum" in game_data:
if network_data_package["games"].get(game, {}).get("checksum") == game_data["checksum"]:
# non-custom. remove from multidata
# network_data_package import could be skipped once all rooms embed data package
del multidata["datapackage"][game]
continue
else:
game_data = restricted_loads(GameDataPackage.get(checksum=game_data["checksum"]).data)
custom_locations.update(
{id_: name for name, id_ in game_data["location_name_to_id"].items()})
custom_items.update(
{id_: name for name, id_ in game_data["item_name_to_id"].items()})
elif "games" in multidata:
games = multidata["games"]
seed_checks_in_area = checks_in_area.copy()

33
WebHostLib/upload.py
View File

@ -1,19 +1,22 @@
import base64
import json
import pickle
import typing
import uuid
import zipfile
from io import BytesIO
import zlib
from io import BytesIO
from flask import request, flash, redirect, url_for, session, render_template, Markup
from pony.orm import flush, select
from pony.orm import commit, flush, select, rollback
from pony.orm.core import TransactionIntegrityError
import MultiServer
from NetUtils import NetworkSlot, SlotType
from Utils import VersionException, __version__
from worlds.Files import AutoPatchRegister
from . import app
from .models import Seed, Room, Slot
from .models import Seed, Room, Slot, GameDataPackage
banned_zip_contents = (".sfc", ".z64", ".n64", ".sms", ".gb")
@ -78,6 +81,27 @@ def upload_zip_to_db(zfile: zipfile.ZipFile, owner=None, meta={"race": False}, s
# Load multi data.
if multidata:
decompressed_multidata = MultiServer.Context.decompress(multidata)
recompress = False
if "datapackage" in decompressed_multidata:
# strip datapackage from multidata, leaving only the checksums
game_data_packages: typing.List[GameDataPackage] = []
for game, game_data in decompressed_multidata["datapackage"].items():
if game_data.get("checksum"):
game_data_package = GameDataPackage(checksum=game_data["checksum"],
data=pickle.dumps(game_data))
decompressed_multidata["datapackage"][game] = {
"version": game_data.get("version", 0),
"checksum": game_data["checksum"]
}
recompress = True
try:
commit() # commit game data package
game_data_packages.append(game_data_package)
except TransactionIntegrityError:
del game_data_package
rollback()
if "slot_info" in decompressed_multidata:
for slot, slot_info in decompressed_multidata["slot_info"].items():
# Ignore Player Groups (e.g. item links)
@ -90,6 +114,9 @@ def upload_zip_to_db(zfile: zipfile.ZipFile, owner=None, meta={"race": False}, s
flush() # commit slots
if recompress:
multidata = multidata[0:1] + zlib.compress(pickle.dumps(decompressed_multidata), 9)
seed = Seed(multidata=multidata, spoiler=spoiler, slots=slots, owner=owner, meta=json.dumps(meta),
id=sid if sid else uuid.uuid4())
flush() # create seed

40
docs/network protocol.md
View File

@ -64,18 +64,19 @@ These packets are are sent from the multiworld server to the client. They are no
### RoomInfo
Sent to clients when they connect to an Archipelago server.
#### Arguments
| Name | Type | Notes |
| ---- | ---- | ----- |
| version | [NetworkVersion](#NetworkVersion) | Object denoting the version of Archipelago which the server is running. |
| tags | list\[str\] | Denotes special features or capabilities that the sender is capable of. Example: `WebHost` |
| password | bool | Denoted whether a password is required to join this room.|
| permissions | dict\[str, [Permission](#Permission)\[int\]\] | Mapping of permission name to [Permission](#Permission), keys are: "release", "collect" and "remaining". |
| hint_cost | int | The amount of points it costs to receive a hint from the server. |
| location_check_points | int | The amount of hint points you receive per item/location check completed. ||
| games | list\[str\] | List of games present in this multiworld. |
| datapackage_versions | dict\[str, int\] | Data versions of the individual games' data packages the server will send. Used to decide which games' caches are outdated. See [Data Package Contents](#Data-Package-Contents). |
| seed_name | str | uniquely identifying name of this generation |
| time | float | Unix time stamp of "now". Send for time synchronization if wanted for things like the DeathLink Bounce. |
| Name | Type | Notes |
|-----------------------|-----------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| version | [NetworkVersion](#NetworkVersion) | Object denoting the version of Archipelago which the server is running. |
| tags | list\[str\] | Denotes special features or capabilities that the sender is capable of. Example: `WebHost` |
| password | bool | Denoted whether a password is required to join this room. |
| permissions | dict\[str, [Permission](#Permission)\[int\]\] | Mapping of permission name to [Permission](#Permission), keys are: "release", "collect" and "remaining". |
| hint_cost | int | The amount of points it costs to receive a hint from the server. |
| location_check_points | int | The amount of hint points you receive per item/location check completed. |
| games | list\[str\] | List of games present in this multiworld. |
| datapackage_versions | dict\[str, int\] | Data versions of the individual games' data packages the server will send. Used to decide which games' caches are outdated. See [Data Package Contents](#Data-Package-Contents). **Deprecated. Use `datapackage_checksums` instead.** |
| datapackage_checksums | dict[str, str] | Checksum hash of the individual games' data packages the server will send. Used by newer clients to decide which games' caches are outdated. See [Data Package Contents](#Data-Package-Contents) for more information. |
| seed_name | str | Uniquely identifying name of this generation |
| time | float | Unix time stamp of "now". Send for time synchronization if wanted for things like the DeathLink Bounce. |
#### release
Dictates what is allowed when it comes to a player releasing their run. A release is an action which distributes the rest of the items in a player's run to those other players awaiting them.
@ -106,8 +107,8 @@ Dictates what is allowed when it comes to a player querying the items remaining
### ConnectionRefused
Sent to clients when the server refuses connection. This is sent during the initial connection handshake.
#### Arguments
| Name | Type | Notes |
| ---- | ---- | ----- |
| Name | Type | Notes |
|--------|-------------|--------------------------------------------------------------------------------------------------------------------------------------------------------|
| errors | list\[str\] | Optional. When provided, should contain any one of: `InvalidSlot`, `InvalidGame`, `IncompatibleVersion`, `InvalidPassword`, or `InvalidItemsHandling`. |
InvalidSlot indicates that the sent 'name' field did not match any auth entry on the server.
@ -644,11 +645,12 @@ Note:
#### GameData
GameData is a **dict** but contains these keys and values. It's broken out into another "type" for ease of documentation.
| Name | Type | Notes |
| ---- | ---- | ----- |
| item_name_to_id | dict[str, int] | Mapping of all item names to their respective ID. |
| location_name_to_id | dict[str, int] | Mapping of all location names to their respective ID. |
| version | int | Version number of this game's data |
| Name | Type | Notes |
|---------------------|----------------|-------------------------------------------------------------------------------------------------------------------------------|
| item_name_to_id | dict[str, int] | Mapping of all item names to their respective ID. |
| location_name_to_id | dict[str, int] | Mapping of all location names to their respective ID. |
| version | int | Version number of this game's data. Deprecated. Used by older clients to request an updated datapackage if cache is outdated. |
| checksum | str | A checksum hash of this game's data. |
### Tags
Tags are represented as a list of strings, the common Client tags follow:

9
docs/world api.md
View File

@ -364,14 +364,9 @@ class MyGameLocation(Location): # or from Locations import MyGameLocation
class MyGameWorld(World):
"""Insert description of the world/game here."""
game: str = "My Game" # name of the game/world
game = "My Game" # name of the game/world
option_definitions = mygame_options # options the player can set
topology_present: bool = True # show path to required location checks in spoiler
# data_version is used to signal that items, locations or their names
# changed. Set this to 0 during development so other games' clients do not
# cache any texts, then increase by 1 for each release that makes changes.
data_version = 0
topology_present = True # show path to required location checks in spoiler
# ID of first item and location, could be hard-coded but code may be easier
# to read with this as a propery.

3
requirements.txt
View File

@ -5,4 +5,5 @@ jellyfish>=0.9.0
jinja2>=3.1.2
schema>=0.7.5
kivy>=2.1.0
bsdiff4>=1.2.2
bsdiff4>=1.2.2
appdirs>=1.4.4

50
worlds/AutoWorld.py
View File

@ -1,20 +1,24 @@
from __future__ import annotations
import hashlib
import logging
import sys
import pathlib
from typing import Dict, FrozenSet, Set, Tuple, List, Optional, TextIO, Any, Callable, Type, Union, TYPE_CHECKING, \
ClassVar
import sys
from typing import Any, Callable, ClassVar, Dict, FrozenSet, List, Optional, Set, TYPE_CHECKING, TextIO, Tuple, Type, \
Union
from Options import AssembleOptions
from BaseClasses import CollectionState
from Options import AssembleOptions
if TYPE_CHECKING:
from BaseClasses import MultiWorld, Item, Location, Tutorial
from . import GamesPackage
class AutoWorldRegister(type):
world_types: Dict[str, Type[World]] = {}
__file__: str
zip_path: Optional[str]
def __new__(mcs, name: str, bases: Tuple[type, ...], dct: Dict[str, Any]) -> AutoWorldRegister:
if "web" in dct:
@ -154,9 +158,14 @@ class World(metaclass=AutoWorldRegister):
data_version: ClassVar[int] = 1
"""
increment this every time something in your world's names/id mappings changes.
While this is set to 0, this world's DataPackage is considered in testing mode and will be inserted to the multidata
and retrieved by clients on every connection.
Increment this every time something in your world's names/id mappings changes.
When this is set to 0, that world's DataPackage is considered in "testing mode", which signals to servers/clients
that it should not be cached, and clients should request that world's DataPackage every connection. Not
recommended for production-ready worlds.
Deprecated. Clients should utilize `checksum` to determine if DataPackage has changed since last connection and
request a new DataPackage, if necessary.
"""
required_client_version: Tuple[int, int, int] = (0, 1, 6)
@ -343,8 +352,35 @@ class World(metaclass=AutoWorldRegister):
def create_filler(self) -> "Item":
return self.create_item(self.get_filler_item_name())
@classmethod
def get_data_package_data(cls) -> "GamesPackage":
sorted_item_name_groups = {
name: sorted(cls.item_name_groups[name]) for name in sorted(cls.item_name_groups)
}
sorted_location_name_groups = {
name: sorted(cls.location_name_groups[name]) for name in sorted(cls.location_name_groups)
}
res: "GamesPackage" = {
# sorted alphabetically
"item_name_groups": sorted_item_name_groups,
"item_name_to_id": cls.item_name_to_id,
"location_name_groups": sorted_location_name_groups,
"location_name_to_id": cls.location_name_to_id,
"version": cls.data_version,
}
res["checksum"] = data_package_checksum(res)
return res
# any methods attached to this can be used as part of CollectionState,
# please use a prefix as all of them get clobbered together
class LogicMixin(metaclass=AutoLogicRegister):
pass
def data_package_checksum(data: "GamesPackage") -> str:
"""Calculates the data package checksum for a game from a dict"""
assert "checksum" not in data, "Checksum already in data"
assert sorted(data) == list(data), "Data not ordered"
from NetUtils import encode
return hashlib.sha1(encode(data).encode()).hexdigest()

18
worlds/__init__.py
View File

@ -20,14 +20,19 @@ if typing.TYPE_CHECKING:
from .AutoWorld import World
class GamesPackage(typing.TypedDict):
class GamesData(typing.TypedDict):
item_name_groups: typing.Dict[str, typing.List[str]]
item_name_to_id: typing.Dict[str, int]
location_name_groups: typing.Dict[str, typing.List[str]]
location_name_to_id: typing.Dict[str, int]
version: int
class GamesPackage(GamesData, total=False):
checksum: str
class DataPackage(typing.TypedDict):
version: int
games: typing.Dict[str, GamesPackage]
@ -75,14 +80,9 @@ games: typing.Dict[str, GamesPackage] = {}
from .AutoWorld import AutoWorldRegister
# Build the data package for each game.
for world_name, world in AutoWorldRegister.world_types.items():
games[world_name] = {
"item_name_to_id": world.item_name_to_id,
"location_name_to_id": world.location_name_to_id,
"version": world.data_version,
# seems clients don't actually want this. Keeping it here in case someone changes their mind.
# "item_name_groups": {name: tuple(items) for name, items in world.item_name_groups.items()}
}
games[world_name] = world.get_data_package_data()
lookup_any_item_id_to_name.update(world.item_id_to_name)
lookup_any_location_id_to_name.update(world.location_id_to_name)

6
worlds/bk_sudoku/__init__.py
View File

@ -1,7 +1,8 @@
from BaseClasses import Tutorial
from ..AutoWorld import World, WebWorld
from typing import Dict
from BaseClasses import Tutorial
from ..AutoWorld import WebWorld, World
class Bk_SudokuWebWorld(WebWorld):
settings_page = "games/Sudoku/info/en"
@ -24,6 +25,7 @@ class Bk_SudokuWorld(World):
"""
game = "Sudoku"
web = Bk_SudokuWebWorld()
data_version = 1
item_name_to_id: Dict[str, int] = {}
location_name_to_id: Dict[str, int] = {}

1
worlds/generic/__init__.py
View File

@ -42,6 +42,7 @@ class GenericWorld(World):
}
hidden = True
web = GenericWeb()
data_version = 1
def generate_early(self):
self.multiworld.player_types[self.player] = SlotType.spectator # mark as spectator

24
worlds/hylics2/__init__.py
View File

@ -2,8 +2,8 @@ import random
from typing import Dict, Any
from BaseClasses import Region, Entrance, Location, Item, Tutorial, ItemClassification
from worlds.generic.Rules import set_rule
from ..AutoWorld import World, WebWorld
from . import Items, Locations, Options, Rules, Exits
from . import Exits, Items, Locations, Options, Rules
from ..AutoWorld import WebWorld, World
class Hylics2Web(WebWorld):
@ -20,13 +20,13 @@ class Hylics2Web(WebWorld):
class Hylics2World(World):
"""
Hylics 2 is a surreal and unusual RPG, with a bizarre yet unique visual style. Play as Wayne,
Hylics 2 is a surreal and unusual RPG, with a bizarre yet unique visual style. Play as Wayne,
travel the world, and gather your allies to defeat the nefarious Gibby in his Hylemxylem!
"""
game: str = "Hylics 2"
web = Hylics2Web()
all_items = {**Items.item_table, **Items.gesture_item_table, **Items.party_item_table,
all_items = {**Items.item_table, **Items.gesture_item_table, **Items.party_item_table,
**Items.medallion_item_table}
all_locations = {**Locations.location_table, **Locations.tv_location_table, **Locations.party_location_table,
**Locations.medallion_location_table}
@ -37,7 +37,7 @@ class Hylics2World(World):
topology_present: bool = True
data_version: 1
data_version = 1
start_location = "Waynehouse"
@ -59,7 +59,7 @@ class Hylics2World(World):
def create_event(self, event: str):
return Hylics2Item(event, ItemClassification.progression_skip_balancing, None, self.player)
# set random starting location if option is enabled
def generate_early(self):
if self.multiworld.random_start[self.player]:
@ -76,7 +76,7 @@ class Hylics2World(World):
def generate_basic(self):
# create item pool
pool = []
# add regular items
for i, data in Items.item_table.items():
if data["count"] > 0:
@ -114,7 +114,7 @@ class Hylics2World(World):
gestures = list(Items.gesture_item_table.items())
tvs = list(Locations.tv_location_table.items())
# if Extra Items in Logic is enabled place CHARGE UP first and make sure it doesn't get
# if Extra Items in Logic is enabled place CHARGE UP first and make sure it doesn't get
# placed at Sage Airship: TV
if self.multiworld.extra_items_in_logic[self.player]:
tv = self.multiworld.random.choice(tvs)
@ -122,7 +122,7 @@ class Hylics2World(World):
while tv[1]["name"] == "Sage Airship: TV":
tv = self.multiworld.random.choice(tvs)
self.multiworld.get_location(tv[1]["name"], self.player)\
.place_locked_item(self.add_item(gestures[gest][1]["name"], gestures[gest][1]["classification"],
.place_locked_item(self.add_item(gestures[gest][1]["name"], gestures[gest][1]["classification"],
gestures[gest]))
gestures.remove(gestures[gest])
tvs.remove(tv)
@ -182,7 +182,7 @@ class Hylics2World(World):
16: Region("Sage Airship", self.player, self.multiworld),
17: Region("Hylemxylem", self.player, self.multiworld)
}
# create regions from table
for i, reg in region_table.items():
self.multiworld.regions.append(reg)
@ -214,7 +214,7 @@ class Hylics2World(World):
for i, data in Locations.tv_location_table.items():
region_table[data["region"]].locations\
.append(Hylics2Location(self.player, data["name"], i, region_table[data["region"]]))
# add party member locations if option is enabled
if self.multiworld.party_shuffle[self.player]:
for i, data in Locations.party_location_table.items():
@ -241,4 +241,4 @@ class Hylics2Location(Location):
class Hylics2Item(Item):
game: str = "Hylics 2"
game: str = "Hylics 2"

1
worlds/oribf/__init__.py
View File

@ -13,6 +13,7 @@ class OriBlindForest(World):
game: str = "Ori and the Blind Forest"
topology_present = True
data_version = 1
item_name_to_id = item_table
location_name_to_id = lookup_name_to_id