diff --git a/.gitignore b/.gitignore index 4166b0ae..ecd316cb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,17 +1,21 @@ .idea .vscode + *_Spoiler.txt *.bmbp *.pyc +*.pyd *.sfc *.wixobj +*.lck +*multidata +*multisave + build bundle/components.wxs dist README.html .vs/ -*multidata -*multisave EnemizerCLI/ .mypy_cache/ RaceRom.py diff --git a/MultiServer.py b/MultiServer.py index 6354a2d2..ce518614 100644 --- a/MultiServer.py +++ b/MultiServer.py @@ -104,6 +104,7 @@ class Context(Node): with open(multidatapath, 'rb') as f: self._load(json.loads(zlib.decompress(f.read()).decode("utf-8-sig")), use_embedded_server_options) + self.data_filename = multidatapath def _load(self, jsonobj: dict, use_embedded_server_options: bool): @@ -1177,7 +1178,7 @@ def parse_args() -> argparse.Namespace: async def auto_shutdown(ctx, to_cancel=None): - await asyncio.sleep(ctx.auto_shutdown * 60) + await asyncio.sleep(ctx.auto_shutdown) while ctx.running: if not ctx.client_activity_timers.values(): asyncio.create_task(ctx.server.ws_server._close()) @@ -1189,7 +1190,7 @@ async def auto_shutdown(ctx, to_cancel=None): else: newest_activity = max(ctx.client_activity_timers.values()) delta = datetime.datetime.now(datetime.timezone.utc) - newest_activity - seconds = ctx.auto_shutdown * 60 - delta.total_seconds() + seconds = ctx.auto_shutdown - delta.total_seconds() if seconds < 0: asyncio.create_task(ctx.server.ws_server._close()) ctx.running = False diff --git a/WebHost.py b/WebHost.py index 5e635ed3..88e994bf 100644 --- a/WebHost.py +++ b/WebHost.py @@ -2,41 +2,17 @@ import os import multiprocessing import logging -from WebHost import app +from WebHost import app as raw_app from waitress import serve -from WebHost.models import db, Room, db_session, select +from WebHost.models import db +from WebHost.autolauncher import autohost + +configpath = "config.yaml" - -def autohost(config: dict): - return - # not implemented yet. https://github.com/ponyorm/pony/issues/527 - import time - from datetime import timedelta, datetime - - def keep_running(): - # db.bind(**config["PONY"]) - # db.generate_mapping(check_tables=False) - while 1: - time.sleep(3) - with db_session: - rooms = select( - room for room in Room if - room.last_activity >= datetime.utcnow() - timedelta(hours=room.timeout)) - logging.info(rooms) - - import threading - threading.Thread(target=keep_running).start() - - -if __name__ == "__main__": - multiprocessing.freeze_support() - multiprocessing.set_start_method('spawn') - logging.basicConfig(format='[%(asctime)s] %(message)s', level=logging.INFO) - - configpath = "config.yaml" - +def get_app(): + app = raw_app if os.path.exists(configpath): import yaml with open(configpath) as c: @@ -45,8 +21,19 @@ if __name__ == "__main__": logging.info(f"Updated config from {configpath}") db.bind(**app.config["PONY"]) db.generate_mapping(create_tables=True) - if app.config["DEBUG"]: + return app + + +if __name__ == "__main__": + multiprocessing.freeze_support() + multiprocessing.set_start_method('spawn') + logging.basicConfig(format='[%(asctime)s] %(message)s', level=logging.INFO) + app = get_app() + if app.config["SELFLAUNCH"]: autohost(app.config) - app.run(debug=True, port=app.config["PORT"]) - else: - serve(app, port=app.config["PORT"], threads=app.config["WAITRESS_THREADS"]) + if app.config["SELFHOST"]: # using WSGI, you just want to run get_app() + if app.config["DEBUG"]: + autohost(app.config) + app.run(debug=True, port=app.config["PORT"]) + else: + serve(app, port=app.config["PORT"], threads=app.config["WAITRESS_THREADS"]) diff --git a/WebHost/__init__.py b/WebHost/__init__.py index 251e4fb8..be56d2f7 100644 --- a/WebHost/__init__.py +++ b/WebHost/__init__.py @@ -2,9 +2,6 @@ So unless you're Berserker you need to include license information.""" import os -import logging -import typing -import multiprocessing import threading from pony.flask import Pony @@ -23,9 +20,12 @@ os.makedirs(LOGS_FOLDER, exist_ok=True) def allowed_file(filename): return filename.endswith(('multidata', ".zip")) + app = Flask(__name__) Pony(app) +app.config["SELFHOST"] = True +app.config["SELFLAUNCH"] = True app.config["DEBUG"] = False app.config["PORT"] = 80 app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER @@ -47,7 +47,6 @@ cache = Cache(app) Compress(app) # this local cache is risky business if app hosting is done with subprocesses as it will not sync. Waitress is fine though -multiworlds = {} @app.before_request @@ -57,29 +56,6 @@ def register_session(): session["_id"] = uuid4() # uniquely identify each session without needing a login -class MultiworldInstance(): - def __init__(self, room: Room): - self.room_id = room.id - self.process: typing.Optional[multiprocessing.Process] = None - multiworlds[self.room_id] = self - - def start(self): - if self.process and self.process.is_alive(): - return False - - logging.info(f"Spinning up {self.room_id}") - with db_session: - self.process = multiprocessing.Process(group=None, target=run_server_process, - args=(self.room_id, app.config["PONY"]), - name="MultiHost") - self.process.start() - - def stop(self): - if self.process: - self.process.terminate() - self.process = None - - @app.route('/seed/') def view_seed(seed: UUID): seed = Seed.get(id=seed) @@ -114,8 +90,6 @@ def display_log(room: UUID): return Response(_read_log(os.path.join("logs", str(room) + ".txt")), mimetype="text/plain;charset=UTF-8") -processstartlock = threading.Lock() - @app.route('/hosted/', methods=['GET', 'POST']) def host_room(room: UUID): @@ -127,13 +101,9 @@ def host_room(room: UUID): cmd = request.form["cmd"] Command(room=room, commandtext=cmd) commit() - with db_session: - multiworld = multiworlds.get(room.id, None) - if not multiworld: - multiworld = MultiworldInstance(room) - with processstartlock: - multiworld.start() + with db_session: + room.last_activity = datetime.utcnow() # will trigger a spinup, if it's not already running return render_template("host_room.html", room=room) diff --git a/WebHost/autolauncher.py b/WebHost/autolauncher.py new file mode 100644 index 00000000..76bf5486 --- /dev/null +++ b/WebHost/autolauncher.py @@ -0,0 +1,120 @@ +from __future__ import annotations +import logging +import multiprocessing +from datetime import timedelta, datetime +import sys +import typing + +from pony.orm import db_session, select + + +class CommonLocker(): + """Uses a file lock to signal that something is already running""" + + def __init__(self, lockname: str): + self.lockname = lockname + self.lockfile = f"./{self.lockname}.lck" + + +class AlreadyRunningException(Exception): + pass + + +if sys.platform == 'win32': + import os + + + class Locker(CommonLocker): + def __enter__(self): + try: + if os.path.exists(self.lockfile): + os.unlink(self.lockfile) + self.fp = os.open( + self.lockfile, os.O_CREAT | os.O_EXCL | os.O_RDWR) + except OSError as e: + raise AlreadyRunningException() from e + + def __exit__(self, _type, value, tb): + fp = getattr(self, "fp", None) + if fp: + os.close(self.fp) + os.unlink(self.lockfile) +else: # unix + import fcntl + + + class Locker(CommonLocker): + def __enter__(self): + self.fp = open(self.lockfile, "rb") + try: + fcntl.flock(self.fp.fileno(), fcntl.LOCK_EX) + except OSError as e: + raise AlreadyRunningException() from e + + def __exit__(self, _type, value, tb): + fcntl.flock(self.fp.fileno(), fcntl.LOCK_UN) + self.fp.close() + + +def launch_room(room: Room, config: dict): + # requires db_session! + if room.last_activity >= datetime.utcnow() - timedelta(minutes=room.timeout): + multiworld = multiworlds.get(room.id, None) + if not multiworld: + multiworld = MultiworldInstance(room, config) + + multiworld.start() + + +def autohost(config: dict): + import time + + def keep_running(): + try: + with Locker("autohost"): + # db.bind(**config["PONY"]) + # db.generate_mapping(check_tables=False) + while 1: + time.sleep(3) + with db_session: + rooms = select( + room for room in Room if + room.last_activity >= datetime.utcnow() - timedelta(days=3)) + for room in rooms: + launch_room(room, config) + + except AlreadyRunningException: + pass + + import threading + threading.Thread(target=keep_running).start() + + +multiworlds = {} + + +class MultiworldInstance(): + def __init__(self, room: Room, config: dict): + self.room_id = room.id + self.process: typing.Optional[multiprocessing.Process] = None + multiworlds[self.room_id] = self + self.ponyconfig = config["PONY"] + + def start(self): + if self.process and self.process.is_alive(): + return False + + logging.info(f"Spinning up {self.room_id}") + self.process = multiprocessing.Process(group=None, target=run_server_process, + args=(self.room_id, self.ponyconfig), + name="MultiHost") + self.process.start() + + def stop(self): + if self.process: + self.process.terminate() + self.process = None + + +from .models import Room +from .customserver import run_server_process diff --git a/WebHost/customserver.py b/WebHost/customserver.py index a80f8b0d..7dc845b4 100644 --- a/WebHost/customserver.py +++ b/WebHost/customserver.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import functools import logging import os @@ -8,7 +10,7 @@ import threading import time import random -from WebHost import LOGS_FOLDER + from .models import * from MultiServer import Context, server, auto_shutdown, ServerCommandProcessor, ClientMessageProcessor @@ -16,6 +18,7 @@ from Utils import get_public_ipv4, get_public_ipv6 class CustomClientMessageProcessor(ClientMessageProcessor): + ctx: WebHostContext def _cmd_video(self, platform, user): """Set a link for your name in the WebHost tracker pointing to a video stream""" if platform.lower().startswith("t"): # twitch @@ -28,7 +31,6 @@ class CustomClientMessageProcessor(ClientMessageProcessor): # inject import MultiServer - MultiServer.client_message_processor = CustomClientMessageProcessor del (MultiServer) @@ -80,7 +82,10 @@ class WebHostContext(Context): @db_session def _save(self) -> bool: - Room.get(id=self.room_id).multisave = self.get_save() + room = Room.get(id=self.room_id) + room.multisave = self.get_save() + # saving only occurs on activity, so we can "abuse" this information to mark this as last_activity + room.last_activity = datetime.utcnow() return True def get_save(self) -> dict: @@ -104,7 +109,6 @@ def run_server_process(room_id, ponyconfig: dict): logging.FileHandler(os.path.join(LOGS_FOLDER, f"{room_id}.txt"), 'a', 'utf-8-sig')]) ctx = WebHostContext() ctx.load(room_id) - ctx.auto_shutdown = 24 * 60 * 60 # 24 hours ctx.init_save() try: @@ -126,9 +130,13 @@ def run_server_process(room_id, ponyconfig: dict): room.last_port = socketname[1] elif wssocket.family == socket.AF_INET: logging.info(f'Hosting game at {get_public_ipv4()}:{socketname[1]}') - ctx.auto_shutdown = 6 * 60 + with db_session: + ctx.auto_shutdown = Room.get(id=room_id).timeout ctx.shutdown_task = asyncio.create_task(auto_shutdown(ctx, [])) await ctx.shutdown_task logging.info("Shutting down") asyncio.run(main()) + + +from WebHost import LOGS_FOLDER diff --git a/WebHost/models.py b/WebHost/models.py index 1dd49633..58a749b5 100644 --- a/WebHost/models.py +++ b/WebHost/models.py @@ -21,7 +21,7 @@ class Room(db.Entity): seed = Required('Seed', index=True) multisave = Optional(Json, lazy=True) show_spoiler = Required(int, default=0) # 0 -> never, 1 -> after completion, -> 2 always - timeout = Required(int, default=lambda: 6) + timeout = Required(int, default=lambda: 6 * 60 * 60) # seconds since last activity to shutdown tracker = Optional(UUID, index=True) last_port = Optional(int, default=lambda: 0) diff --git a/WebHost/requirements.txt b/WebHost/requirements.txt index d82c5314..0e14504f 100644 --- a/WebHost/requirements.txt +++ b/WebHost/requirements.txt @@ -3,4 +3,6 @@ pony>=0.7.13 waitress>=1.4.4 flask-caching>=1.9.0 Flask-Autoversion>=0.2.0 -Flask-Compress>=1.5.0 \ No newline at end of file +Flask-Compress>=1.5.0 +redis>=3.5.3 +rq>=1.4.3 \ No newline at end of file diff --git a/WebHost/templates/host_room.html b/WebHost/templates/host_room.html index 7946f44c..3b0b5212 100644 --- a/WebHost/templates/host_room.html +++ b/WebHost/templates/host_room.html @@ -9,7 +9,7 @@ {% if room.tracker %} This room has a Multiworld Tracker enabled.
{% endif %} - This room will be closed after {{ room.timeout }} hours of inactivity. Should you wish to continue later, + This room will be closed after {{ room.timeout//60//60 }} hours of inactivity. Should you wish to continue later, you can simply refresh this page and the server will be started again.
{% if room.owner == session["_id"] %}
diff --git a/WebHost/templates/landing.html b/WebHost/templates/landing.html index dde05792..4efd0b6b 100644 --- a/WebHost/templates/landing.html +++ b/WebHost/templates/landing.html @@ -30,6 +30,8 @@

+

This webpage is still under heavy construction. Database may be wiped as I see fit + and some stuff may be broken.

This is a randomizer for The Legend of Zelda: A Link to the Past.

It is a multiworld, meaning items get shuffled across multiple players' worlds which get exchanged on pickup through the internet.

diff --git a/host.yaml b/host.yaml index c5bf44af..748cdeaa 100644 --- a/host.yaml +++ b/host.yaml @@ -34,7 +34,7 @@ server_options: # "goal" -> client can ask for remaining items after goal completion # warning: only Berserker's Multiworld clients of version 2.1+ send game beaten information remaining_mode: "goal" - # automatically shut down the server after this many minutes without new location checks, 0 to keep running + # automatically shut down the server after this many seconds without new location checks, 0 to keep running auto_shutdown: 0 #options for MultiMystery.py multi_mystery_options: