monorail
/
Archipelago
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

webhost update

This commit is contained in:
Fabian Dill 2020-07-10 17:42:22 +02:00
parent ccefa8dc9c
commit d190fe65c6
11 changed files with 177 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.gitignore vendored
View File

@ -1,17 +1,21 @@
.idea
.vscode
*_Spoiler.txt
*.bmbp
*.pyc
*.pyd
*.sfc
*.wixobj
*.lck
*multidata
*multisave
build
bundle/components.wxs
dist
README.html
.vs/
*multidata
*multisave
EnemizerCLI/
.mypy_cache/
RaceRom.py

5
MultiServer.py
View File

@ -104,6 +104,7 @@ class Context(Node):
with open(multidatapath, 'rb') as f:
self._load(json.loads(zlib.decompress(f.read()).decode("utf-8-sig")),
use_embedded_server_options)
self.data_filename = multidatapath
def _load(self, jsonobj: dict, use_embedded_server_options: bool):
@ -1177,7 +1178,7 @@ def parse_args() -> argparse.Namespace:
async def auto_shutdown(ctx, to_cancel=None):
await asyncio.sleep(ctx.auto_shutdown * 60)
await asyncio.sleep(ctx.auto_shutdown)
while ctx.running:
if not ctx.client_activity_timers.values():
asyncio.create_task(ctx.server.ws_server._close())
@ -1189,7 +1190,7 @@ async def auto_shutdown(ctx, to_cancel=None):
else:
newest_activity = max(ctx.client_activity_timers.values())
delta = datetime.datetime.now(datetime.timezone.utc) - newest_activity
seconds = ctx.auto_shutdown * 60 - delta.total_seconds()
seconds = ctx.auto_shutdown - delta.total_seconds()
if seconds < 0:
asyncio.create_task(ctx.server.ws_server._close())
ctx.running = False

57
WebHost.py
View File

@ -2,41 +2,17 @@ import os
import multiprocessing
import logging
from WebHost import app
from WebHost import app as raw_app
from waitress import serve
from WebHost.models import db, Room, db_session, select
from WebHost.models import db
from WebHost.autolauncher import autohost
configpath = "config.yaml"
def autohost(config: dict):
return
# not implemented yet. https://github.com/ponyorm/pony/issues/527
import time
from datetime import timedelta, datetime
def keep_running():
# db.bind(**config["PONY"])
# db.generate_mapping(check_tables=False)
while 1:
time.sleep(3)
with db_session:
rooms = select(
room for room in Room if
room.last_activity >= datetime.utcnow() - timedelta(hours=room.timeout))
logging.info(rooms)
import threading
threading.Thread(target=keep_running).start()
if __name__ == "__main__":
multiprocessing.freeze_support()
multiprocessing.set_start_method('spawn')
logging.basicConfig(format='[%(asctime)s] %(message)s', level=logging.INFO)
configpath = "config.yaml"
def get_app():
app = raw_app
if os.path.exists(configpath):
import yaml
with open(configpath) as c:
@ -45,8 +21,19 @@ if __name__ == "__main__":
logging.info(f"Updated config from {configpath}")
db.bind(**app.config["PONY"])
db.generate_mapping(create_tables=True)
if app.config["DEBUG"]:
return app
if __name__ == "__main__":
multiprocessing.freeze_support()
multiprocessing.set_start_method('spawn')
logging.basicConfig(format='[%(asctime)s] %(message)s', level=logging.INFO)
app = get_app()
if app.config["SELFLAUNCH"]:
autohost(app.config)
app.run(debug=True, port=app.config["PORT"])
else:
serve(app, port=app.config["PORT"], threads=app.config["WAITRESS_THREADS"])
if app.config["SELFHOST"]: # using WSGI, you just want to run get_app()
if app.config["DEBUG"]:
autohost(app.config)
app.run(debug=True, port=app.config["PORT"])
else:
serve(app, port=app.config["PORT"], threads=app.config["WAITRESS_THREADS"])

40
WebHost/__init__.py
View File

@ -2,9 +2,6 @@
So unless you're Berserker you need to include license information."""
import os
import logging
import typing
import multiprocessing
import threading
from pony.flask import Pony
@ -23,9 +20,12 @@ os.makedirs(LOGS_FOLDER, exist_ok=True)
def allowed_file(filename):
return filename.endswith(('multidata', ".zip"))
app = Flask(__name__)
Pony(app)
app.config["SELFHOST"] = True
app.config["SELFLAUNCH"] = True
app.config["DEBUG"] = False
app.config["PORT"] = 80
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
@ -47,7 +47,6 @@ cache = Cache(app)
Compress(app)
# this local cache is risky business if app hosting is done with subprocesses as it will not sync. Waitress is fine though
multiworlds = {}
@app.before_request
@ -57,29 +56,6 @@ def register_session():
session["_id"] = uuid4() # uniquely identify each session without needing a login
class MultiworldInstance():
def __init__(self, room: Room):
self.room_id = room.id
self.process: typing.Optional[multiprocessing.Process] = None
multiworlds[self.room_id] = self
def start(self):
if self.process and self.process.is_alive():
return False
logging.info(f"Spinning up {self.room_id}")
with db_session:
self.process = multiprocessing.Process(group=None, target=run_server_process,
args=(self.room_id, app.config["PONY"]),
name="MultiHost")
self.process.start()
def stop(self):
if self.process:
self.process.terminate()
self.process = None
@app.route('/seed/<uuid:seed>')
def view_seed(seed: UUID):
seed = Seed.get(id=seed)
@ -114,8 +90,6 @@ def display_log(room: UUID):
return Response(_read_log(os.path.join("logs", str(room) + ".txt")), mimetype="text/plain;charset=UTF-8")
processstartlock = threading.Lock()
@app.route('/hosted/<uuid:room>', methods=['GET', 'POST'])
def host_room(room: UUID):
@ -127,13 +101,9 @@ def host_room(room: UUID):
cmd = request.form["cmd"]
Command(room=room, commandtext=cmd)
commit()
with db_session:
multiworld = multiworlds.get(room.id, None)
if not multiworld:
multiworld = MultiworldInstance(room)
with processstartlock:
multiworld.start()
with db_session:
room.last_activity = datetime.utcnow() # will trigger a spinup, if it's not already running
return render_template("host_room.html", room=room)

120
WebHost/autolauncher.py Normal file
View File

@ -0,0 +1,120 @@
from __future__ import annotations
import logging
import multiprocessing
from datetime import timedelta, datetime
import sys
import typing
from pony.orm import db_session, select
class CommonLocker():
"""Uses a file lock to signal that something is already running"""
def __init__(self, lockname: str):
self.lockname = lockname
self.lockfile = f"./{self.lockname}.lck"
class AlreadyRunningException(Exception):
pass
if sys.platform == 'win32':
import os
class Locker(CommonLocker):
def __enter__(self):
try:
if os.path.exists(self.lockfile):
os.unlink(self.lockfile)
self.fp = os.open(
self.lockfile, os.O_CREAT | os.O_EXCL | os.O_RDWR)
except OSError as e:
raise AlreadyRunningException() from e
def __exit__(self, _type, value, tb):
fp = getattr(self, "fp", None)
if fp:
os.close(self.fp)
os.unlink(self.lockfile)
else: # unix
import fcntl
class Locker(CommonLocker):
def __enter__(self):
self.fp = open(self.lockfile, "rb")
try:
fcntl.flock(self.fp.fileno(), fcntl.LOCK_EX)
except OSError as e:
raise AlreadyRunningException() from e
def __exit__(self, _type, value, tb):
fcntl.flock(self.fp.fileno(), fcntl.LOCK_UN)
self.fp.close()
def launch_room(room: Room, config: dict):
# requires db_session!
if room.last_activity >= datetime.utcnow() - timedelta(minutes=room.timeout):
multiworld = multiworlds.get(room.id, None)
if not multiworld:
multiworld = MultiworldInstance(room, config)
multiworld.start()
def autohost(config: dict):
import time
def keep_running():
try:
with Locker("autohost"):
# db.bind(**config["PONY"])
# db.generate_mapping(check_tables=False)
while 1:
time.sleep(3)
with db_session:
rooms = select(
room for room in Room if
room.last_activity >= datetime.utcnow() - timedelta(days=3))
for room in rooms:
launch_room(room, config)
except AlreadyRunningException:
pass
import threading
threading.Thread(target=keep_running).start()
multiworlds = {}
class MultiworldInstance():
def __init__(self, room: Room, config: dict):
self.room_id = room.id
self.process: typing.Optional[multiprocessing.Process] = None
multiworlds[self.room_id] = self
self.ponyconfig = config["PONY"]
def start(self):
if self.process and self.process.is_alive():
return False
logging.info(f"Spinning up {self.room_id}")
self.process = multiprocessing.Process(group=None, target=run_server_process,
args=(self.room_id, self.ponyconfig),
name="MultiHost")
self.process.start()
def stop(self):
if self.process:
self.process.terminate()
self.process = None
from .models import Room
from .customserver import run_server_process

18
WebHost/customserver.py
View File

@ -1,3 +1,5 @@
from __future__ import annotations
import functools
import logging
import os
@ -8,7 +10,7 @@ import threading
import time
import random
from WebHost import LOGS_FOLDER
from .models import *
from MultiServer import Context, server, auto_shutdown, ServerCommandProcessor, ClientMessageProcessor
@ -16,6 +18,7 @@ from Utils import get_public_ipv4, get_public_ipv6
class CustomClientMessageProcessor(ClientMessageProcessor):
ctx: WebHostContext
def _cmd_video(self, platform, user):
"""Set a link for your name in the WebHost tracker pointing to a video stream"""
if platform.lower().startswith("t"): # twitch
@ -28,7 +31,6 @@ class CustomClientMessageProcessor(ClientMessageProcessor):
# inject
import MultiServer
MultiServer.client_message_processor = CustomClientMessageProcessor
del (MultiServer)
@ -80,7 +82,10 @@ class WebHostContext(Context):
@db_session
def _save(self) -> bool:
Room.get(id=self.room_id).multisave = self.get_save()
room = Room.get(id=self.room_id)
room.multisave = self.get_save()
# saving only occurs on activity, so we can "abuse" this information to mark this as last_activity
room.last_activity = datetime.utcnow()
return True
def get_save(self) -> dict:
@ -104,7 +109,6 @@ def run_server_process(room_id, ponyconfig: dict):
logging.FileHandler(os.path.join(LOGS_FOLDER, f"{room_id}.txt"), 'a', 'utf-8-sig')])
ctx = WebHostContext()
ctx.load(room_id)
ctx.auto_shutdown = 24 * 60 * 60 # 24 hours
ctx.init_save()
try:
@ -126,9 +130,13 @@ def run_server_process(room_id, ponyconfig: dict):
room.last_port = socketname[1]
elif wssocket.family == socket.AF_INET:
logging.info(f'Hosting game at {get_public_ipv4()}:{socketname[1]}')
ctx.auto_shutdown = 6 * 60
with db_session:
ctx.auto_shutdown = Room.get(id=room_id).timeout
ctx.shutdown_task = asyncio.create_task(auto_shutdown(ctx, []))
await ctx.shutdown_task
logging.info("Shutting down")
asyncio.run(main())
from WebHost import LOGS_FOLDER

2
WebHost/models.py
View File

@ -21,7 +21,7 @@ class Room(db.Entity):
seed = Required('Seed', index=True)
multisave = Optional(Json, lazy=True)
show_spoiler = Required(int, default=0) # 0 -> never, 1 -> after completion, -> 2 always
timeout = Required(int, default=lambda: 6)
timeout = Required(int, default=lambda: 6 * 60 * 60) # seconds since last activity to shutdown
tracker = Optional(UUID, index=True)
last_port = Optional(int, default=lambda: 0)

4
WebHost/requirements.txt
View File

@ -3,4 +3,6 @@ pony>=0.7.13
waitress>=1.4.4
flask-caching>=1.9.0
Flask-Autoversion>=0.2.0
Flask-Compress>=1.5.0
Flask-Compress>=1.5.0
redis>=3.5.3
rq>=1.4.3

2
WebHost/templates/host_room.html
View File

@ -9,7 +9,7 @@
{% if room.tracker %}
This room has a <a href="{{ url_for("get_tracker", tracker=room.tracker) }}">Multiworld Tracker</a> enabled.<br>
{% endif %}
This room will be closed after {{ room.timeout }} hours of inactivity. Should you wish to continue later,
This room will be closed after {{ room.timeout//60//60 }} hours of inactivity. Should you wish to continue later,
you can simply refresh this page and the server will be started again.<br>
{% if room.owner == session["_id"] %}
<form method=post>

2
WebHost/templates/landing.html
View File

@ -30,6 +30,8 @@
</p>
</div>
<div>
<p class="lead">This webpage is still under heavy construction. Database may be wiped as I see fit
and some stuff may be broken.</p>
<p class="lead">This is a randomizer for The Legend of Zelda: A Link to the Past.</p>
<p class="lead">It is a multiworld, meaning items get shuffled across multiple players' worlds
which get exchanged on pickup through the internet.</p>

2
host.yaml
View File

@ -34,7 +34,7 @@ server_options:
# "goal" -> client can ask for remaining items after goal completion
# warning: only Berserker's Multiworld clients of version 2.1+ send game beaten information
remaining_mode: "goal"
# automatically shut down the server after this many minutes without new location checks, 0 to keep running
# automatically shut down the server after this many seconds without new location checks, 0 to keep running
auto_shutdown: 0
#options for MultiMystery.py
multi_mystery_options: