monorail
/
Archipelago
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Database-backed Webhosting

This commit is contained in:
Fabian Dill 2020-06-20 20:03:06 +02:00
parent 7e3ee8101f
commit 9e18c6f1cd
11 changed files with 325 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
MultiServer.py
View File

@ -166,22 +166,24 @@ class Context(Node):
logging.error('No save data found, starting a new game')
except Exception as e:
logging.exception(e)
self._start_async_saving()
if not self.auto_saver_thread:
def save_regularly():
import time
while self.running:
time.sleep(self.auto_save_interval)
if self.save_dirty:
logging.debug("Saving multisave via thread.")
self.save_dirty = False
self._save()
def _start_async_saving(self):
if not self.auto_saver_thread:
def save_regularly():
import time
while self.running:
time.sleep(self.auto_save_interval)
if self.save_dirty:
logging.debug("Saving multisave via thread.")
self.save_dirty = False
self._save()
self.auto_saver_thread = threading.Thread(target=save_regularly, daemon=True)
self.auto_saver_thread.start()
self.auto_saver_thread = threading.Thread(target=save_regularly, daemon=True)
self.auto_saver_thread.start()
import atexit
atexit.register(self._save) # make sure we save on exit too
import atexit
atexit.register(self._save) # make sure we save on exit too
def get_save(self) -> dict:
d = {

31
WebHost.py Normal file
View File

@ -0,0 +1,31 @@
import os
import multiprocessing
import logging
from WebHost import app
from waitress import serve
from WebHost.models import db
DEBUG = False
if __name__ == "__main__":
multiprocessing.freeze_support()
multiprocessing.set_start_method('spawn')
logging.basicConfig(format='[%(asctime)s] %(message)s', level=logging.INFO)
configpath = "config.yaml"
if os.path.exists(configpath):
import yaml
with open(configpath) as c:
app.config.update(yaml.safe_load(c))
logging.info(f"Updated config from {configpath}")
db.bind(**app.config["PONY"])
db.generate_mapping(create_tables=True)
if DEBUG:
app.run(debug=True)
else:
serve(app, port=80, threads=1)

126
WebHost/__init__.py
View File

@ -1,21 +1,19 @@
# module has yet to be made capable of running in multiple processes
import os
import logging
import threading
import typing
import multiprocessing
from pony.orm import Database, db_session
from flask import Flask, flash, request, redirect, url_for, render_template, Response
from werkzeug.utils import secure_filename
import threading
import json
import zlib
from pony.orm import db_session, commit
from pony.flask import Pony
from flask import Flask, flash, request, redirect, url_for, render_template, Response, session
from .models import *
UPLOAD_FOLDER = os.path.relpath('uploads')
LOGS_FOLDER = os.path.relpath('logs')
multidata_folder = os.path.join(UPLOAD_FOLDER, "multidata")
os.makedirs(multidata_folder, exist_ok=True)
os.makedirs(LOGS_FOLDER, exist_ok=True)
@ -24,36 +22,43 @@ def allowed_file(filename):
app = Flask(__name__)
Pony(app)
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.config['MAX_CONTENT_LENGTH'] = 1 * 1024 * 1024 # 1 megabyte limit
# if you want persistent sessions on your server, make sure you make this a constant in your config.yaml
app.config["SECRET_KEY"] = os.urandom(32)
app.config['SESSION_PERMANENT'] = True
app.config["PONY"] = {
'provider': 'sqlite',
'filename': 'db.db3',
'filename': os.path.abspath('db.db3'),
'create_db': True
}
db = Database()
name = "localhost"
multiworlds = {}
class Multiworld():
def __init__(self, multidata: str):
self.multidata = multidata
@app.before_first_request
def register_session():
session.permanent = True # technically 31 days after the last visit
if not session.get("_id", None):
session["_id"] = uuid4() # uniquely identify each session without needing a login
class MultiworldInstance():
def __init__(self, room: Room):
self.room_id = room.id
self.process: typing.Optional[multiprocessing.Process] = None
multiworlds[multidata] = self
multiworlds[self.room_id] = self
def start(self):
if self.process and self.process.is_alive():
return False
logging.info(f"Spinning up {self.multidata}")
self.process = multiprocessing.Process(group=None, target=run_server_process,
args=(self.multidata,),
name="MultiHost")
logging.info(f"Spinning up {self.room_id}")
with db_session:
self.process = multiprocessing.Process(group=None, target=run_server_process,
args=(self.room_id, app.config["PONY"]),
name="MultiHost")
self.process.start()
def stop(self):
@ -67,21 +72,40 @@ def upload_multidata():
# check if the post request has the file part
if 'file' not in request.files:
flash('No file part')
return redirect(request.url)
file = request.files['file']
# if user does not select file, browser also
# submit an empty part without filename
if file.filename == '':
flash('No selected file')
return redirect(request.url)
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
file.save(os.path.join(multidata_folder, filename))
return redirect(url_for('host_multidata',
filename=filename))
else:
file = request.files['file']
# if user does not select file, browser also
# submit an empty part without filename
if file.filename == '':
flash('No selected file')
elif file and allowed_file(file.filename):
try:
multidata = json.loads(zlib.decompress(file.read()).decode("utf-8-sig"))
except:
flash("Could not load multidata. File may be corrupted or incompatible.")
else:
seed = Seed(multidata=multidata)
commit() # place into DB and generate ids
return redirect(url_for("view_seed", seed=seed.id))
else:
flash("Not recognized file format. Awaiting a .multidata file.")
return render_template("upload_multidata.html")
@app.route('/seed/<int:seed>')
def view_seed(seed: int):
seed = Seed.get(id=seed)
return render_template("view_seed.html", seed=seed)
@app.route('/new_room/<int:seed>')
def new_room(seed: int):
seed = Seed.get(id=seed)
room = Room(seed=seed, owner=session["_id"])
commit()
return redirect(url_for("host_room", room=room.id))
def _read_log(path: str):
if os.path.exists(path):
with open(path) as log:
@ -91,34 +115,32 @@ def _read_log(path: str):
f"Likely a crash during spinup of multiworld instance or it is still spinning up."
@app.route('/log/<filename>')
def display_log(filename: str):
filename = secure_filename(filename)
@app.route('/log/<int:room>')
def display_log(room: int):
# noinspection PyTypeChecker
return Response(_read_log(os.path.join("logs", filename + ".txt")), mimetype="text/plain;charset=UTF-8")
return Response(_read_log(os.path.join("logs", str(room) + ".txt")), mimetype="text/plain;charset=UTF-8")
processstartlock = threading.Lock()
@app.route('/hosted/<filename>')
def host_multidata(filename: str):
@app.route('/hosted/<int:room>', methods=['GET', 'POST'])
def host_room(room: int):
room = Room.get(id=room)
if request.method == "POST":
if room.owner == session["_id"]:
cmd = request.form["cmd"]
Command(room=room, commandtext=cmd)
commit()
with db_session:
multiworld = multiworlds.get(filename, None)
multiworld = multiworlds.get(room.id, None)
if not multiworld:
multiworld = Multiworld(filename)
multiworld = MultiworldInstance(room)
with processstartlock:
multiworld.start()
with processstartlock:
multiworld.start()
return render_template("host_multidata.html", filename=filename)
return render_template("host_room.html", room=room)
from WebHost.customserver import run_server_process
if __name__ == "__main__":
multiprocessing.freeze_support()
multiprocessing.set_start_method('spawn')
db.bind(**app.config["PONY"])
db.generate_mapping(create_tables=True)
app.run(debug=True)

75
WebHost/customserver.py
View File

@ -1,21 +1,74 @@
import functools
import logging
import os
import sys
import websockets
import asyncio
import socket
import threading
import time
from WebHost import LOGS_FOLDER, multidata_folder
from WebHost import LOGS_FOLDER
from .models import *
from MultiServer import Context, server, auto_shutdown, ServerCommandProcessor
from Utils import get_public_ipv4, get_public_ipv6
def run_server_process(multidata: str):
class DBCommandProcessor(ServerCommandProcessor):
def output(self, text: str):
logging.info(text)
class WebHostContext(Context):
def __init__(self):
super(WebHostContext, self).__init__("", 0, "", 1, 40, True, "enabled", "enabled", 0)
def listen_to_db_commands(self):
cmdprocessor = DBCommandProcessor(self)
while self.running:
with db_session:
commands = select(command for command in Command if command.room.id == self.room_id)
if commands:
for command in commands:
cmdprocessor(command.commandtext)
command.delete()
commit()
time.sleep(5)
@db_session
def load(self, room_id: int):
self.room_id = room_id
return self._load(Room.get(id=room_id).seed.multidata, True)
@db_session
def init_save(self, enabled: bool = True):
self.saving = enabled
if self.saving:
existings_savegame = Room.get(id=self.room_id).multisave
if existings_savegame:
self.set_save(existings_savegame)
self._start_async_saving()
threading.Thread(target=self.listen_to_db_commands, daemon=True).start()
@db_session
def _save(self) -> bool:
Room.get(id=self.room_id).multisave = self.get_save()
return True
def run_server_process(room_id, ponyconfig: dict):
# establish DB connection for multidata and multisave
db.bind(**ponyconfig)
db.generate_mapping(check_tables=False)
async def main():
logging.basicConfig(format='[%(asctime)s] %(message)s',
level=logging.INFO,
filename=os.path.join(LOGS_FOLDER, multidata + ".txt"))
ctx = Context("", 0, "", 1, 1000,
True, "enabled", "goal", 0)
ctx.load(os.path.join(multidata_folder, multidata), True)
filename=os.path.join(LOGS_FOLDER, f"{room_id}.txt"))
ctx = WebHostContext()
ctx.load(room_id)
ctx.auto_shutdown = 24 * 60 * 60 # 24 hours
ctx.init_save()
@ -34,10 +87,4 @@ def run_server_process(multidata: str):
await ctx.shutdown_task
logging.info("Shutting down")
import asyncio
if ".." not in sys.path:
sys.path.append("..")
from MultiServer import Context, server, auto_shutdown
from Utils import get_public_ipv4, get_public_ipv6
import socket
asyncio.run(main())

49
WebHost/models.py Normal file
View File

@ -0,0 +1,49 @@
from datetime import datetime
from uuid import UUID, uuid4
from pony.orm import *
db = Database()
class Patch(db.Entity):
id = PrimaryKey(int, auto=True)
data = Required(buffer)
simple_seed = Required('Seed')
class Room(db.Entity):
id = PrimaryKey(int, auto=True)
last_activity = Required(datetime, default=lambda: datetime.utcnow())
owner = Required(UUID)
commands = Set('Command')
host_jobs = Set('HostJob')
seed = Required('Seed')
multisave = Optional(Json)
class HostJob(db.Entity):
id = PrimaryKey(int, auto=True)
sockets = Set('Socket')
room = Required(Room)
scheduler_id = Required(int, unique=True)
class Socket(db.Entity):
port = PrimaryKey(int)
ipv6 = Required(bool)
host_job = Required(HostJob)
class Seed(db.Entity):
id = PrimaryKey(int, auto=True)
rooms = Set(Room)
multidata = Optional(Json)
creation_time = Required(datetime, default=lambda: datetime.utcnow())
patches = Set(Patch)
spoiler = Optional(str)
class Command(db.Entity):
id = PrimaryKey(int, auto=True)
room = Required(Room)
commandtext = Required(str)

9
WebHost/run.py
View File

@ -1,9 +0,0 @@
from waitress import serve
import multiprocessing
from __init__ import app
if __name__ == "__main__":
multiprocessing.freeze_support()
multiprocessing.set_start_method('spawn')
serve(app, port=80, threads=1)

30
WebHost/templates/host_multidata.html
View File

@ -1,30 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Multiworld {{ filename }}</title>
</head>
<body>
Log:
<div id="logger"></div>
<script>
var xmlhttp = new XMLHttpRequest();
var url = '{{ url_for('display_log', filename = filename) }}';
xmlhttp.onreadystatechange = function () {
if (this.readyState == 4 && this.status == 200) {
document.getElementById("logger").innerText = this.responseText;
}
};
function request_new() {
xmlhttp.open("GET", url, true);
xmlhttp.send();
}
request_new();
window.setInterval(request_new, 3000);
</script>
</body>
</html>

36
WebHost/templates/host_room.html Normal file
View File

@ -0,0 +1,36 @@
{% extends 'layout.html' %}
{% block head %}
<title>Multiworld {{ room.id }}</title>
{% endblock %}
{% block body %}
Room created from <a href="{{ url_for("view_seed", seed=room.seed.id) }}">Seed #{{ room.seed.id }}</a><br>
{% if room.owner == session["_id"] %}
<form method=post>
<div class="form-group">
<label for="cmd"></label>
<input class="form-control" type="text" id="cmd" name="cmd"
placeholder="Server Command. /help to list them, list gets appended to log.">
</div>
</form>
{% endif %}
Log:
<div id="logger"></div>
<script>
var xmlhttp = new XMLHttpRequest();
var url = '{{ url_for('display_log', room = room.id) }}';
xmlhttp.onreadystatechange = function () {
if (this.readyState == 4 && this.status == 200) {
document.getElementById("logger").innerText = this.responseText;
}
};
function request_new() {
xmlhttp.open("GET", url, true);
xmlhttp.send();
}
window.setTimeout(request_new, 1000);
window.setInterval(request_new, 3000);
</script>
{% endblock %}

21
WebHost/templates/layout.html Normal file
View File

@ -0,0 +1,21 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.0/css/bootstrap.min.css"
integrity="sha384-9aIt2nRpC12Uk9gS9baDl411NQApFmC26EwAOH8WgZl5MYYxFfc+NcPb1dKGj7Sk" crossorigin="anonymous">
{% block head %}<title>Berserker's Multiworld</title>
{% endblock %}
</head>
<body>
{% with messages = get_flashed_messages() %}
{% if messages %}
<ul class=flashes>
{% for message in messages %}
<li>{{ message }}</li>
{% endfor %}
</ul>
{% endif %}
{% endwith %}
{% block body %}{% endblock %}
</body>

18
WebHost/templates/upload_multidata.html
View File

@ -1,7 +1,11 @@
<!doctype html>
<title>Upload Multidata</title>
<h1>Upload Multidata</h1>
<form method=post enctype=multipart/form-data>
<input type=file name=file>
<input type=submit value=Upload>
</form>
{% extends 'layout.html' %}
{% block head %}
<title>Upload Multidata</title>
{% endblock %}
{% block body %}
<h1>Upload Multidata</h1>
<form method=post enctype=multipart/form-data>
<input type=file name=file>
<input type=submit value=Upload>
</form>
{% endblock %}

27
WebHost/templates/view_seed.html Normal file
View File

@ -0,0 +1,27 @@
{% extends 'layout.html' %}
{% block head %}
<title>Multiworld Seed {{ seed.id }}</title>
{% endblock %}
{% block body %}
Seed #{{ seed.id }}<br>
Players:
<ul class="list-group">
{% for team in seed.multidata["names"] %}
<li class="list-group-item">Team #{{ loop.index }} - {{ team | length }}
<ul class="list-group">
{% for player in team %}
<li class="list-group-item">{{ player }}</li>
{% endfor %}
</ul>
</li>
{% endfor %}
</ul>
Rooms:
<ul class="list-group">
{% for room in seed.rooms if room.owner == session["_id"] %}
<li class="list-group-item"><a href="{{ url_for("host_room", room=room.id) }}">Room #{{ room.id }}</a></li>
{% endfor %}
<li class="list-group-item list-group-item-action"><a href="{{ url_for("new_room", seed=seed.id) }}">new
room</a></li>
</ul>
{% endblock %}