Database-backed Webhosting

MultiServer.py

@@ -166,22 +166,24 @@ class Context(Node):
                 logging.error('No save data found, starting a new game')
             except Exception as e:
                 logging.exception(e)
+            self._start_async_saving()
 
-            if not self.auto_saver_thread:
+    def _start_async_saving(self):
-                def save_regularly():
+        if not self.auto_saver_thread:
-                    import time
+            def save_regularly():
-                    while self.running:
+                import time
-                        time.sleep(self.auto_save_interval)
+                while self.running:
-                        if self.save_dirty:
+                    time.sleep(self.auto_save_interval)
-                            logging.debug("Saving multisave via thread.")
+                    if self.save_dirty:
-                            self.save_dirty = False
+                        logging.debug("Saving multisave via thread.")
-                            self._save()
+                        self.save_dirty = False
+                        self._save()
 
-                self.auto_saver_thread = threading.Thread(target=save_regularly, daemon=True)
+            self.auto_saver_thread = threading.Thread(target=save_regularly, daemon=True)
-                self.auto_saver_thread.start()
+            self.auto_saver_thread.start()
 
-            import atexit
+        import atexit
-            atexit.register(self._save)  # make sure we save on exit too
+        atexit.register(self._save)  # make sure we save on exit too
 
     def get_save(self) -> dict:
         d = {

WebHost.py

@@ -0,0 +1,31 @@
+import os
+import multiprocessing
+import logging
+
+from WebHost import app
+from waitress import serve
+
+from WebHost.models import db
+
+DEBUG = False
+
+if __name__ == "__main__":
+    multiprocessing.freeze_support()
+    multiprocessing.set_start_method('spawn')
+    logging.basicConfig(format='[%(asctime)s] %(message)s', level=logging.INFO)
+
+    configpath = "config.yaml"
+
+    if os.path.exists(configpath):
+        import yaml
+
+        with open(configpath) as c:
+            app.config.update(yaml.safe_load(c))
+
+        logging.info(f"Updated config from {configpath}")
+    db.bind(**app.config["PONY"])
+    db.generate_mapping(create_tables=True)
+    if DEBUG:
+        app.run(debug=True)
+    else:
+        serve(app, port=80, threads=1)

WebHost/__init__.py

@@ -1,21 +1,19 @@
-# module has yet to be made capable of running in multiple processes
-
 import os
 import logging
-import threading
 import typing
 import multiprocessing
-from pony.orm import Database, db_session
+import threading
-
+import json
-from flask import Flask, flash, request, redirect, url_for, render_template, Response
+import zlib
-from werkzeug.utils import secure_filename
 
+from pony.orm import db_session, commit
+from pony.flask import Pony
+from flask import Flask, flash, request, redirect, url_for, render_template, Response, session
 
+from .models import *
 
 UPLOAD_FOLDER = os.path.relpath('uploads')
 LOGS_FOLDER = os.path.relpath('logs')
-multidata_folder = os.path.join(UPLOAD_FOLDER, "multidata")
-os.makedirs(multidata_folder, exist_ok=True)
 os.makedirs(LOGS_FOLDER, exist_ok=True)
 
 
@@ -24,36 +22,43 @@ def allowed_file(filename):
 
 
 app = Flask(__name__)
+Pony(app)
 app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
 app.config['MAX_CONTENT_LENGTH'] = 1 * 1024 * 1024  # 1 megabyte limit
+# if you want persistent sessions on your server, make sure you make this a constant in your config.yaml
 app.config["SECRET_KEY"] = os.urandom(32)
+app.config['SESSION_PERMANENT'] = True
 app.config["PONY"] = {
     'provider': 'sqlite',
-    'filename': 'db.db3',
+    'filename': os.path.abspath('db.db3'),
     'create_db': True
 }
 
-db = Database()
-
-name = "localhost"
-
 multiworlds = {}
 
 
-class Multiworld():
+@app.before_first_request
-    def __init__(self, multidata: str):
+def register_session():
-        self.multidata = multidata
+    session.permanent = True  # technically 31 days after the last visit
+    if not session.get("_id", None):
+        session["_id"] = uuid4()  # uniquely identify each session without needing a login
+
+
+class MultiworldInstance():
+    def __init__(self, room: Room):
+        self.room_id = room.id
         self.process: typing.Optional[multiprocessing.Process] = None
-        multiworlds[multidata] = self
+        multiworlds[self.room_id] = self
 
     def start(self):
         if self.process and self.process.is_alive():
             return False
 
-        logging.info(f"Spinning up {self.multidata}")
+        logging.info(f"Spinning up {self.room_id}")
-        self.process = multiprocessing.Process(group=None, target=run_server_process,
+        with db_session:
-                                               args=(self.multidata,),
+            self.process = multiprocessing.Process(group=None, target=run_server_process,
-                                               name="MultiHost")
+                                                   args=(self.room_id, app.config["PONY"]),
+                                                   name="MultiHost")
         self.process.start()
 
     def stop(self):
@@ -67,21 +72,40 @@ def upload_multidata():
         # check if the post request has the file part
         if 'file' not in request.files:
             flash('No file part')
-            return redirect(request.url)
+        else:
-        file = request.files['file']
+            file = request.files['file']
-        # if user does not select file, browser also
+            # if user does not select file, browser also
-        # submit an empty part without filename
+            # submit an empty part without filename
-        if file.filename == '':
+            if file.filename == '':
-            flash('No selected file')
+                flash('No selected file')
-            return redirect(request.url)
+            elif file and allowed_file(file.filename):
-        if file and allowed_file(file.filename):
+                try:
-            filename = secure_filename(file.filename)
+                    multidata = json.loads(zlib.decompress(file.read()).decode("utf-8-sig"))
-            file.save(os.path.join(multidata_folder, filename))
+                except:
-            return redirect(url_for('host_multidata',
+                    flash("Could not load multidata. File may be corrupted or incompatible.")
-                                    filename=filename))
+                else:
+                    seed = Seed(multidata=multidata)
+                    commit()  # place into DB and generate ids
+                    return redirect(url_for("view_seed", seed=seed.id))
+            else:
+                flash("Not recognized file format. Awaiting a .multidata file.")
     return render_template("upload_multidata.html")
 
 
+@app.route('/seed/<int:seed>')
+def view_seed(seed: int):
+    seed = Seed.get(id=seed)
+    return render_template("view_seed.html", seed=seed)
+
+
+@app.route('/new_room/<int:seed>')
+def new_room(seed: int):
+    seed = Seed.get(id=seed)
+    room = Room(seed=seed, owner=session["_id"])
+    commit()
+    return redirect(url_for("host_room", room=room.id))
+
+
 def _read_log(path: str):
     if os.path.exists(path):
         with open(path) as log:
@@ -91,34 +115,32 @@ def _read_log(path: str):
               f"Likely a crash during spinup of multiworld instance or it is still spinning up."
 
 
-@app.route('/log/<filename>')
+@app.route('/log/<int:room>')
-def display_log(filename: str):
+def display_log(room: int):
-    filename = secure_filename(filename)
     # noinspection PyTypeChecker
-    return Response(_read_log(os.path.join("logs", filename + ".txt")), mimetype="text/plain;charset=UTF-8")
+    return Response(_read_log(os.path.join("logs", str(room) + ".txt")), mimetype="text/plain;charset=UTF-8")
 
 
 processstartlock = threading.Lock()
 
 
-@app.route('/hosted/<filename>')
+@app.route('/hosted/<int:room>', methods=['GET', 'POST'])
-def host_multidata(filename: str):
+def host_room(room: int):
+    room = Room.get(id=room)
+    if request.method == "POST":
+        if room.owner == session["_id"]:
+            cmd = request.form["cmd"]
+            Command(room=room, commandtext=cmd)
+            commit()
     with db_session:
-        multiworld = multiworlds.get(filename, None)
+        multiworld = multiworlds.get(room.id, None)
         if not multiworld:
-            multiworld = Multiworld(filename)
+            multiworld = MultiworldInstance(room)
 
-        with processstartlock:
+    with processstartlock:
-            multiworld.start()
+        multiworld.start()
 
-    return render_template("host_multidata.html", filename=filename)
+    return render_template("host_room.html", room=room)
 
 
 from WebHost.customserver import run_server_process
-
-if __name__ == "__main__":
-    multiprocessing.freeze_support()
-    multiprocessing.set_start_method('spawn')
-    db.bind(**app.config["PONY"])
-    db.generate_mapping(create_tables=True)
-    app.run(debug=True)

WebHost/customserver.py

@@ -1,21 +1,74 @@
 import functools
 import logging
 import os
-import sys
-
 import websockets
+import asyncio
+import socket
+import threading
+import time
 
-from WebHost import LOGS_FOLDER, multidata_folder
+from WebHost import LOGS_FOLDER
+from .models import *
+
+from MultiServer import Context, server, auto_shutdown, ServerCommandProcessor
+from Utils import get_public_ipv4, get_public_ipv6
 
 
-def run_server_process(multidata: str):
+class DBCommandProcessor(ServerCommandProcessor):
+    def output(self, text: str):
+        logging.info(text)
+
+
+class WebHostContext(Context):
+    def __init__(self):
+        super(WebHostContext, self).__init__("", 0, "", 1, 40, True, "enabled", "enabled", 0)
+
+    def listen_to_db_commands(self):
+        cmdprocessor = DBCommandProcessor(self)
+
+        while self.running:
+            with db_session:
+                commands = select(command for command in Command if command.room.id == self.room_id)
+                if commands:
+                    for command in commands:
+                        cmdprocessor(command.commandtext)
+                        command.delete()
+                    commit()
+            time.sleep(5)
+
+    @db_session
+    def load(self, room_id: int):
+        self.room_id = room_id
+        return self._load(Room.get(id=room_id).seed.multidata, True)
+
+    @db_session
+    def init_save(self, enabled: bool = True):
+        self.saving = enabled
+        if self.saving:
+            existings_savegame = Room.get(id=self.room_id).multisave
+            if existings_savegame:
+                self.set_save(existings_savegame)
+            self._start_async_saving()
+        threading.Thread(target=self.listen_to_db_commands, daemon=True).start()
+
+    @db_session
+    def _save(self) -> bool:
+        Room.get(id=self.room_id).multisave = self.get_save()
+        return True
+
+
+def run_server_process(room_id, ponyconfig: dict):
+    # establish DB connection for multidata and multisave
+    db.bind(**ponyconfig)
+    db.generate_mapping(check_tables=False)
+
     async def main():
+
         logging.basicConfig(format='[%(asctime)s] %(message)s',
                             level=logging.INFO,
-                            filename=os.path.join(LOGS_FOLDER, multidata + ".txt"))
+                            filename=os.path.join(LOGS_FOLDER, f"{room_id}.txt"))
-        ctx = Context("", 0, "", 1, 1000,
+        ctx = WebHostContext()
-                      True, "enabled", "goal", 0)
+        ctx.load(room_id)
-        ctx.load(os.path.join(multidata_folder, multidata), True)
         ctx.auto_shutdown = 24 * 60 * 60  # 24 hours
         ctx.init_save()
 
@@ -34,10 +87,4 @@ def run_server_process(multidata: str):
         await ctx.shutdown_task
         logging.info("Shutting down")
 
-    import asyncio
-    if ".." not in sys.path:
-        sys.path.append("..")
-    from MultiServer import Context, server, auto_shutdown
-    from Utils import get_public_ipv4, get_public_ipv6
-    import socket
     asyncio.run(main())

WebHost/models.py

@@ -0,0 +1,49 @@
+from datetime import datetime
+from uuid import UUID, uuid4
+from pony.orm import *
+
+db = Database()
+
+
+class Patch(db.Entity):
+    id = PrimaryKey(int, auto=True)
+    data = Required(buffer)
+    simple_seed = Required('Seed')
+
+
+class Room(db.Entity):
+    id = PrimaryKey(int, auto=True)
+    last_activity = Required(datetime, default=lambda: datetime.utcnow())
+    owner = Required(UUID)
+    commands = Set('Command')
+    host_jobs = Set('HostJob')
+    seed = Required('Seed')
+    multisave = Optional(Json)
+
+
+class HostJob(db.Entity):
+    id = PrimaryKey(int, auto=True)
+    sockets = Set('Socket')
+    room = Required(Room)
+    scheduler_id = Required(int, unique=True)
+
+
+class Socket(db.Entity):
+    port = PrimaryKey(int)
+    ipv6 = Required(bool)
+    host_job = Required(HostJob)
+
+
+class Seed(db.Entity):
+    id = PrimaryKey(int, auto=True)
+    rooms = Set(Room)
+    multidata = Optional(Json)
+    creation_time = Required(datetime, default=lambda: datetime.utcnow())
+    patches = Set(Patch)
+    spoiler = Optional(str)
+
+
+class Command(db.Entity):
+    id = PrimaryKey(int, auto=True)
+    room = Required(Room)
+    commandtext = Required(str)

WebHost/run.py

@@ -1,9 +0,0 @@
-from waitress import serve
-import multiprocessing
-
-from __init__ import app
-
-if __name__ == "__main__":
-    multiprocessing.freeze_support()
-    multiprocessing.set_start_method('spawn')
-    serve(app, port=80, threads=1)

WebHost/templates/host_multidata.html

@@ -1,30 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <title>Multiworld {{ filename }}</title>
-</head>
-<body>
-Log:
-<div id="logger"></div>
-<script>
-    var xmlhttp = new XMLHttpRequest();
-    var url = '{{ url_for('display_log', filename = filename) }}';
-
-    xmlhttp.onreadystatechange = function () {
-        if (this.readyState == 4 && this.status == 200) {
-            document.getElementById("logger").innerText = this.responseText;
-        }
-    };
-
-    function request_new() {
-        xmlhttp.open("GET", url, true);
-        xmlhttp.send();
-    }
-
-
-    request_new();
-    window.setInterval(request_new, 3000);
-</script>
-</body>
-</html>

WebHost/templates/host_room.html

@@ -0,0 +1,36 @@
+{% extends 'layout.html' %}
+{% block head %}
+    <title>Multiworld {{ room.id }}</title>
+{% endblock %}
+{% block body %}
+    Room created from <a href="{{ url_for("view_seed", seed=room.seed.id) }}">Seed #{{ room.seed.id }}</a><br>
+    {% if room.owner == session["_id"] %}
+        <form method=post>
+            <div class="form-group">
+                <label for="cmd"></label>
+                <input class="form-control" type="text" id="cmd" name="cmd"
+                       placeholder="Server Command. /help to list them, list gets appended to log.">
+            </div>
+        </form>
+    {% endif %}
+    Log:
+    <div id="logger"></div>
+    <script>
+        var xmlhttp = new XMLHttpRequest();
+        var url = '{{ url_for('display_log', room = room.id) }}';
+
+        xmlhttp.onreadystatechange = function () {
+            if (this.readyState == 4 && this.status == 200) {
+                document.getElementById("logger").innerText = this.responseText;
+            }
+        };
+
+        function request_new() {
+            xmlhttp.open("GET", url, true);
+            xmlhttp.send();
+        }
+
+        window.setTimeout(request_new, 1000);
+        window.setInterval(request_new, 3000);
+    </script>
+{% endblock %}

WebHost/templates/layout.html

@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.0/css/bootstrap.min.css"
+          integrity="sha384-9aIt2nRpC12Uk9gS9baDl411NQApFmC26EwAOH8WgZl5MYYxFfc+NcPb1dKGj7Sk" crossorigin="anonymous">
+    {% block head %}<title>Berserker's Multiworld</title>
+    {% endblock %}
+</head>
+<body>
+{% with messages = get_flashed_messages() %}
+    {% if messages %}
+        <ul class=flashes>
+            {% for message in messages %}
+                <li>{{ message }}</li>
+            {% endfor %}
+        </ul>
+    {% endif %}
+{% endwith %}
+{% block body %}{% endblock %}
+</body>

WebHost/templates/upload_multidata.html

@@ -1,7 +1,11 @@
-<!doctype html>
+{% extends 'layout.html' %}
-<title>Upload Multidata</title>
+{% block head %}
-<h1>Upload Multidata</h1>
+    <title>Upload Multidata</title>
-<form method=post enctype=multipart/form-data>
+{% endblock %}
-    <input type=file name=file>
+{% block body %}
-    <input type=submit value=Upload>
+    <h1>Upload Multidata</h1>
-</form>
+    <form method=post enctype=multipart/form-data>
+        <input type=file name=file>
+        <input type=submit value=Upload>
+    </form>
+{% endblock %}

WebHost/templates/view_seed.html

@@ -0,0 +1,27 @@
+{% extends 'layout.html' %}
+{% block head %}
+    <title>Multiworld Seed {{ seed.id }}</title>
+{% endblock %}
+{% block body %}
+    Seed #{{ seed.id }}<br>
+    Players:
+    <ul class="list-group">
+        {% for team in seed.multidata["names"] %}
+            <li class="list-group-item">Team #{{ loop.index }} - {{ team | length }}
+                <ul class="list-group">
+                    {% for player in team %}
+                        <li class="list-group-item">{{ player }}</li>
+                    {% endfor %}
+                </ul>
+            </li>
+        {% endfor %}
+    </ul>
+    Rooms:
+    <ul class="list-group">
+        {% for room in seed.rooms if room.owner == session["_id"] %}
+            <li class="list-group-item"><a href="{{ url_for("host_room", room=room.id) }}">Room #{{ room.id }}</a></li>
+        {% endfor %}
+        <li class="list-group-item list-group-item-action"><a href="{{ url_for("new_room", seed=seed.id) }}">new
+            room</a></li>
+    </ul>
+{% endblock %}