webhost update

.gitignore

@@ -1,17 +1,21 @@
 .idea
 .vscode
+
 *_Spoiler.txt
 *.bmbp
 *.pyc
+*.pyd
 *.sfc
 *.wixobj
+*.lck
+*multidata
+*multisave
+
 build
 bundle/components.wxs
 dist
 README.html
 .vs/
-*multidata
-*multisave
 EnemizerCLI/
 .mypy_cache/
 RaceRom.py

MultiServer.py

@@ -104,6 +104,7 @@ class Context(Node):
         with open(multidatapath, 'rb') as f:
             self._load(json.loads(zlib.decompress(f.read()).decode("utf-8-sig")),
                        use_embedded_server_options)
+
         self.data_filename = multidatapath
 
     def _load(self, jsonobj: dict, use_embedded_server_options: bool):
@@ -1177,7 +1178,7 @@ def parse_args() -> argparse.Namespace:
 
 
 async def auto_shutdown(ctx, to_cancel=None):
-    await asyncio.sleep(ctx.auto_shutdown * 60)
+    await asyncio.sleep(ctx.auto_shutdown)
     while ctx.running:
         if not ctx.client_activity_timers.values():
             asyncio.create_task(ctx.server.ws_server._close())
@@ -1189,7 +1190,7 @@ async def auto_shutdown(ctx, to_cancel=None):
         else:
             newest_activity = max(ctx.client_activity_timers.values())
             delta = datetime.datetime.now(datetime.timezone.utc) - newest_activity
-            seconds = ctx.auto_shutdown * 60 - delta.total_seconds()
+            seconds = ctx.auto_shutdown - delta.total_seconds()
             if seconds < 0:
                 asyncio.create_task(ctx.server.ws_server._close())
                 ctx.running = False

WebHost.py

@@ -2,41 +2,17 @@ import os
 import multiprocessing
 import logging
 
-from WebHost import app
+from WebHost import app as raw_app
 from waitress import serve
 
-from WebHost.models import db, Room, db_session, select
+from WebHost.models import db
+from WebHost.autolauncher import autohost
+
+configpath = "config.yaml"
 
 
-
+def get_app():
-def autohost(config: dict):
+    app = raw_app
-    return
-    # not implemented yet. https://github.com/ponyorm/pony/issues/527
-    import time
-    from datetime import timedelta, datetime
-
-    def keep_running():
-        # db.bind(**config["PONY"])
-        # db.generate_mapping(check_tables=False)
-        while 1:
-            time.sleep(3)
-            with db_session:
-                rooms = select(
-                    room for room in Room if
-                    room.last_activity >= datetime.utcnow() - timedelta(hours=room.timeout))
-                logging.info(rooms)
-
-    import threading
-    threading.Thread(target=keep_running).start()
-
-
-if __name__ == "__main__":
-    multiprocessing.freeze_support()
-    multiprocessing.set_start_method('spawn')
-    logging.basicConfig(format='[%(asctime)s] %(message)s', level=logging.INFO)
-
-    configpath = "config.yaml"
-
     if os.path.exists(configpath):
         import yaml
         with open(configpath) as c:
@@ -45,6 +21,17 @@ if __name__ == "__main__":
         logging.info(f"Updated config from {configpath}")
     db.bind(**app.config["PONY"])
     db.generate_mapping(create_tables=True)
+    return app
+
+
+if __name__ == "__main__":
+    multiprocessing.freeze_support()
+    multiprocessing.set_start_method('spawn')
+    logging.basicConfig(format='[%(asctime)s] %(message)s', level=logging.INFO)
+    app = get_app()
+    if app.config["SELFLAUNCH"]:
+        autohost(app.config)
+    if app.config["SELFHOST"]:  # using WSGI, you just want to run get_app()
         if app.config["DEBUG"]:
             autohost(app.config)
             app.run(debug=True, port=app.config["PORT"])

WebHost/__init__.py

@@ -2,9 +2,6 @@
 So unless you're Berserker you need to include license information."""
 
 import os
-import logging
-import typing
-import multiprocessing
 import threading
 
 from pony.flask import Pony
@@ -23,9 +20,12 @@ os.makedirs(LOGS_FOLDER, exist_ok=True)
 def allowed_file(filename):
     return filename.endswith(('multidata', ".zip"))
 
+
 app = Flask(__name__)
 Pony(app)
 
+app.config["SELFHOST"] = True
+app.config["SELFLAUNCH"] = True
 app.config["DEBUG"] = False
 app.config["PORT"] = 80
 app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
@@ -47,7 +47,6 @@ cache = Cache(app)
 Compress(app)
 
 # this local cache is risky business if app hosting is done with subprocesses as it will not sync. Waitress is fine though
-multiworlds = {}
 
 
 @app.before_request
@@ -57,29 +56,6 @@ def register_session():
         session["_id"] = uuid4()  # uniquely identify each session without needing a login
 
 
-class MultiworldInstance():
-    def __init__(self, room: Room):
-        self.room_id = room.id
-        self.process: typing.Optional[multiprocessing.Process] = None
-        multiworlds[self.room_id] = self
-
-    def start(self):
-        if self.process and self.process.is_alive():
-            return False
-
-        logging.info(f"Spinning up {self.room_id}")
-        with db_session:
-            self.process = multiprocessing.Process(group=None, target=run_server_process,
-                                                   args=(self.room_id, app.config["PONY"]),
-                                                   name="MultiHost")
-        self.process.start()
-
-    def stop(self):
-        if self.process:
-            self.process.terminate()
-            self.process = None
-
-
 @app.route('/seed/<uuid:seed>')
 def view_seed(seed: UUID):
     seed = Seed.get(id=seed)
@@ -114,8 +90,6 @@ def display_log(room: UUID):
     return Response(_read_log(os.path.join("logs", str(room) + ".txt")), mimetype="text/plain;charset=UTF-8")
 
 
-processstartlock = threading.Lock()
-
 
 @app.route('/hosted/<uuid:room>', methods=['GET', 'POST'])
 def host_room(room: UUID):
@@ -127,13 +101,9 @@ def host_room(room: UUID):
             cmd = request.form["cmd"]
             Command(room=room, commandtext=cmd)
             commit()
-    with db_session:
-        multiworld = multiworlds.get(room.id, None)
-        if not multiworld:
-            multiworld = MultiworldInstance(room)
 
-    with processstartlock:
+    with db_session:
-        multiworld.start()
+        room.last_activity = datetime.utcnow()  # will trigger a spinup, if it's not already running
 
     return render_template("host_room.html", room=room)
 

WebHost/autolauncher.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+import logging
+import multiprocessing
+from datetime import timedelta, datetime
+import sys
+import typing
+
+from pony.orm import db_session, select
+
+
+class CommonLocker():
+    """Uses a file lock to signal that something is already running"""
+
+    def __init__(self, lockname: str):
+        self.lockname = lockname
+        self.lockfile = f"./{self.lockname}.lck"
+
+
+class AlreadyRunningException(Exception):
+    pass
+
+
+if sys.platform == 'win32':
+    import os
+
+
+    class Locker(CommonLocker):
+        def __enter__(self):
+            try:
+                if os.path.exists(self.lockfile):
+                    os.unlink(self.lockfile)
+                self.fp = os.open(
+                    self.lockfile, os.O_CREAT | os.O_EXCL | os.O_RDWR)
+            except OSError as e:
+                raise AlreadyRunningException() from e
+
+        def __exit__(self, _type, value, tb):
+            fp = getattr(self, "fp", None)
+            if fp:
+                os.close(self.fp)
+                os.unlink(self.lockfile)
+else:  # unix
+    import fcntl
+
+
+    class Locker(CommonLocker):
+        def __enter__(self):
+            self.fp = open(self.lockfile, "rb")
+            try:
+                fcntl.flock(self.fp.fileno(), fcntl.LOCK_EX)
+            except OSError as e:
+                raise AlreadyRunningException() from e
+
+        def __exit__(self, _type, value, tb):
+            fcntl.flock(self.fp.fileno(), fcntl.LOCK_UN)
+            self.fp.close()
+
+
+def launch_room(room: Room, config: dict):
+    # requires db_session!
+    if room.last_activity >= datetime.utcnow() - timedelta(minutes=room.timeout):
+        multiworld = multiworlds.get(room.id, None)
+        if not multiworld:
+            multiworld = MultiworldInstance(room, config)
+
+        multiworld.start()
+
+
+def autohost(config: dict):
+    import time
+
+    def keep_running():
+        try:
+            with Locker("autohost"):
+                # db.bind(**config["PONY"])
+                # db.generate_mapping(check_tables=False)
+                while 1:
+                    time.sleep(3)
+                    with db_session:
+                        rooms = select(
+                            room for room in Room if
+                            room.last_activity >= datetime.utcnow() - timedelta(days=3))
+                        for room in rooms:
+                            launch_room(room, config)
+
+        except AlreadyRunningException:
+            pass
+
+    import threading
+    threading.Thread(target=keep_running).start()
+
+
+multiworlds = {}
+
+
+class MultiworldInstance():
+    def __init__(self, room: Room, config: dict):
+        self.room_id = room.id
+        self.process: typing.Optional[multiprocessing.Process] = None
+        multiworlds[self.room_id] = self
+        self.ponyconfig = config["PONY"]
+
+    def start(self):
+        if self.process and self.process.is_alive():
+            return False
+
+        logging.info(f"Spinning up {self.room_id}")
+        self.process = multiprocessing.Process(group=None, target=run_server_process,
+                                               args=(self.room_id, self.ponyconfig),
+                                               name="MultiHost")
+        self.process.start()
+
+    def stop(self):
+        if self.process:
+            self.process.terminate()
+            self.process = None
+
+
+from .models import Room
+from .customserver import run_server_process

WebHost/customserver.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import functools
 import logging
 import os
@@ -8,7 +10,7 @@ import threading
 import time
 import random
 
-from WebHost import LOGS_FOLDER
+
 from .models import *
 
 from MultiServer import Context, server, auto_shutdown, ServerCommandProcessor, ClientMessageProcessor
@@ -16,6 +18,7 @@ from Utils import get_public_ipv4, get_public_ipv6
 
 
 class CustomClientMessageProcessor(ClientMessageProcessor):
+    ctx: WebHostContext
     def _cmd_video(self, platform, user):
         """Set a link for your name in the WebHost tracker pointing to a video stream"""
         if platform.lower().startswith("t"):  # twitch
@@ -28,7 +31,6 @@ class CustomClientMessageProcessor(ClientMessageProcessor):
 
 # inject
 import MultiServer
-
 MultiServer.client_message_processor = CustomClientMessageProcessor
 del (MultiServer)
 
@@ -80,7 +82,10 @@ class WebHostContext(Context):
 
     @db_session
     def _save(self) -> bool:
-        Room.get(id=self.room_id).multisave = self.get_save()
+        room = Room.get(id=self.room_id)
+        room.multisave = self.get_save()
+        # saving only occurs on activity, so we can "abuse" this information to mark this as last_activity
+        room.last_activity = datetime.utcnow()
         return True
 
     def get_save(self) -> dict:
@@ -104,7 +109,6 @@ def run_server_process(room_id, ponyconfig: dict):
                                 logging.FileHandler(os.path.join(LOGS_FOLDER, f"{room_id}.txt"), 'a', 'utf-8-sig')])
         ctx = WebHostContext()
         ctx.load(room_id)
-        ctx.auto_shutdown = 24 * 60 * 60  # 24 hours
         ctx.init_save()
 
         try:
@@ -126,9 +130,13 @@ def run_server_process(room_id, ponyconfig: dict):
                     room.last_port = socketname[1]
             elif wssocket.family == socket.AF_INET:
                 logging.info(f'Hosting game at {get_public_ipv4()}:{socketname[1]}')
-        ctx.auto_shutdown = 6 * 60
+        with db_session:
+            ctx.auto_shutdown = Room.get(id=room_id).timeout
         ctx.shutdown_task = asyncio.create_task(auto_shutdown(ctx, []))
         await ctx.shutdown_task
         logging.info("Shutting down")
 
     asyncio.run(main())
+
+
+from WebHost import LOGS_FOLDER

WebHost/models.py

@@ -21,7 +21,7 @@ class Room(db.Entity):
     seed = Required('Seed', index=True)
     multisave = Optional(Json, lazy=True)
     show_spoiler = Required(int, default=0)  # 0 -> never, 1 -> after completion, -> 2 always
-    timeout = Required(int, default=lambda: 6)
+    timeout = Required(int, default=lambda: 6 * 60 * 60)  # seconds since last activity to shutdown
     tracker = Optional(UUID, index=True)
     last_port = Optional(int, default=lambda: 0)
 

WebHost/requirements.txt

@@ -4,3 +4,5 @@ waitress>=1.4.4
 flask-caching>=1.9.0
 Flask-Autoversion>=0.2.0
 Flask-Compress>=1.5.0
+redis>=3.5.3
+rq>=1.4.3

WebHost/templates/host_room.html

@@ -9,7 +9,7 @@
     {% if room.tracker %}
         This room has a <a href="{{ url_for("get_tracker", tracker=room.tracker) }}">Multiworld Tracker</a> enabled.<br>
     {% endif %}
-    This room will be closed after {{ room.timeout }} hours of inactivity. Should you wish to continue later,
+    This room will be closed after {{ room.timeout//60//60 }} hours of inactivity. Should you wish to continue later,
     you can simply refresh this page and the server will be started again.<br>
     {% if room.owner == session["_id"] %}
         <form method=post>

WebHost/templates/landing.html

@@ -30,6 +30,8 @@
                     </p>
                 </div>
                 <div>
+                    <p class="lead">This webpage is still under heavy construction. Database may be wiped as I see fit
+                        and some stuff may be broken.</p>
                     <p class="lead">This is a randomizer for The Legend of Zelda: A Link to the Past.</p>
                     <p class="lead">It is a multiworld, meaning items get shuffled across multiple players' worlds
                         which get exchanged on pickup through the internet.</p>

host.yaml

@@ -34,7 +34,7 @@ server_options:
   # "goal" -> client can ask for remaining items after goal completion
   # warning: only Berserker's Multiworld clients of version 2.1+ send game beaten information
   remaining_mode: "goal"
-  # automatically shut down the server after this many minutes without new location checks, 0 to keep running
+  # automatically shut down the server after this many seconds without new location checks, 0 to keep running
   auto_shutdown: 0
 #options for MultiMystery.py
 multi_mystery_options: