optimize tracker, using more RAM, less CPU

WebHost.py

@@ -5,25 +5,28 @@ import logging
 from WebHost import app
 from waitress import serve
 
-from WebHost.models import db
+from WebHost.models import db, Room, db_session, select
 
 DEBUG = False
 port = 80
 
 
 def autohost(config: dict):
-    from WebHost.models import *
+    return
+    # not implemented yet. https://github.com/ponyorm/pony/issues/527
     import time
     from datetime import timedelta, datetime
 
     def keep_running():
-        db.bind(**config["pony"])
+        # db.bind(**config["PONY"])
-        db.generate_mapping(check_tables=False)
+        # db.generate_mapping(check_tables=False)
         while 1:
             time.sleep(3)
             with db_session:
-                select(
+                rooms = select(
-                    room for room in Room if (datetime.utcnow() - room.last_activity) < timedelta(hours=room.timeout))
+                    room for room in Room if
+                    room.last_activity >= datetime.utcnow() - timedelta(hours=room.timeout))
+                logging.info(rooms)
 
     import threading
     threading.Thread(target=keep_running).start()
@@ -45,7 +48,7 @@ if __name__ == "__main__":
     db.bind(**app.config["PONY"])
     db.generate_mapping(create_tables=True)
     if DEBUG:
-        autohost(config)
+        autohost(app.config)
         app.run(debug=True, port=port)
     else:
         serve(app, port=port, threads=app.config["WAITRESS_THREADS"])

WebHost/models.py

@@ -8,7 +8,7 @@ db = Database()
 class Patch(db.Entity):
     id = PrimaryKey(int, auto=True)
     player = Required(int)
-    data = Required(buffer)
+    data = Required(buffer, lazy=True)
     seed = Optional('Seed')
 
 
@@ -19,7 +19,7 @@ class Room(db.Entity):
     owner = Required(UUID, index=True)
     commands = Set('Command')
     seed = Required('Seed', index=True)
-    multisave = Optional(Json)
+    multisave = Optional(Json, lazy=True)
     show_spoiler = Required(int, default=0)  # 0 -> never, 1 -> after completion, -> 2 always
     timeout = Required(int, default=lambda: 6)
     tracker = Optional(UUID, index=True)
@@ -29,11 +29,11 @@ class Room(db.Entity):
 class Seed(db.Entity):
     id = PrimaryKey(UUID, default=uuid4)
     rooms = Set(Room)
-    multidata = Optional(Json)
+    multidata = Optional(Json, lazy=True)
     owner = Required(UUID, index=True)
     creation_time = Required(datetime, default=lambda: datetime.utcnow())
     patches = Set(Patch)
-    spoiler = Optional(str)
+    spoiler = Optional(str, lazy=True)
 
 
 class Command(db.Entity):

WebHost/static/static.css

@@ -9,4 +9,8 @@ table.dataTable.table-sm > thead > tr > th :not(.sorting_disabled) {
 
 th {
     padding: 1px !important;
+}
+
+table {
+    width: 100% !important;
 }

WebHost/tracker.py

@@ -192,21 +192,35 @@ def render_timedelta(delta: datetime.timedelta):
     return f"{hours}:{minutes}"
 
 
+_multidata_cache = {}
+
+
+def get_static_room_data(room: Room):
+    result = _multidata_cache.get(room.seed.id, None)
+    if result:
+        return result
+    multidata = room.seed.multidata
+    # in > 100 players this can take a bit of time and is the main reason for the cache
+    locations = {tuple(k): tuple(v) for k, v in multidata['locations']}
+    names = multidata["names"]
+    _multidata_cache[room.seed.id] = locations, names
+    return locations, names
+
+
 @app.route('/tracker/<uuid:tracker>')
 @cache.memoize(timeout=30)  # update every 30 seconds
 def get_tracker(tracker: UUID):
     room = Room.get(tracker=tracker)
     if not room:
         abort(404)
-    multidata = room.seed.multidata
+    locations, names = get_static_room_data(room)
-    locations = {tuple(k): tuple(v) for k, v in multidata['locations']}
 
     inventory = {teamnumber: {playernumber: collections.Counter() for playernumber in range(1, len(team) + 1)}
-                 for teamnumber, team in enumerate(multidata["names"])}
+                 for teamnumber, team in enumerate(names)}
 
     checks_done = {teamnumber: {playernumber: {loc_name: 0 for loc_name in default_locations}
                                 for playernumber in range(1, len(team) + 1)}
-                   for teamnumber, team in enumerate(multidata["names"])}
+                   for teamnumber, team in enumerate(names)}
     precollected_items = room.seed.multidata.get("precollected_items", None)
 
     for (team, player), locations_checked in room.multisave.get("location_checks", {}):
@@ -230,7 +244,7 @@ def get_tracker(tracker: UUID):
         activity_timers[team, player] = now - datetime.datetime.utcfromtimestamp(timestamp)
 
     player_names = {}
-    for team, names in enumerate(multidata['names']):
+    for team, names in enumerate(names):
         for player, name in enumerate(names, 1):
             player_names[(team, player)] = name