customserver: fix memory leak (#3864)

MultiServer.py

@@ -67,6 +67,21 @@ def update_dict(dictionary, entries):
     return dictionary
 
 
+def queue_gc():
+    import gc
+    from threading import Thread
+
+    gc_thread: typing.Optional[Thread] = getattr(queue_gc, "_thread", None)
+    def async_collect():
+        time.sleep(2)
+        setattr(queue_gc, "_thread", None)
+        gc.collect()
+    if not gc_thread:
+        gc_thread = Thread(target=async_collect)
+        setattr(queue_gc, "_thread", gc_thread)
+        gc_thread.start()
+
+
 # functions callable on storable data on the server by clients
 modify_functions = {
     # generic:
@@ -551,6 +566,9 @@ class Context:
                         self.logger.info(f"Saving failed. Retry in {self.auto_save_interval} seconds.")
                     else:
                         self.save_dirty = False
+                if not atexit_save:  # if atexit is used, that keeps a reference anyway
+                    queue_gc()
+
             self.auto_saver_thread = threading.Thread(target=save_regularly, daemon=True)
             self.auto_saver_thread.start()
 

WebHostLib/customserver.py

@@ -72,6 +72,14 @@ class WebHostContext(Context):
         self.video = {}
         self.tags = ["AP", "WebHost"]
 
+    def __del__(self):
+        try:
+            import psutil
+            from Utils import format_SI_prefix
+            self.logger.debug(f"Context destroyed, Mem: {format_SI_prefix(psutil.Process().memory_info().rss, 1024)}iB")
+        except ImportError:
+            self.logger.debug("Context destroyed")
+
     def _load_game_data(self):
         for key, value in self.static_server_data.items():
             # NOTE: attributes are mutable and shared, so they will have to be copied before being modified
@@ -249,6 +257,7 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict,
                 ctx = WebHostContext(static_server_data, logger)
                 ctx.load(room_id)
                 ctx.init_save()
+                assert ctx.server is None
                 try:
                     ctx.server = websockets.serve(
                         functools.partial(server, ctx=ctx), ctx.host, ctx.port, ssl=ssl_context)
@@ -279,6 +288,7 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict,
                     ctx.auto_shutdown = Room.get(id=room_id).timeout
                 if ctx.saving:
                     setattr(asyncio.current_task(), "save", lambda: ctx._save(True))
+                assert ctx.shutdown_task is None
                 ctx.shutdown_task = asyncio.create_task(auto_shutdown(ctx, []))
                 await ctx.shutdown_task
 
@@ -325,7 +335,7 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict,
         def run(self):
             while 1:
                 next_room = rooms_to_run.get(block=True,  timeout=None)
-                gc.collect(0)
+                gc.collect()
                 task = asyncio.run_coroutine_threadsafe(start_room(next_room), loop)
                 self._tasks.append(task)
                 task.add_done_callback(self._done)