WebHost: job pool based world generation
This commit is contained in:
Fabian Dill
parent
52cf99c5c8
commit
39f85aa291
|
|
@ -24,6 +24,7 @@ app.jinja_env.filters['any'] = any
|
|||
app.jinja_env.filters['all'] = all
|
||||
|
||||
app.config["SELFHOST"] = True
|
||||
app.config["GENERATORS"] = 8 # maximum concurrent world gens
|
||||
app.config["SELFLAUNCH"] = True
|
||||
app.config["DEBUG"] = False
|
||||
app.config["PORT"] = 80
|
||||
|
|
|
|||
|
|
@ -4,8 +4,9 @@ import multiprocessing
|
|||
from datetime import timedelta, datetime
|
||||
import sys
|
||||
import typing
|
||||
import time
|
||||
|
||||
from pony.orm import db_session, select
|
||||
from pony.orm import db_session, select, commit
|
||||
|
||||
|
||||
class CommonLocker():
|
||||
|
|
@ -23,7 +24,6 @@ class AlreadyRunningException(Exception):
|
|||
if sys.platform == 'win32':
|
||||
import os
|
||||
|
||||
|
||||
class Locker(CommonLocker):
|
||||
def __enter__(self):
|
||||
try:
|
||||
|
|
@ -42,7 +42,6 @@ if sys.platform == 'win32':
|
|||
else: # unix
|
||||
import fcntl
|
||||
|
||||
|
||||
class Locker(CommonLocker):
|
||||
def __enter__(self):
|
||||
try:
|
||||
|
|
@ -66,24 +65,70 @@ def launch_room(room: Room, config: dict):
|
|||
multiworld.start()
|
||||
|
||||
|
||||
def autohost(config: dict):
|
||||
import time
|
||||
def handle_generation_success(seed_id):
|
||||
logging.info(f"Generation finished for seed {seed_id}")
|
||||
|
||||
|
||||
def handle_generation_failure(result: BaseException):
|
||||
try: # hacky way to get the full RemoteTraceback
|
||||
raise result
|
||||
except Exception as e:
|
||||
logging.exception(e)
|
||||
|
||||
|
||||
def launch_generator(pool: multiprocessing.pool.Pool, generation: Generation):
|
||||
@db_session
|
||||
def handle_fail(result: BaseException):
|
||||
generation.state = STATE_ERROR
|
||||
handle_generation_failure(result)
|
||||
|
||||
logging.info(f"Generating {generation.id} for {len(generation.options)} players")
|
||||
|
||||
pool.apply_async(gen_game, (generation.options,),
|
||||
{"race": generation.meta["race"], "sid": generation.id, "owner": generation.owner},
|
||||
handle_generation_success, handle_generation_failure)
|
||||
generation.state = STATE_STARTED
|
||||
|
||||
|
||||
def init_db(pony_config: dict):
|
||||
db.bind(**pony_config)
|
||||
db.generate_mapping()
|
||||
|
||||
|
||||
def autohost(config: dict):
|
||||
def keep_running():
|
||||
try:
|
||||
with Locker("autohost"):
|
||||
logging.info("Starting autohost service")
|
||||
# db.bind(**config["PONY"])
|
||||
# db.generate_mapping(check_tables=False)
|
||||
while 1:
|
||||
time.sleep(3)
|
||||
with db_session:
|
||||
rooms = select(
|
||||
room for room in Room if
|
||||
room.last_activity >= datetime.utcnow() - timedelta(days=3))
|
||||
for room in rooms:
|
||||
launch_room(room, config)
|
||||
|
||||
with multiprocessing.Pool(config["GENERATORS"], initializer=init_db,
|
||||
initargs=(config["PONY"],)) as generator_pool:
|
||||
with db_session:
|
||||
to_start = select(generation for generation in Generation if generation.state == STATE_STARTED)
|
||||
|
||||
if to_start:
|
||||
logging.info("Resuming generation")
|
||||
for generation in to_start:
|
||||
sid = Seed.get(id=generation.id)
|
||||
if sid:
|
||||
generation.delete()
|
||||
else:
|
||||
launch_generator(generator_pool, generation)
|
||||
|
||||
commit()
|
||||
select(generation for generation in Generation if generation.state == STATE_ERROR).delete()
|
||||
|
||||
while 1:
|
||||
time.sleep(0.50)
|
||||
with db_session:
|
||||
rooms = select(
|
||||
room for room in Room if
|
||||
room.last_activity >= datetime.utcnow() - timedelta(days=3))
|
||||
for room in rooms:
|
||||
launch_room(room, config)
|
||||
to_start = select(
|
||||
generation for generation in Generation if generation.state == STATE_QUEUED)
|
||||
for generation in to_start:
|
||||
launch_generator(generator_pool, generation)
|
||||
except AlreadyRunningException:
|
||||
pass
|
||||
|
||||
|
|
@ -117,5 +162,6 @@ class MultiworldInstance():
|
|||
self.process = None
|
||||
|
||||
|
||||
from .models import Room
|
||||
from .models import Room, Generation, STATE_QUEUED, STATE_STARTED, STATE_ERROR, db, Seed
|
||||
from .customserver import run_server_process
|
||||
from .generate import gen_game
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import tempfile
|
|||
import random
|
||||
import zlib
|
||||
import json
|
||||
import multiprocessing
|
||||
|
||||
from flask import request, flash, redirect, url_for, session, render_template
|
||||
|
||||
|
|
@ -32,57 +33,78 @@ def generate(race=False):
|
|||
if any(type(result) == str for result in results.values()):
|
||||
return render_template("checkresult.html", results=results)
|
||||
elif len(gen_options) > app.config["MAX_ROLL"]:
|
||||
flash(f"Sorry, generating of multiworld is limited to {app.config['MAX_ROLL']} players for now. "
|
||||
flash(f"Sorry, generating of multiworlds is limited to {app.config['MAX_ROLL']} players for now. "
|
||||
f"If you have a larger group, please generate it yourself and upload it.")
|
||||
else:
|
||||
seed_id = gen(gen_options, race=race)
|
||||
return redirect(url_for("view_seed", seed=seed_id))
|
||||
|
||||
gen = Generation(options={name: vars(options) for name, options in gen_options.items()},
|
||||
# convert to json compatible
|
||||
meta={"race": race, "owner": session["_id"].int}, state=STATE_QUEUED,
|
||||
owner=session["_id"])
|
||||
commit()
|
||||
|
||||
return redirect(url_for("wait_seed", seed=gen.id))
|
||||
return render_template("generate.html", race=race)
|
||||
|
||||
|
||||
def gen(gen_options, race=False):
|
||||
def gen_game(gen_options, race=False, owner=None, sid=None):
|
||||
target = tempfile.TemporaryDirectory()
|
||||
with target:
|
||||
playercount = len(gen_options)
|
||||
seed = get_seed()
|
||||
random.seed(seed)
|
||||
playercount = len(gen_options)
|
||||
seed = get_seed()
|
||||
random.seed(seed)
|
||||
|
||||
if race:
|
||||
random.seed() # reset to time-based random source
|
||||
if race:
|
||||
random.seed() # reset to time-based random source
|
||||
|
||||
seedname = "M" + (f"{random.randint(0, pow(10, seeddigits) - 1)}".zfill(seeddigits))
|
||||
seedname = "M" + (f"{random.randint(0, pow(10, seeddigits) - 1)}".zfill(seeddigits))
|
||||
|
||||
erargs = parse_arguments(['--multi', str(playercount)])
|
||||
erargs.seed = seed
|
||||
erargs.name = {x: "" for x in range(1, playercount + 1)} # only so it can be overwrittin in mystery
|
||||
erargs.create_spoiler = not race
|
||||
erargs.race = race
|
||||
erargs.skip_playthrough = race
|
||||
erargs.outputname = seedname
|
||||
erargs.outputpath = target.name
|
||||
erargs.teams = 1
|
||||
erargs.progression_balancing = {}
|
||||
erargs.create_diff = True
|
||||
erargs = parse_arguments(['--multi', str(playercount)])
|
||||
erargs.seed = seed
|
||||
erargs.name = {x: "" for x in range(1, playercount + 1)} # only so it can be overwrittin in mystery
|
||||
erargs.create_spoiler = not race
|
||||
erargs.race = race
|
||||
erargs.skip_playthrough = race
|
||||
erargs.outputname = seedname
|
||||
erargs.outputpath = target.name
|
||||
erargs.teams = 1
|
||||
erargs.progression_balancing = {}
|
||||
erargs.create_diff = True
|
||||
|
||||
for player, (playerfile, settings) in enumerate(gen_options.items(), 1):
|
||||
for k, v in vars(settings).items():
|
||||
if v is not None:
|
||||
getattr(erargs, k)[player] = v
|
||||
for player, (playerfile, settings) in enumerate(gen_options.items(), 1):
|
||||
for k, v in settings.items():
|
||||
if v is not None:
|
||||
getattr(erargs, k)[player] = v
|
||||
|
||||
if not erargs.name[player]:
|
||||
erargs.name[player] = os.path.split(playerfile)[-1].split(".")[0]
|
||||
if not erargs.name[player]:
|
||||
erargs.name[player] = os.path.split(playerfile)[-1].split(".")[0]
|
||||
|
||||
erargs.names = ",".join(erargs.name[i] for i in range(1, playercount + 1))
|
||||
del (erargs.name)
|
||||
erargs.names = ",".join(erargs.name[i] for i in range(1, playercount + 1))
|
||||
del (erargs.name)
|
||||
|
||||
erargs.skip_progression_balancing = {player: not balanced for player, balanced in
|
||||
erargs.progression_balancing.items()}
|
||||
del (erargs.progression_balancing)
|
||||
ERmain(erargs, seed)
|
||||
return upload_to_db(target.name)
|
||||
erargs.skip_progression_balancing = {player: not balanced for player, balanced in
|
||||
erargs.progression_balancing.items()}
|
||||
del (erargs.progression_balancing)
|
||||
ERmain(erargs, seed)
|
||||
|
||||
return upload_to_db(target.name, owner, sid)
|
||||
|
||||
|
||||
def upload_to_db(folder):
|
||||
@app.route('/wait/<suuid:seed>')
|
||||
def wait_seed(seed: UUID):
|
||||
seed_id = seed
|
||||
seed = Seed.get(id=seed_id)
|
||||
if seed:
|
||||
return redirect(url_for("view_seed", seed=seed_id))
|
||||
generation = Generation.get(id=seed_id)
|
||||
|
||||
if not generation:
|
||||
return "Generation not found."
|
||||
elif generation.state == STATE_ERROR:
|
||||
return "Generation failed, please retry."
|
||||
return render_template("wait_seed.html", seed_id=seed_id)
|
||||
|
||||
|
||||
def upload_to_db(folder, owner, sid):
|
||||
patches = set()
|
||||
spoiler = ""
|
||||
multidata = None
|
||||
|
|
@ -99,9 +121,9 @@ def upload_to_db(folder):
|
|||
except Exception as e:
|
||||
flash(e)
|
||||
if multidata:
|
||||
commit() # commit patches
|
||||
seed = Seed(multidata=multidata, spoiler=spoiler, patches=patches, owner=session["_id"])
|
||||
commit() # create seed
|
||||
seed = Seed(multidata=multidata, spoiler=spoiler, patches=patches, owner=owner, id=sid)
|
||||
for patch in patches:
|
||||
patch.seed = seed
|
||||
Generation.get(id=sid).delete()
|
||||
commit()
|
||||
return seed.id
|
||||
|
|
|
|||
|
|
@ -4,6 +4,10 @@ from pony.orm import *
|
|||
|
||||
db = Database()
|
||||
|
||||
STATE_QUEUED = 0
|
||||
STATE_STARTED = 1
|
||||
STATE_ERROR = -1
|
||||
|
||||
|
||||
class Patch(db.Entity):
|
||||
id = PrimaryKey(int, auto=True)
|
||||
|
|
@ -40,3 +44,11 @@ class Command(db.Entity):
|
|||
id = PrimaryKey(int, auto=True)
|
||||
room = Required(Room)
|
||||
commandtext = Required(str)
|
||||
|
||||
|
||||
class Generation(db.Entity):
|
||||
id = PrimaryKey(UUID, default=uuid4)
|
||||
owner = Required(UUID)
|
||||
options = Required(Json, lazy=True)
|
||||
meta = Required(Json)
|
||||
state = Required(int, default=0, index=True)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,16 @@
|
|||
{% extends 'layout.html' %}
|
||||
{% import "macros.html" as macros %}
|
||||
|
||||
{% block head %}
|
||||
<title>Multiworld Seed {{ seed_id|suuid }} (generating...)</title>
|
||||
<meta http-equiv="refresh" content="1">
|
||||
<link rel="stylesheet" type="text/css" href="{{ static_autoversion("styles/view_seed.css") }}"/>
|
||||
{% endblock %}
|
||||
|
||||
{% block body %}
|
||||
<div id="wait-seed-wrapper">
|
||||
<div class="main-content">
|
||||
Waiting for game to generate, this page auto-refreshes to check.
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
Loading…
Reference in New Issue