From ef7d8a6b4fb007683999681f60fb72b4228d7cdc Mon Sep 17 00:00:00 2001 From: Fabian Dill Date: Mon, 13 Nov 2023 06:46:40 +0100 Subject: [PATCH] Core: limit parallel APContainer writing (#2443) Co-authored-by: black-sliver <59490463+black-sliver@users.noreply.github.com> --- worlds/Files.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/worlds/Files.py b/worlds/Files.py index ac1acbf3..52d3c7da 100644 --- a/worlds/Files.py +++ b/worlds/Files.py @@ -2,11 +2,18 @@ from __future__ import annotations import json import zipfile +import os +import threading from typing import ClassVar, Dict, Tuple, Any, Optional, Union, BinaryIO import bsdiff4 +semaphore = threading.Semaphore(os.cpu_count() or 4) + +del threading +del os + class AutoPatchRegister(type): patch_types: ClassVar[Dict[str, AutoPatchRegister]] = {} @@ -57,11 +64,12 @@ class APContainer: zip_file = file if file else self.path if not zip_file: raise FileNotFoundError(f"Cannot write {self.__class__.__name__} due to no path provided.") - with zipfile.ZipFile(zip_file, "w", self.compression_method, True, self.compression_level) \ - as zf: - if file: - self.path = zf.filename - self.write_contents(zf) + with semaphore: # TODO: remove semaphore once generate_output has a thread limit + with zipfile.ZipFile( + zip_file, "w", self.compression_method, True, self.compression_level) as zf: + if file: + self.path = zf.filename + self.write_contents(zf) def write_contents(self, opened_zipfile: zipfile.ZipFile) -> None: manifest = self.get_manifest()