monorail
/
Archipelago
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Core: limit parallel APContainer writing (#2443) 

Co-authored-by: black-sliver <59490463+black-sliver@users.noreply.github.com>
This commit is contained in:
Fabian Dill 2023-11-13 06:46:40 +01:00 committed by GitHub
parent cc0ea6a9e9
commit ef7d8a6b4f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 13 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
worlds/Files.py
View File

@ -2,11 +2,18 @@ from __future__ import annotations
import json import json
import zipfile import zipfile
import os
import threading
from typing import ClassVar, Dict, Tuple, Any, Optional, Union, BinaryIO from typing import ClassVar, Dict, Tuple, Any, Optional, Union, BinaryIO
import bsdiff4 import bsdiff4
semaphore = threading.Semaphore(os.cpu_count() or 4)
del threading
del os
class AutoPatchRegister(type): class AutoPatchRegister(type):
patch_types: ClassVar[Dict[str, AutoPatchRegister]] = {} patch_types: ClassVar[Dict[str, AutoPatchRegister]] = {}
@ -57,11 +64,12 @@ class APContainer:
zip_file = file if file else self.path zip_file = file if file else self.path
if not zip_file: if not zip_file:
raise FileNotFoundError(f"Cannot write {self.__class__.__name__} due to no path provided.") raise FileNotFoundError(f"Cannot write {self.__class__.__name__} due to no path provided.")
with zipfile.ZipFile(zip_file, "w", self.compression_method, True, self.compression_level) \ with semaphore: # TODO: remove semaphore once generate_output has a thread limit
as zf: with zipfile.ZipFile(
if file: zip_file, "w", self.compression_method, True, self.compression_level) as zf:
self.path = zf.filename if file:
self.write_contents(zf) self.path = zf.filename
self.write_contents(zf)
def write_contents(self, opened_zipfile: zipfile.ZipFile) -> None: def write_contents(self, opened_zipfile: zipfile.ZipFile) -> None:
manifest = self.get_manifest() manifest = self.get_manifest()