Skip to content

Commit

Permalink
🏗️ Arbitrarily cap unarchiving to 2 workers (#2384)
Browse files Browse the repository at this point in the history
* Cap unarchiving ProcessTool by default to 2 workers
  • Loading branch information
mguidon authored Jun 16, 2021
1 parent a18f686 commit cfdf4f8
Showing 1 changed file with 8 additions and 3 deletions.
11 changes: 8 additions & 3 deletions packages/service-library/src/servicelib/archiving_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
from pathlib import Path
from typing import Iterator, List, Set

MAX_UNARCHIVING_WORKER_COUNT = 2

log = logging.getLogger(__name__)


Expand Down Expand Up @@ -67,7 +69,10 @@ def ensure_destination_subdirectories_exist(


async def unarchive_dir(
archive_to_extract: Path, destination_folder: Path
archive_to_extract: Path,
destination_folder: Path,
*,
max_workers: int = MAX_UNARCHIVING_WORKER_COUNT,
) -> Set[Path]:
"""Extracts zipped file archive_to_extract to destination_folder,
preserving all relative files and folders inside the archive
Expand All @@ -76,7 +81,7 @@ async def unarchive_dir(
all tree leafs, which might include files or empty folders
"""
with zipfile.ZipFile(archive_to_extract, mode="r") as zip_file_handler:
with ProcessPoolExecutor() as pool:
with ProcessPoolExecutor(max_workers=max_workers) as pool:
loop = asyncio.get_event_loop()

# running in process poll is not ideal for concurrency issues
Expand Down Expand Up @@ -135,7 +140,7 @@ def _serial_add_to_archive(
async def archive_dir(
dir_to_compress: Path, destination: Path, compress: bool, store_relative_path: bool
) -> bool:
""" Returns True if successuly archived """
"""Returns True if successuly archived"""
with ProcessPoolExecutor(max_workers=1) as pool:
return await asyncio.get_event_loop().run_in_executor(
pool,
Expand Down

0 comments on commit cfdf4f8

Please sign in to comment.