from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Callable, List, Optional, TypeVar from .config import Config from .env import Environment T = TypeVar("T") R = TypeVar("R") def parallelize( worker: Callable[[T], R], items: List[T], env: Environment, executor: Optional[ThreadPoolExecutor] = None, ) -> List[R]: if executor is None: executor = ThreadPoolExecutor(max_workers=Config.max_workers) with executor as exec: futures = [exec.submit(worker, item) for item in items] env.logger.info(f"submitted {len(futures)} tasks to executor ₰˜.༄") return [f.result() for f in as_completed(futures)]