diff --git a/sklbench/runner/implementation.py b/sklbench/runner/implementation.py index cac0bba4..59642d00 100644 --- a/sklbench/runner/implementation.py +++ b/sklbench/runner/implementation.py @@ -17,7 +17,7 @@ import argparse import json -from multiprocessing import Pool +from concurrent.futures import ThreadPoolExecutor from typing import Dict, List, Tuple, Union from psutil import cpu_count @@ -102,8 +102,8 @@ def run_benchmarks(args: argparse.Namespace) -> int: logger.debug(f"Unique dataset names to load:\n{list(dataset_cases.keys())}") n_proc = min([16, cpu_count(), n_datasets]) logger.info(f"Prefetching {n_datasets} datasets with {n_proc} processes") - with Pool(n_proc) as pool: - pool.map(load_data_with_cleanup, dataset_cases.values()) + with ThreadPoolExecutor(max_workers=n_proc) as executor: + list(executor.map(load_data_with_cleanup, dataset_cases.values())) # run bench_cases return_code, result = call_benchmarks(