From d9360f1ceafe0595ef2f72a54df5e6376f6c29d3 Mon Sep 17 00:00:00 2001 From: Brett Smith Date: Thu, 17 Sep 2020 10:24:13 -0400 Subject: [PATCH] audit_report: Use concurrent.futures for parallelization. This is basically a pure maintainability change: concurrent.futures is the nicest API that's available in both Python 3.6 and 3.7, and our other tools are using it. --- conservancy_beancount/tools/audit_report.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/conservancy_beancount/tools/audit_report.py b/conservancy_beancount/tools/audit_report.py index b5e2670..428185d 100644 --- a/conservancy_beancount/tools/audit_report.py +++ b/conservancy_beancount/tools/audit_report.py @@ -15,9 +15,9 @@ # along with this program. If not, see . import argparse +import concurrent.futures as futmod import datetime import logging -import multiprocessing import os import runpy import sys @@ -198,10 +198,10 @@ def main(arglist: Optional[Sequence[str]]=None, logger.critical("no books available to load") return os.EX_NOINPUT - with multiprocessing.Pool(args.jobs, maxtasksperchild=1) as pool: + with futmod.ProcessPoolExecutor(args.jobs) as pool: logger.debug("%s: process pool ready with %s workers", now_s(), args.jobs) fy_paths = books._iter_fy_books(fy.range(args.audit_year - 1, args.end_date)) - check_results = pool.imap_unordered(bean_check, fy_paths) + check_results = pool.map(bean_check, fy_paths) if all(exitcode == 0 for exitcode in check_results): logger.debug("%s: bean-check passed", now_s()) else: @@ -214,10 +214,10 @@ def main(arglist: Optional[Sequence[str]]=None, return os.EX_DATAERR report_results = [ - pool.apply_async(report_func, (arglist,), {'config': config}) + pool.submit(report_func, arglist, config=config) for report_func, arglist in reports ] - report_errors = [res.get() for res in report_results if res.get() != 0] + report_errors = [res.result() for res in report_results if res.result() != 0] if not report_errors: logger.debug("%s: all reports generated", now_s()) else: