From 0615c3f1684b01c1fa6fc54fc8c53f7632179fa3 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Sun, 8 Sep 2019 11:06:51 +0200 Subject: depcheck: don't litter temporary files on error When a multiprocessing.Pool is garbage collected or __exit__ed, it calls the terminate method. In that case, temporary files are not cleaned up. This happens for example, when an exception is raised in one of the workers. Thus we close and join the pool explicitly. --- depcheck.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/depcheck.py b/depcheck.py index 743a09d..e417c72 100755 --- a/depcheck.py +++ b/depcheck.py @@ -437,6 +437,18 @@ def main_docheck(mirror, architecture): return (architecture, check_bdsat(mirror, architecture)) +class SequentialPool: + """Sequential variant of multiprocessing.Pool for debugging.""" + def __enter__(self): + return self + def __exit__(self, *args): + pass + def close(self): + pass + def join(self): + pass + imap_unordered = map + def main(): argp = argparse.ArgumentParser() argp.add_argument('-m', '--mirror', @@ -457,11 +469,15 @@ def main(): return print("checking %s" % " ".join(sorted(archs))) now = datetime.datetime.utcnow().replace(microsecond=0) - mapper = multiprocessing.Pool().imap_unordered if args.parallel else map - for architecture, state in mapper(functools.partial(main_docheck, mirror), - archs): - print("update %s" % architecture) - update_depcheck(mirror, db, now, architecture, state) + with multiprocessing.Pool() if args.parallel else SequentialPool() as pool: + docheck = functools.partial(main_docheck, mirror) + try: + for architecture, state in pool.imap_unordered(docheck, archs): + print("update %s" % architecture) + update_depcheck(mirror, db, now, architecture, state) + finally: + pool.close() + pool.join() if __name__ == "__main__": main() -- cgit v1.2.3