From eac32a15bee4a4371f41bf42c2ac03a498ee022d Mon Sep 17 00:00:00 2001 From: Ethan Dalool Date: Tue, 23 Mar 2021 12:15:51 -0700 Subject: [PATCH] Use threadpool job generator instead of list. --- pixelcanvas.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/pixelcanvas.py b/pixelcanvas.py index 610a9dc..d7f1970 100644 --- a/pixelcanvas.py +++ b/pixelcanvas.py @@ -176,16 +176,15 @@ def download_bigchunk_range(bigchunk_xy1, bigchunk_xy2, shuffle=False, threads=1 Given (UPPERLEFT_X, UPPERLEFT_Y), (LOWERRIGHT_X, LOWERRIGHT_Y), download multiple bigchunks, and yield all of the small chunks. ''' + if threads < 1: + raise ValueError(threads) + log.debug('Downloading bigchunk range %s-%s', bigchunk_xy1, bigchunk_xy2) bigchunks = bigchunk_range_iterator(bigchunk_xy1, bigchunk_xy2) - if shuffle: bigchunks = list(bigchunks) random.shuffle(bigchunks) - if threads < 1: - raise ValueError(threads) - if threads == 1: for (x, y) in bigchunks: chunks = download_bigchunk(x, y) @@ -193,13 +192,12 @@ def download_bigchunk_range(bigchunk_xy1, bigchunk_xy2, shuffle=False, threads=1 else: pool = threadpool.ThreadPool(size=threads) - kwargss = [ + job_gen = ( {'function': download_bigchunk, 'args': (x, y), 'name': (x, y),} for (x, y) in bigchunks - ] - jobs = pool.add_many(kwargss) - while jobs: - job = jobs.pop(0) + ) + pool.add_generator(job_gen) + for job in pool.result_generator(): job.join() if job.exception: raise job.exception