Use threadpool job generator instead of list.

This commit is contained in:
voussoir 2021-03-23 12:15:51 -07:00
parent 695507bcec
commit eac32a15be
No known key found for this signature in database
GPG key ID: 5F7554F8C26DACCB

View file

@ -176,16 +176,15 @@ def download_bigchunk_range(bigchunk_xy1, bigchunk_xy2, shuffle=False, threads=1
Given (UPPERLEFT_X, UPPERLEFT_Y), (LOWERRIGHT_X, LOWERRIGHT_Y),
download multiple bigchunks, and yield all of the small chunks.
'''
if threads < 1:
raise ValueError(threads)
log.debug('Downloading bigchunk range %s-%s', bigchunk_xy1, bigchunk_xy2)
bigchunks = bigchunk_range_iterator(bigchunk_xy1, bigchunk_xy2)
if shuffle:
bigchunks = list(bigchunks)
random.shuffle(bigchunks)
if threads < 1:
raise ValueError(threads)
if threads == 1:
for (x, y) in bigchunks:
chunks = download_bigchunk(x, y)
@ -193,13 +192,12 @@ def download_bigchunk_range(bigchunk_xy1, bigchunk_xy2, shuffle=False, threads=1
else:
pool = threadpool.ThreadPool(size=threads)
kwargss = [
job_gen = (
{'function': download_bigchunk, 'args': (x, y), 'name': (x, y),}
for (x, y) in bigchunks
]
jobs = pool.add_many(kwargss)
while jobs:
job = jobs.pop(0)
)
pool.add_generator(job_gen)
for job in pool.result_generator():
job.join()
if job.exception:
raise job.exception