voussoir/pixelcanvas
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Use threadpool job generator instead of list.

Browse source
This commit is contained in:
voussoir 2021-03-23 12:15:51 -07:00
parent 695507bcec
commit eac32a15be
No known key found for this signature in database
GPG key ID: 5F7554F8C26DACCB
1 changed files with 7 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
pixelcanvas.py
View file

@ -176,16 +176,15 @@ def download_bigchunk_range(bigchunk_xy1, bigchunk_xy2, shuffle=False, threads=1
Given (UPPERLEFT_X, UPPERLEFT_Y), (LOWERRIGHT_X, LOWERRIGHT_Y), Given (UPPERLEFT_X, UPPERLEFT_Y), (LOWERRIGHT_X, LOWERRIGHT_Y),
download multiple bigchunks, and yield all of the small chunks. download multiple bigchunks, and yield all of the small chunks.
''' '''
if threads < 1:
raise ValueError(threads)
log.debug('Downloading bigchunk range %s-%s', bigchunk_xy1, bigchunk_xy2) log.debug('Downloading bigchunk range %s-%s', bigchunk_xy1, bigchunk_xy2)
bigchunks = bigchunk_range_iterator(bigchunk_xy1, bigchunk_xy2) bigchunks = bigchunk_range_iterator(bigchunk_xy1, bigchunk_xy2)
if shuffle: if shuffle:
bigchunks = list(bigchunks) bigchunks = list(bigchunks)
random.shuffle(bigchunks) random.shuffle(bigchunks)
if threads < 1:
raise ValueError(threads)
if threads == 1: if threads == 1:
for (x, y) in bigchunks: for (x, y) in bigchunks:
chunks = download_bigchunk(x, y) chunks = download_bigchunk(x, y)
@ -193,13 +192,12 @@ def download_bigchunk_range(bigchunk_xy1, bigchunk_xy2, shuffle=False, threads=1
else: else:
pool = threadpool.ThreadPool(size=threads) pool = threadpool.ThreadPool(size=threads)
kwargss = [ job_gen = (
{'function': download_bigchunk, 'args': (x, y), 'name': (x, y),} {'function': download_bigchunk, 'args': (x, y), 'name': (x, y),}
for (x, y) in bigchunks for (x, y) in bigchunks
] )
jobs = pool.add_many(kwargss) pool.add_generator(job_gen)
while jobs: for job in pool.result_generator():
job = jobs.pop(0)
job.join() job.join()
if job.exception: if job.exception:
raise job.exception raise job.exception