Use while and pop so that finished chunks can be garbaged.

Keeps memory usage stable instead of infinitely growing!
This commit is contained in:
voussoir 2020-02-12 19:33:16 -08:00
parent 488b816b7b
commit 80f9330cb4

View file

@ -191,7 +191,9 @@ def download_bigchunk_range(bigchunk_xy1, bigchunk_xy2, threads=1):
for (x, y) in bigchunks for (x, y) in bigchunks
] ]
jobs = pool.add_many(kwargss) jobs = pool.add_many(kwargss)
while jobs:
for job in jobs: for job in jobs:
job = jobs.pop(0)
job.join() job.join()
if job.exception: if job.exception:
raise job.exception raise job.exception