Commit chunks regularly so that http errors don't lose big updates.
This commit is contained in:
parent
0b11b522c9
commit
318f4b379e
1 changed files with 3 additions and 1 deletions
|
@ -140,7 +140,9 @@ def insert_chunk(chunk_x, chunk_y, data, commit=True):
|
|||
sql.commit()
|
||||
|
||||
def insert_chunks(chunks, commit=True):
|
||||
for chunk in chunks:
|
||||
for (index, chunk) in enumerate(chunks):
|
||||
if index % 25000 == 0 and commit:
|
||||
sql.commit()
|
||||
insert_chunk(*chunk, commit=False)
|
||||
if commit:
|
||||
sql.commit()
|
||||
|
|
Loading…
Reference in a new issue