Commit chunks regularly so that http errors don't lose big updates.
This commit is contained in:
parent
0b11b522c9
commit
318f4b379e
1 changed files with 3 additions and 1 deletions
|
@ -140,7 +140,9 @@ def insert_chunk(chunk_x, chunk_y, data, commit=True):
|
||||||
sql.commit()
|
sql.commit()
|
||||||
|
|
||||||
def insert_chunks(chunks, commit=True):
|
def insert_chunks(chunks, commit=True):
|
||||||
for chunk in chunks:
|
for (index, chunk) in enumerate(chunks):
|
||||||
|
if index % 25000 == 0 and commit:
|
||||||
|
sql.commit()
|
||||||
insert_chunk(*chunk, commit=False)
|
insert_chunk(*chunk, commit=False)
|
||||||
if commit:
|
if commit:
|
||||||
sql.commit()
|
sql.commit()
|
||||||
|
|
Loading…
Reference in a new issue