voussoir/etiquette
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Use batches of 999 for get_things_by_id.

Browse source 
SQLite has a limit on how many variables you can use per query.
This commit is contained in:
voussoir 2018-05-01 17:48:02 -07:00
parent 97e1f881d0
commit fa97512586
1 changed files with 8 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
etiquette/photodb.py
View file

@ -1455,22 +1455,23 @@ class PhotoDB(
thing_cache = self.caches[thing_type] thing_cache = self.caches[thing_type]
ids_needed = set() ids_needed = set()
things = set()
for thing_id in thing_ids: for thing_id in thing_ids:
try: try:
thing = thing_cache[thing_id] thing = thing_cache[thing_id]
except KeyError: except KeyError:
ids_needed.add(thing_id) ids_needed.add(thing_id)
else: else:
things.add(thing) yield thing
yield from things ids_needed = list(ids_needed)
while ids_needed:
# SQLite3 has a limit of 999 ? in a query, so we must batch them.
id_batch = ids_needed[:999]
ids_needed = ids_needed[999:]
if ids_needed: qmarks = '(%s)' % ','.join('?' * len(id_batch))
qmarks = '(%s)' % ','.join('?' * len(ids_needed))
query = 'SELECT * FROM %s WHERE id IN %s' % (thing_map['table'], qmarks) query = 'SELECT * FROM %s WHERE id IN %s' % (thing_map['table'], qmarks)
bindings = list(ids_needed) more_things = self.sql_select(query, id_batch)
more_things = self.sql_select(query, bindings)
for thing_row in more_things: for thing_row in more_things:
thing = thing_map['class'](self, db_row=thing_row) thing = thing_map['class'](self, db_row=thing_row)
thing_cache[thing.id] = thing thing_cache[thing.id] = thing