From f510bb88c0a48ca66dc26a284023312792fba974 Mon Sep 17 00:00:00 2001 From: Ethan Dalool Date: Wed, 15 Sep 2021 22:41:38 -0700 Subject: [PATCH] Use voussoirkit/worms. --- etiquette/decorators.py | 48 +-- etiquette/exceptions.py | 11 +- etiquette/objects.py | 353 ++++++++---------- etiquette/photodb.py | 645 ++++++++------------------------- utilities/database_upgrader.py | 75 ++-- 5 files changed, 353 insertions(+), 779 deletions(-) diff --git a/etiquette/decorators.py b/etiquette/decorators.py index 1c3ff30..5e9a7b4 100644 --- a/etiquette/decorators.py +++ b/etiquette/decorators.py @@ -4,13 +4,6 @@ import warnings from . import exceptions -def _get_relevant_photodb(instance): - if isinstance(instance, objects.ObjectBase): - photodb = instance.photodb - else: - photodb = instance - return photodb - def required_feature(features): ''' Declare that the photodb or object method requires certain 'enable_*' @@ -22,7 +15,7 @@ def required_feature(features): def wrapper(method): @functools.wraps(method) def wrapped_required_feature(self, *args, **kwargs): - photodb = _get_relevant_photodb(self) + photodb = self._photodb config = photodb.config['enable_feature'] # Using the received string like "photo.new", try to navigate the @@ -67,42 +60,3 @@ def time_me(function): print(f'{function.__name__}: {duration:0.8f}') return result return timed_function - -def transaction(method): - ''' - Open a savepoint before running the method. - If the method fails, roll back to that savepoint. - ''' - @functools.wraps(method) - def wrapped_transaction(self, *args, **kwargs): - if isinstance(self, objects.ObjectBase): - self.assert_not_deleted() - - photodb = _get_relevant_photodb(self) - - commit = kwargs.pop('commit', False) - is_root = len(photodb.savepoints) == 0 - - savepoint_id = photodb.savepoint(message=method.__qualname__) - - try: - result = method(self, *args, **kwargs) - except Exception as exc: - photodb.log.debug(f'{method} raised {repr(exc)}.') - photodb.rollback(savepoint=savepoint_id) - raise - - if commit: - photodb.commit(message=method.__qualname__) - elif not is_root: - photodb.release_savepoint(savepoint=savepoint_id) - return result - - return wrapped_transaction - -# Circular dependency. -# I would like to un-circularize this, but as long as objects and photodb are -# using the same decorators, and the decorator needs to follow the photodb -# instance of the object... -# I'd rather not create separate decorators, or write hasattr-based decisions. -from . import objects diff --git a/etiquette/exceptions.py b/etiquette/exceptions.py index c16f3dd..2ee6181 100644 --- a/etiquette/exceptions.py +++ b/etiquette/exceptions.py @@ -71,6 +71,9 @@ class NoSuchUser(NoSuch): # EXISTS ########################################################################################### +# The following inits store a copy of the object so that the exception catcher +# can do something with it. It's not related to the string formatting. + class Exists(EtiquetteException): pass @@ -160,14 +163,6 @@ class Unauthorized(EtiquetteException): class WrongLogin(EtiquetteException): error_message = 'Wrong username-password combination.' -# SQL ERRORS ####################################################################################### - -class BadSQL(EtiquetteException): - pass - -class BadTable(BadSQL): - error_message = 'Table "{}" does not exist.' - # GENERAL ERRORS ################################################################################### class BadDataDirectory(EtiquetteException): diff --git a/etiquette/objects.py b/etiquette/objects.py index 2f119de..790052c 100644 --- a/etiquette/objects.py +++ b/etiquette/objects.py @@ -20,6 +20,10 @@ from voussoirkit import sentinel from voussoirkit import spinal from voussoirkit import sqlhelpers from voussoirkit import stringtools +from voussoirkit import vlogging +from voussoirkit import worms + +log = vlogging.getLogger(__name__) from . import constants from . import decorators @@ -28,52 +32,14 @@ from . import helpers BAIL = sentinel.Sentinel('BAIL') -def normalize_db_row(db_row, table) -> dict: - ''' - Raises KeyError if table is not one of the recognized tables. - - Raises TypeError if db_row is not the right type. - ''' - if isinstance(db_row, dict): - return db_row - - if isinstance(db_row, (list, tuple)): - return dict(zip(constants.SQL_COLUMNS[table], db_row)) - - raise TypeError(f'db_row should be {dict}, {list}, or {tuple}, not {type(db_row)}.') - -class ObjectBase: +class ObjectBase(worms.Object): def __init__(self, photodb): - super().__init__() + super().__init__(photodb) self.photodb = photodb + # Used by decorators.required_feature. + self._photodb = photodb self.deleted = False - def __reinit__(self): - ''' - Reload the row from the database and do __init__ with it. - ''' - row = self.photodb.sql_select_one(f'SELECT * FROM {self.table} WHERE id == ?', [self.id]) - if row is None: - self.deleted = True - else: - self.__init__(self.photodb, row) - - def __eq__(self, other): - return ( - isinstance(other, type(self)) and - self.photodb == other.photodb and - self.id == other.id - ) - - def __format__(self, formcode): - if formcode == 'r': - return repr(self) - else: - return str(self) - - def __hash__(self): - return hash(self.id) - @staticmethod def normalize_author_id(author_id) -> typing.Optional[str]: ''' @@ -126,7 +92,7 @@ class GroupableMixin(metaclass=abc.ABCMeta): if not children: return - self.photodb.sql_delete(table=self.group_table, pairs={'parentid': self.id}) + self.photodb.delete(table=self.group_table, pairs={'parentid': self.id}) parents = self.get_parents() for parent in parents: @@ -141,7 +107,7 @@ class GroupableMixin(metaclass=abc.ABCMeta): if self.has_child(member): return BAIL - self.photodb.log.info('Adding child %s to %s.', member, self) + log.info('Adding child %s to %s.', member, self) for my_ancestor in self.walk_parents(): if my_ancestor == member: @@ -151,7 +117,7 @@ class GroupableMixin(metaclass=abc.ABCMeta): 'parentid': self.id, 'memberid': member.id, } - self.photodb.sql_insert(table=self.group_table, data=data) + self.photodb.insert(table=self.group_table, data=data) @abc.abstractmethod def add_child(self, member): @@ -197,23 +163,21 @@ class GroupableMixin(metaclass=abc.ABCMeta): # Note that this part comes after the deletion of children to prevent # issues of recursion. - self.photodb.sql_delete(table=self.group_table, pairs={'memberid': self.id}) + self.photodb.delete(table=self.group_table, pairs={'memberid': self.id}) self._uncache() self.deleted = True def get_children(self) -> set: - child_rows = self.photodb.sql_select( + child_ids = self.photodb.select_column( f'SELECT memberid FROM {self.group_table} WHERE parentid == ?', [self.id] ) - child_ids = (child_id for (child_id,) in child_rows) children = set(self.group_getter_many(child_ids)) return children def get_parents(self) -> set: query = f'SELECT parentid FROM {self.group_table} WHERE memberid == ?' - parent_rows = self.photodb.sql_select(query, [self.id]) - parent_ids = (parent_id for (parent_id,) in parent_rows) + parent_ids = self.photodb.select_column(query, [self.id]) parents = set(self.group_getter_many(parent_ids)) return parents @@ -222,18 +186,18 @@ class GroupableMixin(metaclass=abc.ABCMeta): def has_any_child(self) -> bool: query = f'SELECT 1 FROM {self.group_table} WHERE parentid == ? LIMIT 1' - row = self.photodb.sql_select_one(query, [self.id]) + row = self.photodb.select_one(query, [self.id]) return row is not None def has_any_parent(self) -> bool: query = f'SELECT 1 FROM {self.group_table} WHERE memberid == ? LIMIT 1' - row = self.photodb.sql_select_one(query, [self.id]) + row = self.photodb.select_one(query, [self.id]) return row is not None def has_child(self, member) -> bool: self.assert_same_type(member) query = f'SELECT 1 FROM {self.group_table} WHERE parentid == ? AND memberid == ?' - row = self.photodb.sql_select_one(query, [self.id, member.id]) + row = self.photodb.select_one(query, [self.id, member.id]) return row is not None def has_descendant(self, descendant) -> bool: @@ -242,20 +206,20 @@ class GroupableMixin(metaclass=abc.ABCMeta): def has_parent(self, parent) -> bool: self.assert_same_type(parent) query = f'SELECT 1 FROM {self.group_table} WHERE parentid == ? AND memberid == ?' - row = self.photodb.sql_select_one(query, [parent.id, self.id]) + row = self.photodb.select_one(query, [parent.id, self.id]) return row is not None def _remove_child(self, member): if not self.has_child(member): return BAIL - self.photodb.log.info('Removing child %s from %s.', member, self) + log.info('Removing child %s from %s.', member, self) pairs = { 'parentid': self.id, 'memberid': member.id, } - self.photodb.sql_delete(table=self.group_table, pairs=pairs) + self.photodb.delete(table=self.group_table, pairs=pairs) @abc.abstractmethod def remove_child(self, member): @@ -295,10 +259,11 @@ class GroupableMixin(metaclass=abc.ABCMeta): class Album(ObjectBase, GroupableMixin): table = 'albums' group_table = 'album_group_rel' + no_such_exception = exceptions.NoSuchAlbum def __init__(self, photodb, db_row): super().__init__(photodb) - db_row = normalize_db_row(db_row, self.table) + db_row = self.photodb.normalize_db_row(db_row, self.table) self.id = db_row['id'] self.title = self.normalize_title(db_row['title']) @@ -349,7 +314,7 @@ class Album(ObjectBase, GroupableMixin): return title def _uncache(self): - self.photodb.caches['album'].remove(self.id) + self.photodb.caches[Album].remove(self.id) def _add_associated_directory(self, path): path = pathclass.Path(path) @@ -360,12 +325,12 @@ class Album(ObjectBase, GroupableMixin): if self.has_associated_directory(path): return - self.photodb.log.info('Adding directory "%s" to %s.', path.absolute_path, self) + log.info('Adding directory "%s" to %s.', path.absolute_path, self) data = {'albumid': self.id, 'directory': path.absolute_path} - self.photodb.sql_insert(table='album_associated_directories', data=data) + self.photodb.insert(table='album_associated_directories', data=data) @decorators.required_feature('album.edit') - @decorators.transaction + @worms.transaction def add_associated_directory(self, path) -> None: ''' Add a directory from which this album will pull files during rescans. @@ -377,7 +342,7 @@ class Album(ObjectBase, GroupableMixin): self._add_associated_directory(path) @decorators.required_feature('album.edit') - @decorators.transaction + @worms.transaction def add_associated_directories(self, paths) -> None: ''' Add multiple associated directories. @@ -388,7 +353,7 @@ class Album(ObjectBase, GroupableMixin): self._add_associated_directory(path) @decorators.required_feature('album.edit') - @decorators.transaction + @worms.transaction def add_child(self, member): ''' Raises exceptions.CantGroupSelf if member is self. @@ -398,17 +363,17 @@ class Album(ObjectBase, GroupableMixin): return super().add_child(member) @decorators.required_feature('album.edit') - @decorators.transaction + @worms.transaction def add_children(self, *args, **kwargs): return super().add_children(*args, **kwargs) def _add_photo(self, photo): - self.photodb.log.info('Adding photo %s to %s.', photo, self) + log.info('Adding photo %s to %s.', photo, self) data = {'albumid': self.id, 'photoid': photo.id} - self.photodb.sql_insert(table='album_photo_rel', data=data) + self.photodb.insert(table='album_photo_rel', data=data) @decorators.required_feature('album.edit') - @decorators.transaction + @worms.transaction def add_photo(self, photo) -> None: if self.has_photo(photo): return @@ -416,7 +381,7 @@ class Album(ObjectBase, GroupableMixin): self._add_photo(photo) @decorators.required_feature('album.edit') - @decorators.transaction + @worms.transaction def add_photos(self, photos) -> None: existing_photos = set(self.get_photos()) photos = set(photos) @@ -429,7 +394,7 @@ class Album(ObjectBase, GroupableMixin): self._add_photo(photo) # Photo.add_tag already has @required_feature - @decorators.transaction + @worms.transaction def add_tag_to_all(self, tag, *, nested_children=True) -> None: ''' Add this tag to every photo in the album. Saves you from having to @@ -449,13 +414,13 @@ class Album(ObjectBase, GroupableMixin): photo.add_tag(tag) @decorators.required_feature('album.edit') - @decorators.transaction + @worms.transaction def delete(self, *, delete_children=False) -> None: - self.photodb.log.info('Deleting %s.', self) + log.info('Deleting %s.', self) GroupableMixin.delete(self, delete_children=delete_children) - self.photodb.sql_delete(table='album_associated_directories', pairs={'albumid': self.id}) - self.photodb.sql_delete(table='album_photo_rel', pairs={'albumid': self.id}) - self.photodb.sql_delete(table='albums', pairs={'id': self.id}) + self.photodb.delete(table='album_associated_directories', pairs={'albumid': self.id}) + self.photodb.delete(table='album_photo_rel', pairs={'albumid': self.id}) + self.photodb.delete(table=Album, pairs={'id': self.id}) self._uncache() self.deleted = True @@ -467,7 +432,7 @@ class Album(ObjectBase, GroupableMixin): return self.id @decorators.required_feature('album.edit') - @decorators.transaction + @worms.transaction def edit(self, title=None, description=None) -> None: ''' Change the title or description. Leave None to keep current value. @@ -486,7 +451,7 @@ class Album(ObjectBase, GroupableMixin): 'title': title, 'description': description, } - self.photodb.sql_update(table='albums', pairs=data, where_key='id') + self.photodb.update(table=Album, pairs=data, where_key='id') self.title = title self.description = description @@ -498,19 +463,18 @@ class Album(ObjectBase, GroupableMixin): return self.id def get_associated_directories(self) -> set[pathclass.Path]: - directory_rows = self.photodb.sql_select( + directories = self.photodb.select_column( 'SELECT directory FROM album_associated_directories WHERE albumid == ?', [self.id] ) - directories = set(pathclass.Path(directory) for (directory,) in directory_rows) + directories = set(pathclass.Path(d) for d in directories) return directories def get_photos(self) -> set: - photo_rows = self.photodb.sql_select( + photo_ids = self.photodb.select_column( 'SELECT photoid FROM album_photo_rel WHERE albumid == ?', [self.id] ) - photo_ids = (photo_id for (photo_id,) in photo_rows) photos = set(self.photodb.get_photos_by_id(photo_ids)) return photos @@ -518,7 +482,7 @@ class Album(ObjectBase, GroupableMixin): ''' Return True if this album has at least 1 associated directory. ''' - row = self.photodb.sql_select_one( + row = self.photodb.select_one( 'SELECT 1 FROM album_associated_directories WHERE albumid == ?', [self.id] ) @@ -532,7 +496,7 @@ class Album(ObjectBase, GroupableMixin): If True, photos in child albums satisfy. If False, only consider this album. ''' - row = self.photodb.sql_select_one( + row = self.photodb.select_one( 'SELECT 1 FROM album_photo_rel WHERE albumid == ? LIMIT 1', [self.id] ) @@ -551,14 +515,14 @@ class Album(ObjectBase, GroupableMixin): def has_associated_directory(self, path) -> bool: path = pathclass.Path(path) - row = self.photodb.sql_select_one( + row = self.photodb.select_one( 'SELECT 1 FROM album_associated_directories WHERE albumid == ? AND directory == ?', [self.id, path.absolute_path] ) return row is not None def has_photo(self, photo) -> bool: - row = self.photodb.sql_select_one( + row = self.photodb.select_one( 'SELECT 1 FROM album_photo_rel WHERE albumid == ? AND photoid == ?', [self.id, photo.id] ) @@ -583,27 +547,27 @@ class Album(ObjectBase, GroupableMixin): return j @decorators.required_feature('album.edit') - @decorators.transaction + @worms.transaction def remove_child(self, *args, **kwargs): return super().remove_child(*args, **kwargs) @decorators.required_feature('album.edit') - @decorators.transaction + @worms.transaction def remove_children(self, *args, **kwargs): return super().remove_children(*args, **kwargs) def _remove_photo(self, photo): - self.photodb.log.info('Removing photo %s from %s.', photo, self) + log.info('Removing photo %s from %s.', photo, self) pairs = {'albumid': self.id, 'photoid': photo.id} - self.photodb.sql_delete(table='album_photo_rel', pairs=pairs) + self.photodb.delete(table='album_photo_rel', pairs=pairs) @decorators.required_feature('album.edit') - @decorators.transaction + @worms.transaction def remove_photo(self, photo) -> None: self._remove_photo(photo) @decorators.required_feature('album.edit') - @decorators.transaction + @worms.transaction def remove_photos(self, photos) -> None: existing_photos = set(self.get_photos()) photos = set(photos) @@ -616,7 +580,7 @@ class Album(ObjectBase, GroupableMixin): self._remove_photo(photo) @decorators.required_feature('album.edit') - @decorators.transaction + @worms.transaction def set_thumbnail_photo(self, photo) -> None: ''' Raises TypeError if photo is not a Photo. @@ -639,7 +603,7 @@ class Album(ObjectBase, GroupableMixin): 'id': self.id, 'thumbnail_photo': photo_id, } - self.photodb.sql_update(table='albums', pairs=pairs, where_key='id') + self.photodb.update(table=Album, pairs=pairs, where_key='id') self._thumbnail_photo = photo def sum_bytes(self, recurse=True) -> int: @@ -660,7 +624,7 @@ class Album(ObjectBase, GroupableMixin): albumids = sqlhelpers.listify(albumids) query = query.format(albumids=albumids) - total = self.photodb.sql_select_one(query)[0] + total = self.photodb.select_one(query)[0] return total def sum_children(self, recurse=True) -> int: @@ -682,7 +646,7 @@ class Album(ObjectBase, GroupableMixin): WHERE parentid == ? ''') bindings = [self.id] - total = self.photodb.sql_select_one(query, bindings)[0] + total = self.photodb.select_one(query, bindings)[0] return total def sum_photos(self, recurse=True) -> int: @@ -704,7 +668,7 @@ class Album(ObjectBase, GroupableMixin): albumids = sqlhelpers.listify(albumids) query = query.format(albumids=albumids) - total = self.photodb.sql_select_one(query)[0] + total = self.photodb.select_one(query)[0] return total # Will add -> Photo when forward references are supported by Python. @@ -728,10 +692,11 @@ class Album(ObjectBase, GroupableMixin): class Bookmark(ObjectBase): table = 'bookmarks' + no_such_exception = exceptions.NoSuchBookmark def __init__(self, photodb, db_row): super().__init__(photodb) - db_row = normalize_db_row(db_row, self.table) + db_row = self.photodb.normalize_db_row(db_row, self.table) self.id = db_row['id'] self.title = self.normalize_title(db_row['title']) @@ -778,12 +743,12 @@ class Bookmark(ObjectBase): return url def _uncache(self): - self.photodb.caches['bookmark'].remove(self.id) + self.photodb.caches[Bookmark].remove(self.id) @decorators.required_feature('bookmark.edit') - @decorators.transaction + @worms.transaction def delete(self) -> None: - self.photodb.sql_delete(table='bookmarks', pairs={'id': self.id}) + self.photodb.delete(table=Bookmark, pairs={'id': self.id}) self._uncache() self.deleted = True @@ -795,7 +760,7 @@ class Bookmark(ObjectBase): return self.id @decorators.required_feature('bookmark.edit') - @decorators.transaction + @worms.transaction def edit(self, title=None, url=None) -> None: ''' Change the title or URL. Leave None to keep current. @@ -814,7 +779,7 @@ class Bookmark(ObjectBase): 'title': title, 'url': url, } - self.photodb.sql_update(table='bookmarks', pairs=data, where_key='id') + self.photodb.update(table=Bookmark, pairs=data, where_key='id') self.title = title self.url = url @@ -837,10 +802,11 @@ class Photo(ObjectBase): Photos are not the actual image data, just the database entry. ''' table = 'photos' + no_such_exception = exceptions.NoSuchPhoto def __init__(self, photodb, db_row): super().__init__(photodb) - db_row = normalize_db_row(db_row, self.table) + db_row = self.photodb.normalize_db_row(db_row, self.table) self.real_path = db_row['filepath'] self.real_path = pathclass.Path(self.real_path) @@ -924,11 +890,11 @@ class Photo(ObjectBase): self.simple_mimetype = self.mimetype.split('/')[0] def _uncache(self): - self.photodb.caches['photo'].remove(self.id) + self.photodb.caches[Photo].remove(self.id) # Will add -> Tag when forward references are supported by Python. @decorators.required_feature('photo.add_remove_tag') - @decorators.transaction + @worms.transaction def add_tag(self, tag): tag = self.photodb.get_tag(name=tag) @@ -940,27 +906,27 @@ class Photo(ObjectBase): # keep our current one. existing = self.has_tag(tag, check_children=True) if existing: - self.photodb.log.debug('Preferring existing %s over %s.', existing, tag) + log.debug('Preferring existing %s over %s.', existing, tag) return existing # If the new tag is more specific, remove our current one for it. for parent in tag.walk_parents(): if self.has_tag(parent, check_children=False): - self.photodb.log.debug('Preferring new %s over %s.', tag, parent) + log.debug('Preferring new %s over %s.', tag, parent) self.remove_tag(parent) - self.photodb.log.info('Applying %s to %s.', tag, self) + log.info('Applying %s to %s.', tag, self) data = { 'photoid': self.id, 'tagid': tag.id } - self.photodb.sql_insert(table='photo_tag_rel', data=data) + self.photodb.insert(table='photo_tag_rel', data=data) data = { 'id': self.id, 'tagged_at': helpers.now(), } - self.photodb.sql_update(table='photos', pairs=data, where_key='id') + self.photodb.update(table=Photo, pairs=data, where_key='id') return tag @@ -982,7 +948,7 @@ class Photo(ObjectBase): return '??? b' # Photo.add_tag already has @required_feature add_remove_tag - @decorators.transaction + @worms.transaction def copy_tags(self, other_photo) -> None: ''' Take all of the tags owned by other_photo and apply them to this photo. @@ -991,23 +957,23 @@ class Photo(ObjectBase): self.add_tag(tag) @decorators.required_feature('photo.edit') - @decorators.transaction + @worms.transaction def delete(self, *, delete_file=False) -> None: ''' Delete the Photo and its relation to any tags and albums. ''' - self.photodb.log.info('Deleting %s.', self) - self.photodb.sql_delete(table='photo_tag_rel', pairs={'photoid': self.id}) - self.photodb.sql_delete(table='album_photo_rel', pairs={'photoid': self.id}) - self.photodb.sql_delete(table='photos', pairs={'id': self.id}) + log.info('Deleting %s.', self) + self.photodb.delete(table='photo_tag_rel', pairs={'photoid': self.id}) + self.photodb.delete(table='album_photo_rel', pairs={'photoid': self.id}) + self.photodb.delete(table=Photo, pairs={'id': self.id}) if delete_file and self.real_path.exists: path = self.real_path.absolute_path if self.photodb.config['recycle_instead_of_delete']: - self.photodb.log.debug('Recycling %s.', path) + log.debug('Recycling %s.', path) action = send2trash.send2trash else: - self.photodb.log.debug('Deleting %s.', path) + log.debug('Deleting %s.', path) action = os.remove self.photodb.on_commit_queue.append({ @@ -1030,7 +996,7 @@ class Photo(ObjectBase): return hms.seconds_to_hms(self.duration) @decorators.required_feature('photo.generate_thumbnail') - @decorators.transaction + @worms.transaction def generate_thumbnail(self, **special) -> pathclass.Path: ''' special: @@ -1040,7 +1006,7 @@ class Photo(ObjectBase): return_filepath = None if self.simple_mimetype == 'image': - self.photodb.log.info('Thumbnailing %s.', self.real_path.absolute_path) + log.info('Thumbnailing %s.', self.real_path.absolute_path) try: image = helpers.generate_image_thumbnail( self.real_path.absolute_path, @@ -1054,7 +1020,7 @@ class Photo(ObjectBase): return_filepath = hopeful_filepath elif self.simple_mimetype == 'video' and constants.ffmpeg: - self.photodb.log.info('Thumbnailing %s.', self.real_path.absolute_path) + log.info('Thumbnailing %s.', self.real_path.absolute_path) try: success = helpers.generate_video_thumbnail( self.real_path.absolute_path, @@ -1066,7 +1032,7 @@ class Photo(ObjectBase): if success: return_filepath = hopeful_filepath except Exception: - self.photodb.log.warning(traceback.format_exc()) + log.warning(traceback.format_exc()) if return_filepath != self.thumbnail: if return_filepath is None: @@ -1077,7 +1043,7 @@ class Photo(ObjectBase): 'id': self.id, 'thumbnail': store_as, } - self.photodb.sql_update(table='photos', pairs=data, where_key='id') + self.photodb.update(table=Photo, pairs=data, where_key='id') self.thumbnail = return_filepath self._uncache() @@ -1089,11 +1055,10 @@ class Photo(ObjectBase): ''' Return the albums of which this photo is a member. ''' - album_rows = self.photodb.sql_select( + album_ids = self.photodb.select_column( 'SELECT albumid FROM album_photo_rel WHERE photoid == ?', [self.id] ) - album_ids = (album_id for (album_id,) in album_rows) albums = set(self.photodb.get_albums_by_id(album_ids)) return albums @@ -1101,11 +1066,10 @@ class Photo(ObjectBase): ''' Return the tags assigned to this Photo. ''' - tag_rows = self.photodb.sql_select( + tag_ids = self.photodb.select_column( 'SELECT tagid FROM photo_tag_rel WHERE photoid == ?', [self.id] ) - tag_ids = (tag_id for (tag_id,) in tag_rows) tags = set(self.photodb.get_tags_by_id(tag_ids)) return tags @@ -1128,7 +1092,7 @@ class Photo(ObjectBase): tag_by_id = {t.id: t for t in tag_options} tag_option_ids = sqlhelpers.listify(tag_by_id) - rel_row = self.photodb.sql_select_one( + rel_row = self.photodb.select_one( f'SELECT tagid FROM photo_tag_rel WHERE photoid == ? AND tagid IN {tag_option_ids}', [self.id] ) @@ -1181,7 +1145,7 @@ class Photo(ObjectBase): return hopeful_filepath # Photo.rename_file already has @required_feature - @decorators.transaction + @worms.transaction def move_file(self, directory) -> None: directory = pathclass.Path(directory) directory.assert_is_directory() @@ -1232,12 +1196,12 @@ class Photo(ObjectBase): self.duration = probe.audio.duration @decorators.required_feature('photo.reload_metadata') - @decorators.transaction + @worms.transaction def reload_metadata(self, hash_kwargs=None) -> None: ''' Load the file's height, width, etc as appropriate for this type of file. ''' - self.photodb.log.info('Reloading metadata for %s.', self) + log.info('Reloading metadata for %s.', self) self.mtime = None self.sha256 = None @@ -1284,12 +1248,12 @@ class Photo(ObjectBase): 'duration': self.duration, 'bytes': self.bytes, } - self.photodb.sql_update(table='photos', pairs=data, where_key='id') + self.photodb.update(table=Photo, pairs=data, where_key='id') self._uncache() @decorators.required_feature('photo.edit') - @decorators.transaction + @worms.transaction def relocate(self, new_filepath) -> None: ''' Point the Photo object to a different filepath. @@ -1311,57 +1275,57 @@ class Photo(ObjectBase): self.photodb.assert_no_such_photo_by_path(filepath=new_filepath) - self.photodb.log.info('Relocating %s to "%s".', self, new_filepath.absolute_path) + log.info('Relocating %s to "%s".', self, new_filepath.absolute_path) data = { 'id': self.id, 'filepath': new_filepath.absolute_path, 'basename': new_filepath.basename, 'extension': new_filepath.extension.no_dot, } - self.photodb.sql_update(table='photos', pairs=data, where_key='id') + self.photodb.update(table=Photo, pairs=data, where_key='id') self.real_path = new_filepath self._assign_mimetype() self._uncache() @decorators.required_feature('photo.add_remove_tag') - @decorators.transaction + @worms.transaction def remove_tag(self, tag) -> None: tag = self.photodb.get_tag(name=tag) - self.photodb.log.info('Removing %s from %s.', tag, self) + log.info('Removing %s from %s.', tag, self) pairs = { 'photoid': self.id, 'tagid': tag.id, } - self.photodb.sql_delete(table='photo_tag_rel', pairs=pairs) + self.photodb.delete(table='photo_tag_rel', pairs=pairs) data = { 'id': self.id, 'tagged_at': helpers.now(), } - self.photodb.sql_update(table='photos', pairs=data, where_key='id') + self.photodb.update(table=Photo, pairs=data, where_key='id') @decorators.required_feature('photo.add_remove_tag') - @decorators.transaction + @worms.transaction def remove_tags(self, tags) -> None: tags = [self.photodb.get_tag(name=tag) for tag in tags] - self.photodb.log.info('Removing %s from %s.', tags, self) + log.info('Removing %s from %s.', tags, self) query = f''' DELETE FROM photo_tag_rel WHERE photoid == "{self.id}" AND tagid IN {sqlhelpers.listify(tag.id for tag in tags)} ''' - self.photodb.sql_execute(query) + self.photodb.execute(query) data = { 'id': self.id, 'tagged_at': helpers.now(), } - self.photodb.sql_update(table='photos', pairs=data, where_key='id') + self.photodb.update(table=Photo, pairs=data, where_key='id') @decorators.required_feature('photo.edit') - @decorators.transaction + @worms.transaction def rename_file(self, new_filename, *, move=False) -> None: ''' Rename the file on the disk as well as in the database. @@ -1395,7 +1359,7 @@ class Photo(ObjectBase): new_path.assert_not_exists() - self.photodb.log.info( + log.info( 'Renaming file "%s" -> "%s".', old_path.absolute_path, new_path.absolute_path, @@ -1422,7 +1386,7 @@ class Photo(ObjectBase): 'basename': new_path.basename, 'extension': new_path.extension.no_dot, } - self.photodb.sql_update(table='photos', pairs=data, where_key='id') + self.photodb.update(table=Photo, pairs=data, where_key='id') self.real_path = new_path self._assign_mimetype() @@ -1448,7 +1412,7 @@ class Photo(ObjectBase): self.__reinit__() @decorators.required_feature('photo.edit') - @decorators.transaction + @worms.transaction def set_override_filename(self, new_filename) -> None: new_filename = self.normalize_override_filename(new_filename) @@ -1456,19 +1420,19 @@ class Photo(ObjectBase): 'id': self.id, 'override_filename': new_filename, } - self.photodb.sql_update(table='photos', pairs=data, where_key='id') + self.photodb.update(table=Photo, pairs=data, where_key='id') self.override_filename = new_filename self.__reinit__() @decorators.required_feature('photo.edit') - @decorators.transaction + @worms.transaction def set_searchhidden(self, searchhidden) -> None: data = { 'id': self.id, 'searchhidden': bool(searchhidden), } - self.photodb.sql_update(table='photos', pairs=data, where_key='id') + self.photodb.update(table=Photo, pairs=data, where_key='id') self.searchhidden = searchhidden class Tag(ObjectBase, GroupableMixin): @@ -1477,10 +1441,11 @@ class Tag(ObjectBase, GroupableMixin): ''' table = 'tags' group_table = 'tag_group_rel' + no_such_exception = exceptions.NoSuchTag def __init__(self, photodb, db_row): super().__init__(photodb) - db_row = normalize_db_row(db_row, self.table) + db_row = self.photodb.normalize_db_row(db_row, self.table) self.id = db_row['id'] # Do not pass the name through the normalizer. It may be grandfathered @@ -1549,7 +1514,7 @@ class Tag(ObjectBase, GroupableMixin): return name def _uncache(self): - self.photodb.caches['tag'].remove(self.id) + self.photodb.caches[Tag].remove(self.id) def _add_child(self, member): ret = super()._add_child(member) @@ -1574,7 +1539,7 @@ class Tag(ObjectBase, GroupableMixin): photo.remove_tags(ancestors) @decorators.required_feature('tag.edit') - @decorators.transaction + @worms.transaction def add_child(self, member): ''' Raises exceptions.CantGroupSelf if member is self. @@ -1589,7 +1554,7 @@ class Tag(ObjectBase, GroupableMixin): return ret @decorators.required_feature('tag.edit') - @decorators.transaction + @worms.transaction def add_children(self, members): ret = super().add_children(members) if ret is BAIL: @@ -1599,7 +1564,7 @@ class Tag(ObjectBase, GroupableMixin): return ret @decorators.required_feature('tag.edit') - @decorators.transaction + @worms.transaction def add_synonym(self, synname) -> str: ''' Raises any exceptions from photodb.normalize_tagname. @@ -1615,7 +1580,7 @@ class Tag(ObjectBase, GroupableMixin): self.photodb.assert_no_such_tag(name=synname) - self.photodb.log.info('New synonym %s of %s.', synname, self.name) + log.info('New synonym %s of %s.', synname, self.name) self.photodb.caches['tag_exports'].clear() @@ -1623,7 +1588,7 @@ class Tag(ObjectBase, GroupableMixin): 'name': synname, 'mastername': self.name, } - self.photodb.sql_insert(table='tag_synonyms', data=data) + self.photodb.insert(table='tag_synonyms', data=data) if self._cached_synonyms is not None: self._cached_synonyms.add(synname) @@ -1631,7 +1596,7 @@ class Tag(ObjectBase, GroupableMixin): return synname @decorators.required_feature('tag.edit') - @decorators.transaction + @worms.transaction def convert_to_synonym(self, mastertag) -> None: ''' Convert this tag into a synonym for a different tag. @@ -1650,7 +1615,7 @@ class Tag(ObjectBase, GroupableMixin): data = { 'mastername': (self.name, mastertag.name), } - self.photodb.sql_update(table='tag_synonyms', pairs=data, where_key='mastername') + self.photodb.update(table='tag_synonyms', pairs=data, where_key='mastername') # Because these were two separate tags, perhaps in separate trees, it # is possible for a photo to have both at the moment. @@ -1671,8 +1636,7 @@ class Tag(ObjectBase, GroupableMixin): ) ''' bindings = [self.id, mastertag.id] - photo_rows = self.photodb.sql_execute(query, bindings) - replace_photoids = [photo_id for (photo_id,) in photo_rows] + replace_photoids = list(self.photodb.select_column(query, bindings)) # For those photos that only had the syn, simply replace with master. if replace_photoids: @@ -1683,7 +1647,7 @@ class Tag(ObjectBase, GroupableMixin): AND photoid IN {sqlhelpers.listify(replace_photoids)} ''' bindings = [mastertag.id, self.id] - self.photodb.sql_execute(query, bindings) + self.photodb.execute(query, bindings) # For photos that have the old tag and DO already have the new one, # don't worry because the old rels will be deleted when the tag is @@ -1694,19 +1658,19 @@ class Tag(ObjectBase, GroupableMixin): mastertag.add_synonym(self.name) @decorators.required_feature('tag.edit') - @decorators.transaction + @worms.transaction def delete(self, *, delete_children=False) -> None: - self.photodb.log.info('Deleting %s.', self) + log.info('Deleting %s.', self) super().delete(delete_children=delete_children) - self.photodb.sql_delete(table='photo_tag_rel', pairs={'tagid': self.id}) - self.photodb.sql_delete(table='tag_synonyms', pairs={'mastername': self.name}) - self.photodb.sql_delete(table='tags', pairs={'id': self.id}) + self.photodb.delete(table='photo_tag_rel', pairs={'tagid': self.id}) + self.photodb.delete(table='tag_synonyms', pairs={'mastername': self.name}) + self.photodb.delete(table=Tag, pairs={'id': self.id}) self.photodb.caches['tag_exports'].clear() self._uncache() self.deleted = True @decorators.required_feature('tag.edit') - @decorators.transaction + @worms.transaction def edit(self, description=None) -> None: ''' Change the description. Leave None to keep current value. @@ -1720,7 +1684,7 @@ class Tag(ObjectBase, GroupableMixin): 'id': self.id, 'description': description, } - self.photodb.sql_update(table='tags', pairs=data, where_key='id') + self.photodb.update(table=Tag, pairs=data, where_key='id') self.description = description self._uncache() @@ -1729,11 +1693,11 @@ class Tag(ObjectBase, GroupableMixin): if self._cached_synonyms is not None: return self._cached_synonyms - syn_rows = self.photodb.sql_select( + synonyms = self.photodb.select_column( 'SELECT name FROM tag_synonyms WHERE mastername == ?', [self.name] ) - synonyms = set(name for (name,) in syn_rows) + synonyms = set(synonyms) self._cached_synonyms = synonyms return synonyms @@ -1756,7 +1720,7 @@ class Tag(ObjectBase, GroupableMixin): return j @decorators.required_feature('tag.edit') - @decorators.transaction + @worms.transaction def remove_child(self, *args, **kwargs): ret = super().remove_child(*args, **kwargs) if ret is BAIL: @@ -1766,7 +1730,7 @@ class Tag(ObjectBase, GroupableMixin): return ret @decorators.required_feature('tag.edit') - @decorators.transaction + @worms.transaction def remove_children(self, *args, **kwargs): ret = super().remove_children(*args, **kwargs) if ret is BAIL: @@ -1776,7 +1740,7 @@ class Tag(ObjectBase, GroupableMixin): return ret @decorators.required_feature('tag.edit') - @decorators.transaction + @worms.transaction def remove_synonym(self, synname) -> str: ''' Delete a synonym. @@ -1793,7 +1757,7 @@ class Tag(ObjectBase, GroupableMixin): if synname == self.name: raise exceptions.NoSuchSynonym(synname) - syn_exists = self.photodb.sql_select_one( + syn_exists = self.photodb.select_one( 'SELECT 1 FROM tag_synonyms WHERE mastername == ? AND name == ?', [self.name, synname] ) @@ -1802,13 +1766,13 @@ class Tag(ObjectBase, GroupableMixin): raise exceptions.NoSuchSynonym(synname) self.photodb.caches['tag_exports'].clear() - self.photodb.sql_delete(table='tag_synonyms', pairs={'name': synname}) + self.photodb.delete(table='tag_synonyms', pairs={'name': synname}) if self._cached_synonyms is not None: self._cached_synonyms.remove(synname) return synname @decorators.required_feature('tag.edit') - @decorators.transaction + @worms.transaction def rename(self, new_name, *, apply_to_synonyms=True) -> None: ''' Rename the tag. Does not affect its relation to Photos or tag groups. @@ -1837,13 +1801,13 @@ class Tag(ObjectBase, GroupableMixin): 'id': self.id, 'name': new_name, } - self.photodb.sql_update(table='tags', pairs=data, where_key='id') + self.photodb.update(table=Tag, pairs=data, where_key='id') if apply_to_synonyms: data = { 'mastername': (old_name, new_name), } - self.photodb.sql_update(table='tag_synonyms', pairs=data, where_key='mastername') + self.photodb.update(table='tag_synonyms', pairs=data, where_key='mastername') self.name = new_name self._uncache() @@ -1853,10 +1817,11 @@ class User(ObjectBase): A dear friend of ours. ''' table = 'users' + no_such_exception = exceptions.NoSuchUser def __init__(self, photodb, db_row): super().__init__(photodb) - db_row = normalize_db_row(db_row, self.table) + db_row = self.photodb.normalize_db_row(db_row, self.table) self.id = db_row['id'] self.username = db_row['username'] @@ -1895,10 +1860,10 @@ class User(ObjectBase): return display_name def _uncache(self): - self.photodb.caches['user'].remove(self.id) + self.photodb.caches[User].remove(self.id) @decorators.required_feature('user.edit') - @decorators.transaction + @worms.transaction def delete(self, *, disown_authored_things) -> None: ''' If disown_authored_things is True then all of this user's albums, @@ -1913,10 +1878,10 @@ class User(ObjectBase): ''' if disown_authored_things: pairs = {'author_id': (self.id, None)} - self.photodb.sql_update(table='albums', pairs=pairs, where_key='author_id') - self.photodb.sql_update(table='bookmarks', pairs=pairs, where_key='author_id') - self.photodb.sql_update(table='photos', pairs=pairs, where_key='author_id') - self.photodb.sql_update(table='tags', pairs=pairs, where_key='author_id') + self.photodb.update(table=Album, pairs=pairs, where_key='author_id') + self.photodb.update(table=Bookmark, pairs=pairs, where_key='author_id') + self.photodb.update(table=Photo, pairs=pairs, where_key='author_id') + self.photodb.update(table=Tag, pairs=pairs, where_key='author_id') else: fail = ( self.has_any_albums() or @@ -1926,7 +1891,7 @@ class User(ObjectBase): ) if fail: raise exceptions.CantDeleteUser(self) - self.photodb.sql_delete(table='users', pairs={'id': self.id}) + self.photodb.delete(table='users', pairs={'id': self.id}) self._uncache() self.deleted = True @@ -1987,22 +1952,22 @@ class User(ObjectBase): def has_any_albums(self) -> bool: query = f'SELECT 1 FROM albums WHERE author_id == ? LIMIT 1' - row = self.photodb.sql_select_one(query, [self.id]) + row = self.photodb.select_one(query, [self.id]) return row is not None def has_any_bookmarks(self) -> bool: query = f'SELECT 1 FROM bookmarks WHERE author_id == ? LIMIT 1' - row = self.photodb.sql_select_one(query, [self.id]) + row = self.photodb.select_one(query, [self.id]) return row is not None def has_any_photos(self) -> bool: query = f'SELECT 1 FROM photos WHERE author_id == ? LIMIT 1' - row = self.photodb.sql_select_one(query, [self.id]) + row = self.photodb.select_one(query, [self.id]) return row is not None def has_any_tags(self) -> bool: query = f'SELECT 1 FROM tags WHERE author_id == ? LIMIT 1' - row = self.photodb.sql_select_one(query, [self.id]) + row = self.photodb.select_one(query, [self.id]) return row is not None def jsonify(self) -> dict: @@ -2016,7 +1981,7 @@ class User(ObjectBase): return j @decorators.required_feature('user.edit') - @decorators.transaction + @worms.transaction def set_display_name(self, display_name) -> None: display_name = self.normalize_display_name( display_name, @@ -2027,11 +1992,11 @@ class User(ObjectBase): 'id': self.id, 'display_name': display_name, } - self.photodb.sql_update(table='users', pairs=data, where_key='id') + self.photodb.update(table='users', pairs=data, where_key='id') self._display_name = display_name @decorators.required_feature('user.edit') - @decorators.transaction + @worms.transaction def set_password(self, password) -> None: if not isinstance(password, bytes): password = password.encode('utf-8') @@ -2043,7 +2008,7 @@ class User(ObjectBase): 'id': self.id, 'password': hashed_password, } - self.photodb.sql_update(table='users', pairs=data, where_key='id') + self.photodb.update(table='users', pairs=data, where_key='id') self.hashed_password = hashed_password class WarningBag: diff --git a/etiquette/photodb.py b/etiquette/photodb.py index e3a1b37..50e19d4 100644 --- a/etiquette/photodb.py +++ b/etiquette/photodb.py @@ -3,7 +3,6 @@ import hashlib import json import os import random -import re import sqlite3 import tempfile import time @@ -13,13 +12,15 @@ import typing from voussoirkit import cacheclass from voussoirkit import configlayers from voussoirkit import expressionmatch -from voussoirkit import passwordy from voussoirkit import pathclass from voussoirkit import ratelimiter from voussoirkit import spinal from voussoirkit import sqlhelpers from voussoirkit import stringtools from voussoirkit import vlogging +from voussoirkit import worms + +log = vlogging.getLogger(__name__) from . import constants from . import decorators @@ -36,16 +37,16 @@ class PDBAlbumMixin: super().__init__() def get_album(self, id) -> objects.Album: - return self.get_thing_by_id('album', id) + return self.get_object_by_id(objects.Album, id) def get_album_count(self) -> int: - return self.sql_select_one('SELECT COUNT(id) FROM albums')[0] + return self.select_one('SELECT COUNT(id) FROM albums')[0] def get_albums(self) -> typing.Iterable[objects.Album]: - return self.get_things(thing_type='album') + return self.get_objects(objects.Album) def get_albums_by_id(self, ids) -> typing.Iterable[objects.Album]: - return self.get_things_by_id('album', ids) + return self.get_objects_by_id(objects.Album, ids) def get_albums_by_path(self, directory) -> typing.Iterable[objects.Album]: ''' @@ -55,12 +56,11 @@ class PDBAlbumMixin: directory = pathclass.Path(directory) query = 'SELECT albumid FROM album_associated_directories WHERE directory == ?' bindings = [directory.absolute_path] - album_rows = self.sql_select(query, bindings) - album_ids = (album_id for (album_id,) in album_rows) + album_ids = self.select_column(query, bindings) return self.get_albums_by_id(album_ids) def get_albums_by_sql(self, query, bindings=None) -> typing.Iterable[objects.Album]: - return self.get_things_by_sql('album', query, bindings) + return self.get_objects_by_sql(objects.Album, query, bindings) def get_albums_within_directory(self, directory) -> typing.Iterable[objects.Album]: # This function is something of a stopgap measure since `search` only @@ -71,11 +71,10 @@ class PDBAlbumMixin: directory.assert_is_directory() pattern = directory.absolute_path.rstrip(os.sep) pattern = f'{pattern}{os.sep}%' - album_rows = self.sql_select( + album_ids = self.select_column( 'SELECT DISTINCT albumid FROM album_associated_directories WHERE directory LIKE ?', [pattern] ) - album_ids = (album_id for (album_id,) in album_rows) albums = self.get_albums_by_id(album_ids) return albums @@ -83,10 +82,10 @@ class PDBAlbumMixin: ''' Yield Albums that have no parent. ''' - return self.get_root_things('album') + return self.get_root_objects(objects.Album) @decorators.required_feature('album.new') - @decorators.transaction + @worms.transaction def new_album( self, title=None, @@ -106,7 +105,7 @@ class PDBAlbumMixin: # Ok. album_id = self.generate_id(table='albums') - self.log.info('New Album: %s %s.', album_id, title) + log.info('New Album: %s %s.', album_id, title) data = { 'id': album_id, @@ -116,9 +115,9 @@ class PDBAlbumMixin: 'thumbnail_photo': None, 'author_id': author_id, } - self.sql_insert(table='albums', data=data) + self.insert(table=objects.Album, data=data) - album = self.get_cached_instance('album', data) + album = self.get_cached_instance(objects.Album, data) associated_directories = associated_directories or () if isinstance(associated_directories, str): @@ -131,24 +130,25 @@ class PDBAlbumMixin: return album - @decorators.transaction + @worms.transaction def purge_deleted_associated_directories(self, albums=None) -> typing.Iterable[pathclass.Path]: - directories = self.sql_select('SELECT DISTINCT directory FROM album_associated_directories') - directories = (pathclass.Path(directory) for (directory,) in directories) - directories = [directory for directory in directories if not directory.is_dir] + query = 'SELECT DISTINCT directory FROM album_associated_directories' + directories = self.select_column(query) + directories = (pathclass.Path(d) for d in directories) + directories = [d for d in directories if not d.is_dir] if not directories: return - self.log.info('Purging associated directories %s.', directories) + log.info('Purging associated directories %s.', directories) - d_query = sqlhelpers.listify(directory.absolute_path for directory in directories) + d_query = sqlhelpers.listify(d.absolute_path for d in directories) query = f'DELETE FROM album_associated_directories WHERE directory in {d_query}' if albums is not None: album_ids = sqlhelpers.listify(a.id for a in albums) query += f' AND albumid IN {album_ids}' - self.sql_execute(query) + self.execute(query) yield from directories - @decorators.transaction + @worms.transaction def purge_empty_albums(self, albums=None) -> typing.Iterable[objects.Album]: if albums is None: to_check = set(self.get_albums()) @@ -173,22 +173,22 @@ class PDBBookmarkMixin: super().__init__() def get_bookmark(self, id) -> objects.Bookmark: - return self.get_thing_by_id('bookmark', id) + return self.get_object_by_id(objects.Bookmark, id) def get_bookmark_count(self) -> int: - return self.sql_select_one('SELECT COUNT(id) FROM bookmarks')[0] + return self.select_one('SELECT COUNT(id) FROM bookmarks')[0] def get_bookmarks(self) -> typing.Iterable[objects.Bookmark]: - return self.get_things(thing_type='bookmark') + return self.get_objects(objects.Bookmark) def get_bookmarks_by_id(self, ids) -> typing.Iterable[objects.Bookmark]: - return self.get_things_by_id('bookmark', ids) + return self.get_objects_by_id(objects.Bookmark, ids) def get_bookmarks_by_sql(self, query, bindings=None) -> typing.Iterable[objects.Bookmark]: - return self.get_things_by_sql('bookmark', query, bindings) + return self.get_objects_by_sql(objects.Bookmark, query, bindings) @decorators.required_feature('bookmark.new') - @decorators.transaction + @worms.transaction def new_bookmark(self, url, title=None, *, author=None) -> objects.Bookmark: # These might raise exceptions. title = objects.Bookmark.normalize_title(title) @@ -197,7 +197,7 @@ class PDBBookmarkMixin: # Ok. bookmark_id = self.generate_id(table='bookmarks') - self.log.info('New Bookmark: %s %s %s.', bookmark_id, title, url) + log.info('New Bookmark: %s %s %s.', bookmark_id, title, url) data = { 'id': bookmark_id, @@ -206,224 +206,37 @@ class PDBBookmarkMixin: 'created': helpers.now(), 'author_id': author_id, } - self.sql_insert(table='bookmarks', data=data) + self.insert(table=objects.Bookmark, data=data) - bookmark = self.get_cached_instance('bookmark', data) + bookmark = self.get_cached_instance(objects.Bookmark, data) return bookmark #################################################################################################### -class PDBCacheManagerMixin: - _THING_CLASSES = { - 'album': - { - 'class': objects.Album, - 'exception': exceptions.NoSuchAlbum, - }, - 'bookmark': - { - 'class': objects.Bookmark, - 'exception': exceptions.NoSuchBookmark, - }, - 'photo': - { - 'class': objects.Photo, - 'exception': exceptions.NoSuchPhoto, - }, - 'tag': - { - 'class': objects.Tag, - 'exception': exceptions.NoSuchTag, - }, - 'user': - { - 'class': objects.User, - 'exception': exceptions.NoSuchUser, - } - } - +class PDBGroupableMixin: def __init__(self): super().__init__() - def clear_all_caches(self) -> None: - self.caches['album'].clear() - self.caches['bookmark'].clear() - self.caches['photo'].clear() - self.caches['tag'].clear() - self.caches['tag_exports'].clear() - self.caches['user'].clear() - - def get_cached_instance(self, thing_type, db_row): + def get_root_objects(self, object_class): ''' - Check if there is already an instance in the cache and return that. - Otherwise, a new instance is created, cached, and returned. - - Note that in order to call this method you have to already have a - db_row which means performing some select. If you only have the ID, - use get_thing_by_id, as there may already be a cached instance to save - you the select. + For Groupable types, yield objects which have no parent. ''' - thing_map = self._THING_CLASSES[thing_type] - - thing_class = thing_map['class'] - thing_table = thing_class.table - thing_cache = self.caches[thing_type] - - if isinstance(db_row, dict): - thing_id = db_row['id'] - else: - thing_index = constants.SQL_INDEX[thing_table] - thing_id = db_row[thing_index['id']] - - try: - thing = thing_cache[thing_id] - except KeyError: - self.log.loud('Cache miss %s %s.', thing_type, thing_id) - thing = thing_class(self, db_row) - thing_cache[thing_id] = thing - return thing - - def get_cached_tag_export(self, function, **kwargs): - if isinstance(function, str): - function = getattr(tag_export, function) - if 'tags' in kwargs: - kwargs['tags'] = tuple(kwargs['tags']) - key = (function.__name__,) + helpers.dict_to_tuple(kwargs) - try: - exp = self.caches['tag_exports'][key] - return exp - except KeyError: - exp = function(**kwargs) - if isinstance(exp, types.GeneratorType): - exp = tuple(exp) - self.caches['tag_exports'][key] = exp - return exp - - def get_root_things(self, thing_type): - ''' - For Groupable types, yield things which have no parent. - ''' - thing_map = self._THING_CLASSES[thing_type] - - thing_class = thing_map['class'] - thing_table = thing_class.table - group_table = thing_class.group_table + object_table = object_class.table + group_table = object_class.group_table query = f''' - SELECT * FROM {thing_table} + SELECT * FROM {object_table} WHERE NOT EXISTS ( SELECT 1 FROM {group_table} - WHERE memberid == {thing_table}.id + WHERE memberid == {object_table}.id ) ''' - rows = self.sql_select(query) + rows = self.select(query) for row in rows: - thing = self.get_cached_instance(thing_type, row) - yield thing - - def get_thing_by_id(self, thing_type, thing_id): - ''' - This method will first check the cache to see if there is already an - instance with that ID, in which case we don't need to perform any SQL - select. If it is not in the cache, then a new instance is created, - cached, and returned. - ''' - thing_map = self._THING_CLASSES[thing_type] - - thing_class = thing_map['class'] - if isinstance(thing_id, thing_class): - # This could be used to check if your old reference to an object is - # still in the cache, or re-select it from the db to make sure it - # still exists and re-cache. - # Probably an uncommon need but... no harm I think. - thing_id = thing_id.id - - thing_cache = self.caches[thing_type] - try: - return thing_cache[thing_id] - except KeyError: - pass - - query = f'SELECT * FROM {thing_class.table} WHERE id == ?' - bindings = [thing_id] - thing_row = self.sql_select_one(query, bindings) - if thing_row is None: - raise thing_map['exception'](thing_id) - thing = thing_class(self, thing_row) - thing_cache[thing_id] = thing - return thing - - def get_things(self, thing_type): - ''' - Yield things, unfiltered, in whatever order they appear in the database. - ''' - thing_map = self._THING_CLASSES[thing_type] - table = thing_map['class'].table - query = f'SELECT * FROM {table}' - - things = self.sql_select(query) - for thing_row in things: - thing = self.get_cached_instance(thing_type, thing_row) - yield thing - - def get_things_by_id(self, thing_type, thing_ids): - ''' - Given multiple IDs, this method will find which ones are in the cache - and which ones need to be selected from the db. - This is better than calling get_thing_by_id in a loop because we can - use a single SQL select to get batches of up to 999 items. - - Note: The order of the output will most likely not match the order of - the input, because we first pull items from the cache before requesting - the rest from the database. - ''' - thing_map = self._THING_CLASSES[thing_type] - thing_class = thing_map['class'] - thing_cache = self.caches[thing_type] - - ids_needed = set() - for thing_id in thing_ids: - try: - thing = thing_cache[thing_id] - except KeyError: - ids_needed.add(thing_id) - else: - yield thing - - if not ids_needed: - return - - self.log.loud('Cache miss %s %s.', thing_type, ids_needed) - - ids_needed = list(ids_needed) - while ids_needed: - # SQLite3 has a limit of 999 ? in a query, so we must batch them. - id_batch = ids_needed[:999] - ids_needed = ids_needed[999:] - - qmarks = ','.join('?' * len(id_batch)) - qmarks = f'({qmarks})' - query = f'SELECT * FROM {thing_class.table} WHERE id IN {qmarks}' - more_things = self.sql_select(query, id_batch) - for thing_row in more_things: - # Normally we would call `get_cached_instance` instead of - # constructing here. But we already know for a fact that this - # object is not in the cache because it made it past the - # previous loop. - thing = thing_class(self, db_row=thing_row) - thing_cache[thing.id] = thing - yield thing - - def get_things_by_sql(self, thing_type, query, bindings=None): - ''' - Use an arbitrary SQL query to select things from the database. - Your query select *, all the columns of the thing's table. - ''' - thing_rows = self.sql_select(query, bindings) - for thing_row in thing_rows: - yield self.get_cached_instance(thing_type, thing_row) + instance = self.get_cached_instance(object_class, row) + yield instance #################################################################################################### @@ -440,23 +253,26 @@ class PDBPhotoMixin: raise exceptions.PhotoExists(existing) def get_photo(self, id) -> objects.Photo: - return self.get_thing_by_id('photo', id) + return self.get_object_by_id(objects.Photo, id) def get_photo_by_path(self, filepath) -> objects.Photo: filepath = pathclass.Path(filepath) query = 'SELECT * FROM photos WHERE filepath == ?' bindings = [filepath.absolute_path] - photo_row = self.sql_select_one(query, bindings) + photo_row = self.select_one(query, bindings) if photo_row is None: raise exceptions.NoSuchPhoto(filepath) - photo = self.get_cached_instance('photo', photo_row) + photo = self.get_cached_instance(objects.Photo, photo_row) return photo def get_photo_count(self) -> int: - return self.sql_select_one('SELECT COUNT(id) FROM photos')[0] + return self.select_one('SELECT COUNT(id) FROM photos')[0] + + def get_photos(self) -> typing.Iterable[objects.Photo]: + return self.get_objects(objects.Photo) def get_photos_by_id(self, ids) -> typing.Iterable[objects.Photo]: - return self.get_things_by_id('photo', ids) + return self.get_objects_by_id(objects.Photo, ids) def get_photos_by_recent(self, count=None) -> typing.Iterable[objects.Photo]: ''' @@ -466,9 +282,9 @@ class PDBPhotoMixin: return query = 'SELECT * FROM photos ORDER BY created DESC' - photo_rows = self.sql_select(query) + photo_rows = self.select(query) for photo_row in photo_rows: - photo = self.get_cached_instance('photo', photo_row) + photo = self.get_cached_instance(objects.Photo, photo_row) yield photo if count is None: @@ -486,10 +302,10 @@ class PDBPhotoMixin: yield from self.get_photos_by_sql(query, bindings) def get_photos_by_sql(self, query, bindings=None) -> typing.Iterable[objects.Photo]: - return self.get_things_by_sql('photo', query, bindings) + return self.get_objects_by_sql(objects.Photo, query, bindings) @decorators.required_feature('photo.new') - @decorators.transaction + @worms.transaction def new_photo( self, filepath, @@ -534,7 +350,7 @@ class PDBPhotoMixin: # Ok. photo_id = self.generate_id(table='photos') - self.log.info('New Photo: %s %s.', photo_id, filepath.absolute_path) + log.info('New Photo: %s %s.', photo_id, filepath.absolute_path) data = { 'id': photo_id, @@ -557,9 +373,9 @@ class PDBPhotoMixin: 'duration': None, 'thumbnail': None, } - self.sql_insert(table='photos', data=data) + self.insert(table=objects.Photo, data=data) - photo = self.get_cached_instance('photo', data) + photo = self.get_cached_instance(objects.Photo, data) if do_metadata: hash_kwargs = hash_kwargs or {} @@ -574,7 +390,7 @@ class PDBPhotoMixin: return photo - @decorators.transaction + @worms.transaction def purge_deleted_files(self, photos=None) -> typing.Iterable[objects.Photo]: ''' Delete Photos whose corresponding file on disk is missing. @@ -625,8 +441,8 @@ class PDBPhotoMixin: offset=None, orderby=None, warning_bag=None, - give_back_parameters=False, + give_back_parameters=False, yield_albums=True, yield_photos=True, ): @@ -723,16 +539,20 @@ class PDBPhotoMixin: back to you as the final object. Without the bag, exceptions may be raised. + YIELD OPTIONS give_back_parameters: If True, the generator's first yield will be a dictionary of all the cleaned up, normalized parameters. The user may have given us loads of trash, so we should show them the formatting we want. yield_albums: - If True, albums which contain photos matching the search will also - be returned. + If True, albums which contain photos matching the search + will be yielded. + + yield_photos: + If True, photos matching the search will be yielded. ''' - start_time = time.time() + start_time = time.perf_counter() maximums = {} minimums = {} @@ -964,14 +784,14 @@ class PDBPhotoMixin: query = f'{"-" * 80}\n{query}\n{"-" * 80}' - self.log.debug('\n%s %s', query, bindings) - # explain = self.sql_execute('EXPLAIN QUERY PLAN ' + query, bindings) + log.debug('\n%s %s', query, bindings) + # explain = self.execute('EXPLAIN QUERY PLAN ' + query, bindings) # print('\n'.join(str(x) for x in explain.fetchall())) - generator = self.sql_select(query, bindings) + generator = self.select(query, bindings) seen_albums = set() results_received = 0 for row in generator: - photo = self.get_cached_instance('photo', row) + photo = self.get_cached_instance(objects.Photo, row) if mimetype and photo.simple_mimetype not in mimetype: continue @@ -1008,181 +828,8 @@ class PDBPhotoMixin: if warning_bag and warning_bag.warnings: yield warning_bag - end_time = time.time() - self.log.debug('Search took %s.', end_time - start_time) - -#################################################################################################### - -class PDBSQLMixin: - def __init__(self): - super().__init__() - self.on_commit_queue = [] - self.on_rollback_queue = [] - self.savepoints = [] - self._cached_sql_tables = None - - def assert_table_exists(self, table) -> None: - if not self._cached_sql_tables: - self._cached_sql_tables = self.get_sql_tables() - if table not in self._cached_sql_tables: - raise exceptions.BadTable(table) - - def commit(self, message=None) -> None: - if message is not None: - self.log.debug('Committing - %s.', message) - - while len(self.on_commit_queue) > 0: - task = self.on_commit_queue.pop(-1) - if isinstance(task, str): - # savepoints. - continue - args = task.get('args', []) - kwargs = task.get('kwargs', {}) - action = task['action'] - try: - action(*args, **kwargs) - except Exception as exc: - self.log.debug(f'{action} raised {repr(exc)}.') - self.rollback() - raise - - self.savepoints.clear() - self.sql.commit() - - def get_sql_tables(self) -> set[str]: - query = 'SELECT name FROM sqlite_master WHERE type = "table"' - table_rows = self.sql_select(query) - tables = set(name for (name,) in table_rows) - return tables - - def release_savepoint(self, savepoint, allow_commit=False) -> None: - ''' - Releasing a savepoint removes that savepoint from the timeline, so that - you can no longer roll back to it. Then your choices are to commit - everything, or roll back to a previous point. If you release the - earliest savepoint, the database will commit. - ''' - if savepoint not in self.savepoints: - self.log.warn('Tried to release nonexistent savepoint %s.', savepoint) - return - - is_commit = savepoint == self.savepoints[0] - if is_commit and not allow_commit: - self.log.debug('Not committing %s without allow_commit=True.', savepoint) - return - - if is_commit: - # We want to perform the on_commit_queue so let's use our commit - # method instead of allowing sql's release to commit. - self.commit() - else: - self.sql_execute(f'RELEASE "{savepoint}"') - self.savepoints = helpers.slice_before(self.savepoints, savepoint) - - def rollback(self, savepoint=None) -> None: - ''' - Given a savepoint, roll the database back to the moment before that - savepoint was created. Keep in mind that a @transaction savepoint is - always created *before* the method actually does anything. - - If no savepoint is provided then rollback the entire transaction. - ''' - if savepoint is not None and savepoint not in self.savepoints: - self.log.warn('Tried to restore nonexistent savepoint %s.', savepoint) - return - - if len(self.savepoints) == 0: - self.log.debug('Nothing to roll back.') - return - - while len(self.on_rollback_queue) > 0: - task = self.on_rollback_queue.pop(-1) - if task == savepoint: - break - if isinstance(task, str): - # Intermediate savepoints. - continue - args = task.get('args', []) - kwargs = task.get('kwargs', {}) - task['action'](*args, **kwargs) - - if savepoint is not None: - self.log.debug('Rolling back to %s.', savepoint) - self.sql_execute(f'ROLLBACK TO "{savepoint}"') - self.savepoints = helpers.slice_before(self.savepoints, savepoint) - self.on_commit_queue = helpers.slice_before(self.on_commit_queue, savepoint) - - else: - self.log.debug('Rolling back.') - self.sql_execute('ROLLBACK') - self.savepoints.clear() - self.on_commit_queue.clear() - - def savepoint(self, message=None) -> str: - savepoint_id = passwordy.random_hex(length=16) - if message: - self.log.log(5, 'Savepoint %s for %s.', savepoint_id, message) - else: - self.log.log(5, 'Savepoint %s.', savepoint_id) - query = f'SAVEPOINT "{savepoint_id}"' - self.sql_execute(query) - self.savepoints.append(savepoint_id) - self.on_commit_queue.append(savepoint_id) - self.on_rollback_queue.append(savepoint_id) - return savepoint_id - - def sql_delete(self, table, pairs) -> None: - self.assert_table_exists(table) - (qmarks, bindings) = sqlhelpers.delete_filler(pairs) - query = f'DELETE FROM {table} {qmarks}' - self.sql_execute(query, bindings) - - def sql_execute(self, query, bindings=[]) -> sqlite3.Cursor: - if bindings is None: - bindings = [] - cur = self.sql.cursor() - self.log.loud('%s %s', query, bindings) - cur.execute(query, bindings) - return cur - - def sql_executescript(self, script) -> None: - ''' - The problem with Python's default executescript is that it executes a - COMMIT before running your script. If I wanted a commit I'd write one! - ''' - lines = re.split(r';(:?\n|$)', script) - lines = (line.strip() for line in lines) - lines = (line for line in lines if line) - cur = self.sql.cursor() - for line in lines: - self.log.loud(line) - cur.execute(line) - - def sql_insert(self, table, data) -> None: - self.assert_table_exists(table) - column_names = constants.SQL_COLUMNS[table] - (qmarks, bindings) = sqlhelpers.insert_filler(column_names, data) - - query = f'INSERT INTO {table} VALUES({qmarks})' - self.sql_execute(query, bindings) - - def sql_select(self, query, bindings=None) -> typing.Iterable: - cur = self.sql_execute(query, bindings) - while True: - fetch = cur.fetchone() - if fetch is None: - break - yield fetch - - def sql_select_one(self, query, bindings=None): - cur = self.sql_execute(query, bindings) - return cur.fetchone() - - def sql_update(self, table, pairs, where_key) -> None: - self.assert_table_exists(table) - (qmarks, bindings) = sqlhelpers.update_filler(pairs, where_key=where_key) - query = f'UPDATE {table} {qmarks}' - self.sql_execute(query, bindings) + end_time = time.perf_counter() + log.debug('Search took %s.', end_time - start_time) #################################################################################################### @@ -1200,8 +847,7 @@ class PDBTagMixin: def _get_all_tag_names(self): query = 'SELECT name FROM tags' - tag_rows = self.sql_select(query) - names = set(name for (name,) in tag_rows) + names = set(self.select_column(query)) return names def get_all_tag_names(self) -> set[str]: @@ -1213,7 +859,7 @@ class PDBTagMixin: def _get_all_synonyms(self): query = 'SELECT name, mastername FROM tag_synonyms' - syn_rows = self.sql_select(query) + syn_rows = self.select(query) synonyms = {syn: tag for (syn, tag) in syn_rows} return synonyms @@ -1223,11 +869,27 @@ class PDBTagMixin: ''' return self.get_cached_tag_export(self._get_all_synonyms) + def get_cached_tag_export(self, function, **kwargs): + if isinstance(function, str): + function = getattr(tag_export, function) + if 'tags' in kwargs: + kwargs['tags'] = tuple(kwargs['tags']) + key = (function.__name__,) + helpers.dict_to_tuple(kwargs) + try: + exp = self.caches['tag_exports'][key] + return exp + except KeyError: + exp = function(**kwargs) + if isinstance(exp, types.GeneratorType): + exp = tuple(exp) + self.caches['tag_exports'][key] = exp + return exp + def get_root_tags(self) -> typing.Iterable[objects.Tag]: ''' Yield Tags that have no parent. ''' - return self.get_root_things('tag') + return self.get_root_objects(objects.Tag) def get_tag(self, name=None, id=None) -> objects.Tag: ''' @@ -1242,7 +904,7 @@ class PDBTagMixin: return self.get_tag_by_name(name) def get_tag_by_id(self, id) -> objects.Tag: - return self.get_thing_by_id('tag', thing_id=id) + return self.get_object_by_id(objects.Tag, id) def get_tag_by_name(self, tagname) -> objects.Tag: if isinstance(tagname, objects.Tag): @@ -1262,39 +924,39 @@ class PDBTagMixin: while True: # Return if it's a toplevel... - tag_row = self.sql_select_one('SELECT * FROM tags WHERE name == ?', [tagname]) + tag_row = self.select_one('SELECT * FROM tags WHERE name == ?', [tagname]) if tag_row is not None: break # ...or resolve the synonym and try again. query = 'SELECT mastername FROM tag_synonyms WHERE name == ?' bindings = [tagname] - name_row = self.sql_select_one(query, bindings) + name_row = self.select_one(query, bindings) if name_row is None: # was not a master tag or synonym raise exceptions.NoSuchTag(tagname) tagname = name_row[0] - tag = self.get_cached_instance('tag', tag_row) + tag = self.get_cached_instance(objects.Tag, tag_row) return tag def get_tag_count(self) -> int: - return self.sql_select_one('SELECT COUNT(id) FROM tags')[0] + return self.select_one('SELECT COUNT(id) FROM tags')[0] def get_tags(self) -> typing.Iterable[objects.Tag]: ''' Yield all Tags in the database. ''' - return self.get_things(thing_type='tag') + return self.get_objects(objects.Tag) def get_tags_by_id(self, ids) -> typing.Iterable[objects.Tag]: - return self.get_things_by_id('tag', ids) + return self.get_objects_by_id(objects.Tag, ids) def get_tags_by_sql(self, query, bindings=None) -> typing.Iterable[objects.Tag]: - return self.get_things_by_sql('tag', query, bindings) + return self.get_objects_by_sql(objects.Tag, query, bindings) @decorators.required_feature('tag.new') - @decorators.transaction + @worms.transaction def new_tag(self, tagname, description=None, *, author=None) -> objects.Tag: ''' Register a new tag and return the Tag object. @@ -1308,7 +970,7 @@ class PDBTagMixin: # Ok. tag_id = self.generate_id(table='tags') - self.log.info('New Tag: %s %s.', tag_id, tagname) + log.info('New Tag: %s %s.', tag_id, tagname) self.caches['tag_exports'].clear() @@ -1319,9 +981,9 @@ class PDBTagMixin: 'created': helpers.now(), 'author_id': author_id, } - self.sql_insert(table='tags', data=data) + self.insert(table=objects.Tag, data=data) - tag = self.get_cached_instance('tag', data) + tag = self.get_cached_instance(objects.Tag, data) return tag @@ -1384,7 +1046,7 @@ class PDBUserMixin: for retry in range(20): user_id = ''.join(random.choices(constants.USER_ID_CHARACTERS, k=length)) - user_exists = self.sql_select_one('SELECT 1 FROM users WHERE id == ?', [user_id]) + user_exists = self.select_one('SELECT 1 FROM users WHERE id == ?', [user_id]) if user_exists is None: break else: @@ -1405,18 +1067,18 @@ class PDBUserMixin: return self.get_user_by_username(username) def get_user_by_id(self, id) -> objects.User: - return self.get_thing_by_id('user', id) + return self.get_object_by_id(objects.User, id) def get_user_by_username(self, username) -> objects.User: - user_row = self.sql_select_one('SELECT * FROM users WHERE username == ?', [username]) + user_row = self.select_one('SELECT * FROM users WHERE username == ?', [username]) if user_row is None: raise exceptions.NoSuchUser(username) - return self.get_cached_instance('user', user_row) + return self.get_cached_instance(objects.User, user_row) def get_user_count(self) -> int: - return self.sql_select_one('SELECT COUNT(id) FROM users')[0] + return self.select_one('SELECT COUNT(id) FROM users')[0] def get_user_id_or_none(self, user_obj_or_id) -> typing.Optional[str]: ''' @@ -1448,13 +1110,13 @@ class PDBUserMixin: return author_id def get_users(self) -> typing.Iterable[objects.User]: - return self.get_things('user') + return self.get_objects(objects.User) def get_users_by_id(self, ids) -> typing.Iterable[objects.User]: - return self.get_things_by_id('user', ids) + return self.get_objects_by_id(objects.User, ids) def get_users_by_sql(self, query, bindings=None) -> typing.Iterable[objects.User]: - return self.get_things_by_sql('user', query, bindings) + return self.get_objects_by_sql(objects.User, query, bindings) @decorators.required_feature('user.login') def login(self, username=None, id=None, *, password) -> objects.User: @@ -1476,7 +1138,7 @@ class PDBUserMixin: return user @decorators.required_feature('user.new') - @decorators.transaction + @worms.transaction def new_user(self, username, password, *, display_name=None) -> objects.User: # These might raise exceptions. self.assert_valid_username(username) @@ -1494,7 +1156,7 @@ class PDBUserMixin: # Ok. user_id = self.generate_user_id() - self.log.info('New User: %s %s.', user_id, username) + log.info('New User: %s %s.', user_id, username) hashed_password = bcrypt.hashpw(password, bcrypt.gensalt()) @@ -1505,9 +1167,9 @@ class PDBUserMixin: 'display_name': display_name, 'created': helpers.now(), } - self.sql_insert(table='users', data=data) + self.insert(table=objects.User, data=data) - return self.get_cached_instance('user', data) + return self.get_cached_instance(objects.User, data) #################################################################################################### @@ -1515,7 +1177,7 @@ class PDBUtilMixin: def __init__(self): super().__init__() - @decorators.transaction + @worms.transaction def digest_directory( self, directory, @@ -1626,14 +1288,13 @@ class PDBUtilMixin: def _normalize_new_photo_ratelimit(new_photo_ratelimit): if new_photo_ratelimit is None: - pass + return new_photo_ratelimit elif isinstance(new_photo_ratelimit, ratelimiter.Ratelimiter): - pass + return new_photo_ratelimit elif isinstance(new_photo_ratelimit, (int, float)): new_photo_ratelimit = ratelimiter.Ratelimiter(allowance=1, period=new_photo_ratelimit) - else: - raise TypeError(new_photo_ratelimit) - return new_photo_ratelimit + return new_photo_ratelimit + raise TypeError(new_photo_ratelimit) def check_renamed(filepath): ''' @@ -1647,10 +1308,10 @@ class PDBUtilMixin: same_meta = [photo for photo in same_meta if not photo.real_path.is_file] if len(same_meta) == 1: photo = same_meta[0] - self.log.debug('Found mtime+bytesize match %s.', photo) + log.debug('Found mtime+bytesize match %s.', photo) return photo - self.log.loud('Hashing file %s to check for rename.', filepath) + log.loud('Hashing file %s to check for rename.', filepath) sha256 = spinal.hash_file( filepath, hash_class=hashlib.sha256, **hash_kwargs, @@ -1735,7 +1396,7 @@ class PDBUtilMixin: albums_by_path = {} - self.log.info('Digesting directory "%s".', directory.absolute_path) + log.info('Digesting directory "%s".', directory.absolute_path) walk_generator = spinal.walk( directory, exclude_directories=exclude_directories, @@ -1775,7 +1436,7 @@ class PDBUtilMixin: if yield_albums: yield from current_albums - @decorators.transaction + @worms.transaction def easybake(self, ebstring, author=None): ''' Easily create tags, groups, and synonyms with a string like @@ -1827,12 +1488,12 @@ class PDBUtilMixin: class PhotoDB( PDBAlbumMixin, PDBBookmarkMixin, - PDBCacheManagerMixin, + PDBGroupableMixin, PDBPhotoMixin, - PDBSQLMixin, PDBTagMixin, PDBUserMixin, PDBUtilMixin, + worms.DatabaseWithCaching, ): def __init__( self, @@ -1862,6 +1523,8 @@ class PhotoDB( Beware of modifying any data in this state. ''' super().__init__() + # Used by decorators.required_feature. + self._photodb = self ephemeral = bool(ephemeral) if data_directory is not None and ephemeral: @@ -1887,9 +1550,6 @@ class PhotoDB( if self.data_directory.exists and not self.data_directory.is_dir: raise exceptions.BadDataDirectory(self.data_directory.absolute_path) - # LOGGING - self.log = vlogging.getLogger(f'{__name__}:{self.data_directory.absolute_path}') - # DATABASE if self.ephemeral: existing_database = False @@ -1920,13 +1580,17 @@ class PhotoDB( self.config_filepath = self.data_directory.with_child(constants.DEFAULT_CONFIGNAME) self.load_config() + # WORMS + self.COLUMNS = constants.SQL_COLUMNS + self.COLUMN_INDEX = constants.SQL_INDEX + self.caches = { - 'album': cacheclass.Cache(maxlen=self.config['cache_size']['album']), - 'bookmark': cacheclass.Cache(maxlen=self.config['cache_size']['bookmark']), - 'photo': cacheclass.Cache(maxlen=self.config['cache_size']['photo']), - 'tag': cacheclass.Cache(maxlen=self.config['cache_size']['tag']), + objects.Album: cacheclass.Cache(maxlen=self.config['cache_size']['album']), + objects.Bookmark: cacheclass.Cache(maxlen=self.config['cache_size']['bookmark']), + objects.Photo: cacheclass.Cache(maxlen=self.config['cache_size']['photo']), + objects.Tag: cacheclass.Cache(maxlen=self.config['cache_size']['tag']), + objects.User: cacheclass.Cache(maxlen=self.config['cache_size']['user']), 'tag_exports': cacheclass.Cache(maxlen=100), - 'user': cacheclass.Cache(maxlen=self.config['cache_size']['user']), } def _check_version(self): @@ -1934,7 +1598,7 @@ class PhotoDB( Compare database's user_version against constants.DATABASE_VERSION, raising exceptions.DatabaseOutOfDate if not correct. ''' - existing = self.sql_execute('PRAGMA user_version').fetchone()[0] + existing = self.execute('PRAGMA user_version').fetchone()[0] if existing != constants.DATABASE_VERSION: raise exceptions.DatabaseOutOfDate( existing=existing, @@ -1943,13 +1607,13 @@ class PhotoDB( ) def _first_time_setup(self): - self.log.info('Running first-time database setup.') - self.sql_executescript(constants.DB_INIT) + log.info('Running first-time database setup.') + self.executescript(constants.DB_INIT) self.commit() def _load_pragmas(self): - self.log.debug('Reloading pragmas.') - self.sql_executescript(constants.DB_PRAGMAS) + log.debug('Reloading pragmas.') + self.executescript(constants.DB_PRAGMAS) self.commit() # Will add -> PhotoDB when forward references are supported @@ -1964,21 +1628,22 @@ class PhotoDB( starting = path while True: - if path.with_child(constants.DEFAULT_DATADIR).is_dir: + possible = path.with_child(constants.DEFAULT_DATADIR) + if possible.is_dir: break parent = path.parent if path == parent: raise exceptions.NoClosestPhotoDB(starting.absolute_path) path = parent - path = path.with_child(constants.DEFAULT_DATADIR) + path = possible photodb = cls( path, create=False, *args, **kwargs, ) - photodb.log.debug('Found closest PhotoDB at %s.', path) + log.debug('Found closest PhotoDB at %s.', path) return photodb def __del__(self): @@ -1991,11 +1656,7 @@ class PhotoDB( return f'PhotoDB(data_directory={self.data_directory})' def close(self) -> None: - # Wrapped in hasattr because if the object fails __init__, Python will - # still call __del__ and thus close(), even though the attributes - # we're trying to clean up never got set. - if hasattr(self, 'sql'): - self.sql.close() + super().close() if getattr(self, 'ephemeral', False): self.ephemeral_directory.cleanup() @@ -2011,7 +1672,7 @@ class PhotoDB( if table not in ['photos', 'tags', 'albums', 'bookmarks']: raise ValueError(f'Invalid table requested: {table}.') - last_id = self.sql_select_one('SELECT last_id FROM id_numbers WHERE tab == ?', [table]) + last_id = self.select_one('SELECT last_id FROM id_numbers WHERE tab == ?', [table]) if last_id is None: # Register new value new_id_int = 1 @@ -2028,13 +1689,13 @@ class PhotoDB( 'last_id': new_id, } if do_insert: - self.sql_insert(table='id_numbers', data=pairs) + self.insert(table='id_numbers', data=pairs) else: - self.sql_update(table='id_numbers', pairs=pairs, where_key='tab') + self.update(table='id_numbers', pairs=pairs, where_key='tab') return new_id def load_config(self) -> None: - self.log.debug('Loading config file.') + log.debug('Loading config file.') (config, needs_rewrite) = configlayers.load_file( filepath=self.config_filepath, default_config=constants.DEFAULT_CONFIGURATION, @@ -2045,6 +1706,6 @@ class PhotoDB( self.save_config() def save_config(self) -> None: - self.log.debug('Saving config file.') + log.debug('Saving config file.') with self.config_filepath.open('w', encoding='utf-8') as handle: handle.write(json.dumps(self.config, indent=4, sort_keys=True)) diff --git a/utilities/database_upgrader.py b/utilities/database_upgrader.py index e4a497b..64de8d8 100644 --- a/utilities/database_upgrader.py +++ b/utilities/database_upgrader.py @@ -28,7 +28,7 @@ class Migrator: query = 'SELECT name, sql FROM sqlite_master WHERE type == "table"' self.tables = { name: {'create': sql, 'transfer': f'INSERT INTO {name} SELECT * FROM {name}_old'} - for (name, sql) in self.photodb.sql_select(query) + for (name, sql) in self.photodb.select(query) } # The user may be adding entirely new tables derived from the data of @@ -37,7 +37,7 @@ class Migrator: self.existing_tables = set(self.tables) query = 'SELECT name, sql FROM sqlite_master WHERE type == "index" AND name NOT LIKE "sqlite_%"' - self.indices = list(self.photodb.sql_select(query)) + self.indices = list(self.photodb.select(query)) def go(self): # This loop is split in many parts, because otherwise if table A @@ -45,33 +45,33 @@ class Migrator: # be pointing to the version of B which has not been reconstructed yet, # which is about to get renamed to B_old and then A's reference will be # broken. - self.photodb.sql_execute('PRAGMA foreign_keys = OFF') - self.photodb.sql_execute('BEGIN') + self.photodb.execute('PRAGMA foreign_keys = OFF') + self.photodb.execute('BEGIN') for (name, table) in self.tables.items(): if name not in self.existing_tables: continue - self.photodb.sql_execute(f'ALTER TABLE {name} RENAME TO {name}_old') + self.photodb.execute(f'ALTER TABLE {name} RENAME TO {name}_old') for (name, table) in self.tables.items(): - self.photodb.sql_execute(table['create']) + self.photodb.execute(table['create']) for (name, table) in self.tables.items(): - self.photodb.sql_execute(table['transfer']) + self.photodb.execute(table['transfer']) for (name, query) in self.tables.items(): if name not in self.existing_tables: continue - self.photodb.sql_execute(f'DROP TABLE {name}_old') + self.photodb.execute(f'DROP TABLE {name}_old') for (name, query) in self.indices: - self.photodb.sql_execute(query) + self.photodb.execute(query) def upgrade_1_to_2(photodb): ''' In this version, a column `tagged_at` was added to the Photos table, to keep track of the last time the photo's tags were edited (added or removed). ''' - photodb.sql_executescript(''' + photodb.executescript(''' BEGIN; ALTER TABLE photos ADD COLUMN tagged_at INT; ''') @@ -82,7 +82,7 @@ def upgrade_2_to_3(photodb): with id, username, password hash, and a timestamp. Plus some indices. ''' - photodb.sql_executescript(''' + photodb.executescript(''' BEGIN; CREATE TABLE users( @@ -101,7 +101,7 @@ def upgrade_3_to_4(photodb): ''' Add an `author_id` column to Photos. ''' - photodb.sql_executescript(''' + photodb.executescript(''' BEGIN; ALTER TABLE photos ADD COLUMN author_id TEXT; @@ -113,7 +113,7 @@ def upgrade_4_to_5(photodb): ''' Add table `bookmarks` and its indices. ''' - photodb.sql_executescript(''' + photodb.executescript(''' BEGIN; CREATE TABLE bookmarks( @@ -140,15 +140,15 @@ def upgrade_5_to_6(photodb): into `album_group_rel` - Gives albums their own last_id value, starting with the current tag value. ''' - photodb.sql_execute('BEGIN') + photodb.execute('BEGIN') # 1. Start the id_numbers.albums value at the tags value so that the number # can continue to increment safely and separately, instead of starting at # zero and bumping into existing albums. - last_id = photodb.sql_select_one('SELECT last_id FROM id_numbers WHERE tab == "tags"')[0] - photodb.sql_execute('INSERT INTO id_numbers VALUES("albums", ?)', [last_id]) + last_id = photodb.select_one('SELECT last_id FROM id_numbers WHERE tab == "tags"')[0] + photodb.execute('INSERT INTO id_numbers VALUES("albums", ?)', [last_id]) # 2. Now's a good chance to rename 'index_grouprel' to 'index_taggroup'. - photodb.sql_executescript(''' + photodb.executescript(''' DROP INDEX IF EXISTS index_grouprel_parentid; DROP INDEX IF EXISTS index_grouprel_memberid; CREATE INDEX index_taggroup_parentid ON tag_group_rel(parentid); @@ -157,27 +157,27 @@ def upgrade_5_to_6(photodb): # 3. All of the album group relationships need to be moved into their # own table, out of tag_group_rel - photodb.sql_executescript(''' + photodb.executescript(''' CREATE TABLE album_group_rel(parentid TEXT, memberid TEXT); CREATE INDEX index_albumgroup_parentid ON album_group_rel(parentid); CREATE INDEX index_albumgroup_memberid ON album_group_rel(memberid); ''') - album_ids = [id for (id,) in photodb.sql_select('SELECT id FROM albums')] + album_ids = list(photodb.select_column('SELECT id FROM albums')) for album_id in album_ids: query = 'SELECT * FROM tag_group_rel WHERE parentid == ? OR memberid == ?' bindings = [album_id, album_id] - grouprels = list(photodb.sql_select(query, bindings)) + grouprels = list(photodb.select(query, bindings)) if not grouprels: continue for grouprel in grouprels: - photodb.sql_execute('INSERT INTO album_group_rel VALUES(?, ?)', grouprel) + photodb.execute('INSERT INTO album_group_rel VALUES(?, ?)', grouprel) query = 'DELETE FROM tag_group_rel WHERE parentid == ? OR memberid == ?' bindings = [album_id, album_id] - photodb.sql_execute(query, bindings) + photodb.execute(query, bindings) def upgrade_6_to_7(photodb): ''' @@ -187,12 +187,11 @@ def upgrade_6_to_7(photodb): Most of the indices were renamed. ''' - photodb.sql_execute('BEGIN') + photodb.execute('BEGIN') query = 'SELECT name FROM sqlite_master WHERE type == "index" AND name NOT LIKE "sqlite_%"' - indices = photodb.sql_select(query) - indices = [name for (name,) in indices] + indices = photodb.select_column(query) for index in indices: - photodb.sql_execute(f'DROP INDEX {index}') + photodb.execute(f'DROP INDEX {index}') m = Migrator(photodb) m.tables['album_associated_directories']['create'] = ''' @@ -226,7 +225,7 @@ def upgrade_6_to_7(photodb): m.go() - photodb.sql_executescript(''' + photodb.executescript(''' CREATE INDEX IF NOT EXISTS index_album_associated_directories_albumid on album_associated_directories(albumid); CREATE INDEX IF NOT EXISTS index_album_associated_directories_directory on @@ -260,7 +259,7 @@ def upgrade_7_to_8(photodb): ''' Give the Tags table a description field. ''' - photodb.sql_executescript(''' + photodb.executescript(''' BEGIN; ALTER TABLE tags ADD COLUMN description TEXT; ''') @@ -269,7 +268,7 @@ def upgrade_8_to_9(photodb): ''' Give the Photos table a searchhidden field. ''' - photodb.sql_executescript(''' + photodb.executescript(''' BEGIN; ALTER TABLE photos ADD COLUMN searchhidden INT; @@ -286,7 +285,7 @@ def upgrade_9_to_10(photodb): Previously, the stored path was unnecessarily high and contained the PDB's data_directory, reducing portability. ''' - photodb.sql_execute('BEGIN') + photodb.execute('BEGIN') photos = list(photodb.search(has_thumbnail=True, is_searchhidden=None, yield_albums=False)) # Since we're doing it all at once, I'm going to cheat and skip the @@ -296,7 +295,7 @@ def upgrade_9_to_10(photodb): new_thumbnail_path = photo.make_thumbnail_filepath() new_thumbnail_path = new_thumbnail_path.absolute_path new_thumbnail_path = '.' + new_thumbnail_path.replace(thumbnail_dir, '') - photodb.sql_execute( + photodb.execute( 'UPDATE photos SET thumbnail = ? WHERE id == ?', [new_thumbnail_path, photo.id] ) @@ -352,7 +351,7 @@ def upgrade_11_to_12(photodb): improve the speed of individual relation searching, important for the new intersection-based search. ''' - photodb.sql_executescript(''' + photodb.executescript(''' BEGIN; CREATE INDEX IF NOT EXISTS index_photo_tag_rel_photoid_tagid on photo_tag_rel(photoid, tagid); @@ -439,7 +438,7 @@ def upgrade_14_to_15(photodb): m.go() - photodb.sql_execute('CREATE INDEX index_photos_dev_ino ON photos(dev_ino);') + photodb.execute('CREATE INDEX index_photos_dev_ino ON photos(dev_ino);') for photo in photodb.get_photos_by_recent(): if not photo.real_path.is_file: @@ -449,7 +448,7 @@ def upgrade_14_to_15(photodb): if dev == 0 or ino == 0: continue dev_ino = f'{dev},{ino}' - photodb.sql_execute('UPDATE photos SET dev_ino = ? WHERE id == ?', [dev_ino, photo.id]) + photodb.execute('UPDATE photos SET dev_ino = ? WHERE id == ?', [dev_ino, photo.id]) def upgrade_15_to_16(photodb): ''' @@ -502,9 +501,9 @@ def upgrade_15_to_16(photodb): m.go() - for (id, filepath) in photodb.sql_select('SELECT id, filepath FROM photos'): + for (id, filepath) in photodb.select('SELECT id, filepath FROM photos'): basename = os.path.basename(filepath) - photodb.sql_execute('UPDATE photos SET basename = ? WHERE id == ?', [basename, id]) + photodb.execute('UPDATE photos SET basename = ? WHERE id == ?', [basename, id]) def upgrade_16_to_17(photodb): ''' @@ -677,7 +676,7 @@ def upgrade_all(data_directory): ''' photodb = etiquette.photodb.PhotoDB(data_directory, create=False, skip_version_check=True) - current_version = photodb.sql_execute('PRAGMA user_version').fetchone()[0] + current_version = photodb.execute('PRAGMA user_version').fetchone()[0] needed_version = etiquette.constants.DATABASE_VERSION if current_version == needed_version: @@ -690,7 +689,7 @@ def upgrade_all(data_directory): upgrade_function = eval(upgrade_function) try: - photodb.sql_execute('PRAGMA foreign_keys = ON') + photodb.execute('PRAGMA foreign_keys = ON') upgrade_function(photodb) except Exception as exc: photodb.rollback()