diff --git a/README.md b/README.md index f97a691..5fed471 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,7 @@ Here is a brief overview of the project to help you learn your way around: - Fix album size cache when photo reload metadata and generally improve that validation. - Better bookmark url validation. - Create a textbox which gives autocomplete tag names. +- In the same vein, use a dedicated endpoint with etag caching for providing the full list of tag names, so the client can check up on it and store the results in localstorage, and use for the autocomplete system. - Consider if the "did you commit too early" warning should actually be an exception. - Extension currently does not believe in the override filename. On one hand this is kind of good because if they override the name to have no extension, we can still provide a downloadable file with the correct extension by remembering it. But on the other hand it does break the illusion of override_filename. - When batch fetching objects, consider whether or not a NoSuch should be raised. Perhaps a warningbag should be used. diff --git a/etiquette/helpers.py b/etiquette/helpers.py index 26a6651..312a152 100644 --- a/etiquette/helpers.py +++ b/etiquette/helpers.py @@ -445,6 +445,7 @@ def split_easybake_string(ebstring): if not tagname: raise exceptions.EasyBakeError('No tag supplied') + tagname = tagname.strip('.') return (tagname, synonym, rename_to) def sql_listify(items): diff --git a/etiquette/jsonify.py b/etiquette/jsonify.py index 26c11f3..d1bb5ba 100644 --- a/etiquette/jsonify.py +++ b/etiquette/jsonify.py @@ -12,12 +12,8 @@ def album(a, minimal=False): } if not minimal: j['photos'] = [photo(p) for p in a.get_photos()] - parent = a.get_parent() - if parent is not None: - j['parent'] = album(parent, minimal=True) - else: - j['parent'] = None - j['sub_albums'] = [child.id for child in a.get_children()] + j['parents'] = [album(p, minimal=True) for p in a.get_parents()] + j['sub_albums'] = [album(c, minimal=True) for c in a.get_children()] return j @@ -74,7 +70,7 @@ def tag(t, include_synonyms=False, minimal=False): if not minimal: j['author'] = user_or_none(t.get_author()) j['description'] = t.description - j['qualified_name'] = t.qualified_name() + j['children'] = [tag(c, minimal=True) for c in t.get_children()] if include_synonyms: j['synonyms'] = list(t.get_synonyms()) @@ -85,6 +81,7 @@ def user(u): 'id': u.id, 'username': u.username, 'created': u.created, + 'display_name': u.display_name, } return j diff --git a/etiquette/objects.py b/etiquette/objects.py index 20f6684..6056055 100644 --- a/etiquette/objects.py +++ b/etiquette/objects.py @@ -23,14 +23,6 @@ class ObjectBase: super().__init__() self.photodb = photodb - @property - def log(self): - return self.photodb.log - - @property - def sql(self): - return self.photodb.sql - def __eq__(self, other): return ( isinstance(other, type(self)) and @@ -78,47 +70,27 @@ class GroupableMixin: def _lift_children(self): ''' - If this object is a root, all of its children become roots. - If this object is a child, its parent adopts all of its children. + If this object has parents, the parents adopt all of its children. + Otherwise the parental relationship is simply deleted. ''' - parent = self.get_parent() - if parent is None: - pairs = { - 'parentid': self.id, - } - self.photodb.sql_delete(table=self.group_table, pairs=pairs) - else: - pairs = { - 'parentid': (self.id, parent.id), - } - self.photodb.sql_update(table=self.group_table, pairs=pairs, where_key='parentid') + children = self.get_children() + if not children: + return + + self.photodb.sql_delete(table=self.group_table, pairs={'parentid': self.id}) + + parents = self.get_parents() + for parent in parents: + parent.add_children(children) @decorators.transaction def add_child(self, member, *, commit=True): - ''' - Add a child object to this group. - Child must be of the same type as the calling object. + self.assert_same_type(member) - If that object is already a member of another group, an - exceptions.GroupExists is raised. - ''' - if not isinstance(member, type(self)): - raise TypeError('Member must be of type %s' % type(self)) + if self.has_child(member): + return - self.photodb.log.debug('Adding child %s to %s' % (member, self)) - - # Groupables are only allowed to have 1 parent. - # Unlike photos which can exist in multiple albums. - parent_row = self.photodb.sql_select_one( - 'SELECT parentid FROM %s WHERE memberid == ?' % self.group_table, - [member.id] - ) - if parent_row is not None: - parent_id = parent_row[0] - if parent_id == self.id: - return - that_group = self.group_getter(id=parent_id) - raise exceptions.GroupExists(member=member, group=that_group) + self.photodb.log.debug(f'Adding child {member} to {self}.') for my_ancestor in self.walk_parents(): if my_ancestor == member: @@ -136,6 +108,21 @@ class GroupableMixin: self.photodb.log.debug('Committing - add to group') self.photodb.commit() + @decorators.transaction + def add_children(self, members, *, commit=True): + for member in members: + self.add_child(member, commit=False) + + if commit: + self.photodb.log.debug('Committing - add multiple to group') + self.photodb.commit() + + def assert_same_type(self, other): + if not isinstance(other, type(self)): + raise TypeError(f'Object must be of type {type(self)}, not {type(other)}.') + if self.photodb != other.photodb: + raise TypeError(f'Objects must belong to the same PhotoDB.') + @decorators.transaction def delete(self, *, delete_children=False, commit=True): ''' @@ -153,7 +140,7 @@ class GroupableMixin: self.photodb._cached_frozen_children = None if delete_children: for child in self.get_children(): - child.delete(delete_children=delete_children, commit=False) + child.delete(delete_children=True, commit=False) else: self._lift_children() @@ -179,46 +166,45 @@ class GroupableMixin: children = sorted(children, key=lambda x: x.id) return children - def get_parent(self): - ''' - Return the group of which this is a member, or None. - Returned object will be of the same type as calling object. - ''' - parent_row = self.photodb.sql_select_one( - 'SELECT parentid FROM %s WHERE memberid == ?' % self.group_table, - [self.id] - ) - if parent_row is None: - return None + def get_parents(self): + query = f'SELECT parentid FROM {self.group_table} WHERE memberid == ?' + parent_rows = self.photodb.sql_select(query, [self.id]) + parent_ids = [row[0] for row in parent_rows] + parents = list(self.group_getter_many(parent_ids)) + return parents - return self.group_getter(id=parent_row[0]) + def has_any_child(self): + query = f'SELECT 1 FROM {self.group_table} WHERE parentid == ? LIMIT 1' + row = self.photodb.sql_select_one(query, [self.id]) + return row is not None + + def has_any_parent(self): + query = f'SELECT 1 FROM {self.group_table} WHERE memberid == ? LIMIT 1' + row = self.photodb.sql_select_one(query, [self.id]) + return row is not None + + def has_child(self, member): + self.assert_same_type(member) + query = f'SELECT 1 FROM {self.group_table} WHERE parentid == ? AND memberid == ?' + row = self.photodb.sql_select_one(query, [self.id, member.id]) + return row is not None @decorators.transaction - def join_group(self, group, *, commit=True): - ''' - Leave the current group, then call `group.add_child(self)`. - ''' - if not isinstance(group, type(self)): - raise TypeError('Group must also be %s' % type(self)) - - if self == group: - raise ValueError('Cant join self') - - if self.get_parent() == group: + def remove_child(self, member, *, commit=True): + if not self.has_child(member): return - self.leave_group(commit=False) - group.add_child(self, commit=commit) + self.photodb.log.debug(f'Removing child {member} from {self}.') - @decorators.transaction - def leave_group(self, *, commit=True): - ''' - Leave the current group and become independent. - ''' + pairs = { + 'parentid': self.id, + 'memberid': member.id, + } + self.photodb.sql_delete(table=self.group_table, pairs=pairs) self.photodb._cached_frozen_children = None - self.photodb.sql_delete(table=self.group_table, pairs={'memberid': self.id}) + if commit: - self.photodb.log.debug('Committing - leave group') + self.photodb.log.debug('Committing - remove from group') self.photodb.commit() def walk_children(self): @@ -227,10 +213,19 @@ class GroupableMixin: yield from child.walk_children() def walk_parents(self): - parent = self.get_parent() - while parent is not None: + ''' + Yield all ancestors in no particular order. + ''' + parents = self.get_parents() + seen = set(parents) + todo = list(parents) + while len(todo) > 0: + parent = todo.pop(-1) yield parent - parent = parent.get_parent() + seen.add(parent) + more_parents = set(parent.get_parents()) + more_parents = more_parents.difference(seen) + todo.extend(more_parents) class Album(ObjectBase, GroupableMixin): @@ -247,7 +242,6 @@ class Album(ObjectBase, GroupableMixin): self.description = self.normalize_description(db_row['description']) self.author_id = self.normalize_author_id(db_row['author_id']) - self.name = 'Album %s' % self.id self.group_getter = self.photodb.get_album self.group_getter_many = self.photodb.get_albums_by_id @@ -264,7 +258,7 @@ class Album(ObjectBase, GroupableMixin): return '' if not isinstance(description, str): - raise TypeError('Description must be string, not %s' % type(description)) + raise TypeError(f'Description must be string, not {type(description)}') description = description.strip() @@ -276,7 +270,7 @@ class Album(ObjectBase, GroupableMixin): return '' if not isinstance(title, str): - raise TypeError('Title must be string, not %s' % type(title)) + raise TypeError(f'Title must be string, not {type(title)}') title = title.strip() for whitespace in string.whitespace: @@ -285,24 +279,19 @@ class Album(ObjectBase, GroupableMixin): return title def _uncache(self): - self._uncache_sums() self.photodb.caches['album'].remove(self.id) - def _uncache_sums(self): - self._sum_photos_recursive = None - self._sum_bytes_local = None - self._sum_bytes_recursive = None - parent = self.get_parent() - if parent is not None: - parent._uncache_sums() - @decorators.required_feature('album.edit') # GroupableMixin.add_child already has @transaction. def add_child(self, *args, **kwargs): result = super().add_child(*args, **kwargs) - self._uncache_sums() return result + @decorators.required_feature('album.edit') + # GroupableMixin.add_children already has @transaction. + def add_children(self, *args, **kwargs): + return super().add_children(*args, **kwargs) + @decorators.required_feature('album.edit') @decorators.transaction def add_associated_directory(self, filepath, *, commit=True): @@ -349,7 +338,6 @@ class Album(ObjectBase, GroupableMixin): return self._add_photo(photo) - self._uncache_sums() if commit: self.photodb.log.debug('Committing - add photo to album') @@ -364,7 +352,6 @@ class Album(ObjectBase, GroupableMixin): for photo in photos: self._add_photo(photo) - self._uncache_sums() if commit: self.photodb.log.debug('Committing - add photos to album') @@ -484,21 +471,6 @@ class Album(ObjectBase, GroupableMixin): ) return rel_row is not None - @decorators.required_feature('album.edit') - # GroupableMixin.join_group already has @transaction. - def join_group(self, *args, **kwargs): - result = super().join_group(*args, **kwargs) - return result - - @decorators.required_feature('album.edit') - # GroupableMixin.leave_group already has @transaction. - def leave_group(self, *args, **kwargs): - parent = self.get_parent() - if parent is not None: - parent._uncache_sums() - result = super().leave_group(*args, **kwargs) - return result - def _remove_photo(self, photo): self.photodb.log.debug('Removing photo %s from %s', photo, self) pairs = {'albumid': self.id, 'photoid': photo.id} @@ -508,7 +480,6 @@ class Album(ObjectBase, GroupableMixin): @decorators.transaction def remove_photo(self, photo, *, commit=True): self._remove_photo(photo) - self._uncache_sums() if commit: self.photodb.log.debug('Committing - remove photo from album') @@ -523,7 +494,6 @@ class Album(ObjectBase, GroupableMixin): for photo in photos: self._remove_photo(photo) - self._uncache_sums() if commit: self.photodb.log.debug('Committing - remove photos from album') @@ -534,7 +504,7 @@ class Album(ObjectBase, GroupableMixin): SELECT SUM(bytes) FROM photos WHERE photos.id IN ( SELECT photoid FROM album_photo_rel WHERE - albumid IN {qmarks} + albumid IN {albumids} ) ''' if recurse: @@ -542,20 +512,26 @@ class Album(ObjectBase, GroupableMixin): else: albumids = [self.id] - query = query.format(qmarks='(%s)' % ','.join('?' * len(albumids))) - bindings = albumids - total = self.photodb.sql_select_one(query, bindings)[0] + albumids = helpers.sql_listify(albumids) + query = query.format(albumids=albumids) + total = self.photodb.sql_select_one(query)[0] return total - def sum_photos(self): - if self._sum_photos_recursive is None: - #print(self, 'sumphotos cache miss') - total = 0 - total += sum(1 for x in self.get_photos()) - total += sum(child.sum_photos() for child in self.get_children()) - self._sum_photos_recursive = total + def sum_photos(self, recurse=True): + query = ''' + SELECT COUNT(photoid) + FROM album_photo_rel + WHERE albumid IN {albumids} + ''' + if recurse: + albumids = [child.id for child in self.walk_children()] + else: + albumids = [self.id] - return self._sum_photos_recursive + albumids = helpers.sql_listify(albumids) + query = query.format(albumids=albumids) + total = self.photodb.sql_select_one(query)[0] + return total def walk_photos(self): yield from self.get_photos() @@ -609,10 +585,14 @@ class Bookmark(ObjectBase): return url + def _uncache(self): + self.photodb.caches['bookmark'].remove(self.id) + @decorators.required_feature('bookmark.edit') @decorators.transaction def delete(self, *, commit=True): self.photodb.sql_delete(table='bookmarks', pairs={'id': self.id}) + self._uncache() if commit: self.photodb.commit() @@ -684,11 +664,6 @@ class Photo(ObjectBase): self.searchhidden = db_row['searchhidden'] - if self.duration and self.bytes is not None: - self.bitrate = (self.bytes / 128) / self.duration - else: - self.bitrate = None - self.mimetype = helpers.get_mimetype(self.real_path.basename) if self.mimetype is None: self.simple_mimetype = None @@ -748,13 +723,20 @@ class Photo(ObjectBase): self.photodb.commit() return tag + @property + def bitrate(self): + if self.duration and self.bytes is not None: + return (self.bytes / 128) / self.duration + else: + return None + @property def bytestring(self): if self.bytes is not None: return bytestring.bytestring(self.bytes) return '??? b' - @decorators.required_feature('photo.add_remove_tag') + # Photo.add_tag already has required_feature add_remove_tag # Photo.add_tag already has @transaction. def copy_tags(self, other_photo, *, commit=True): ''' @@ -773,9 +755,9 @@ class Photo(ObjectBase): Delete the Photo and its relation to any tags and albums. ''' self.photodb.log.debug('Deleting %s', self) - self.photodb.sql_delete(table='photos', pairs={'id': self.id}) self.photodb.sql_delete(table='photo_tag_rel', pairs={'photoid': self.id}) self.photodb.sql_delete(table='album_photo_rel', pairs={'photoid': self.id}) + self.photodb.sql_delete(table='photos', pairs={'id': self.id}) if delete_file: path = self.real_path.absolute_path @@ -1149,6 +1131,9 @@ class Tag(ObjectBase, GroupableMixin): if isinstance(db_row, (list, tuple)): db_row = dict(zip(constants.SQL_COLUMNS['tags'], db_row)) self.id = db_row['id'] + + # Do not pass the name through the normalizer. It may be grandfathered + # from previous character / length rules. self.name = db_row['name'] self.description = self.normalize_description(db_row['description']) self.author_id = self.normalize_author_id(db_row['author_id']) @@ -1157,12 +1142,6 @@ class Tag(ObjectBase, GroupableMixin): self.group_getter_many = self.photodb.get_tags_by_id self._cached_synonyms = None - def __eq__(self, other): - return self.name == other or ObjectBase.__eq__(self, other) - - def __hash__(self): - return hash(self.name) - def __repr__(self): return f'Tag:{self.id}:{self.name}' @@ -1185,13 +1164,14 @@ class Tag(ObjectBase, GroupableMixin): def normalize_name(name, valid_chars=None, min_length=None, max_length=None): original_name = name if valid_chars is None: - valid_chars = constants.DEFAULT_CONFIG['tag']['valid_chars'] + valid_chars = constants.DEFAULT_CONFIGURATION['tag']['valid_chars'] - name = name.lower() + name = name.lower().strip() + name = name.strip('.+') + name = name.split('+')[0].split('.')[-1] name = name.replace('-', '_') name = name.replace(' ', '_') - name = (c for c in name if c in valid_chars) - name = ''.join(name) + name = ''.join(c for c in name if c in valid_chars) if min_length is not None and len(name) < min_length: raise exceptions.TagTooShort(original_name) @@ -1209,6 +1189,11 @@ class Tag(ObjectBase, GroupableMixin): def add_child(self, *args, **kwargs): return super().add_child(*args, **kwargs) + @decorators.required_feature('tag.edit') + # GroupableMixin.add_children already has @transaction. + def add_children(self, *args, **kwargs): + return super().add_children(*args, **kwargs) + @decorators.required_feature('tag.edit') @decorators.transaction def add_synonym(self, synname, *, commit=True): @@ -1219,7 +1204,7 @@ class Tag(ObjectBase, GroupableMixin): self.photodb.assert_no_such_tag(name=synname) - self.log.debug('New synonym %s of %s', synname, self.name) + self.photodb.log.debug('New synonym %s of %s', synname, self.name) self.photodb._cached_frozen_children = None @@ -1354,49 +1339,11 @@ class Tag(ObjectBase, GroupableMixin): self._cached_synonyms = synonyms.copy() return synonyms - @decorators.required_feature('tag.edit') - # GroupableMixin.join_group already has @transaction. - def join_group(self, *args, **kwargs): - return super().join_group(*args, **kwargs) - @decorators.required_feature('tag.edit') # GroupableMixin.leave_group already has @transaction. def leave_group(self, *args, **kwargs): return super().leave_group(*args, **kwargs) - def qualified_name(self, *, max_len=None): - ''' - Return the 'group1.group2.tag' string for this tag. - - If `max_len` is not None, bring the length of the qualname down - by first stripping off ancestors, then slicing the end off of the - name if necessary. - - ('people.family.mother', max_len=25) -> 'people.family.mother' - ('people.family.mother', max_len=15) -> 'family.mother' - ('people.family.mother', max_len=10) -> 'mother' - ('people.family.mother', max_len=4) -> 'moth' - ''' - if max_len is not None: - if len(self.name) == max_len: - return self.name - if len(self.name) > max_len: - return self.name[:max_len] - - parent = self.get_parent() - if parent is None: - qualname = self.name - else: - qualname = parent.qualified_name() + '.' + self.name - - if max_len is None or len(qualname) <= max_len: - return qualname - - while len(qualname) > max_len: - qualname = qualname.split('.', 1)[1] - - return qualname - @decorators.required_feature('tag.edit') @decorators.transaction def remove_synonym(self, synname, *, commit=True): @@ -1477,6 +1424,7 @@ class User(ObjectBase): self.username = db_row['username'] self.created = db_row['created'] self.password_hash = db_row['password'] + # Do not enforce maxlen here, they may be grandfathered in. self._display_name = self.normalize_display_name(db_row['display_name']) def __repr__(self): diff --git a/etiquette/photodb.py b/etiquette/photodb.py index 3056aee..57ae403 100644 --- a/etiquette/photodb.py +++ b/etiquette/photodb.py @@ -292,17 +292,23 @@ class PDBPhotoMixin: if photo.real_path.exists: continue photo.delete(commit=False) + if commit: self.log.debug('Committing - purge deleted photos') self.commit() @decorators.transaction def purge_empty_albums(self, *, commit=True): - albums = self.get_albums() - for album in albums: + to_check = list(self.get_albums()) + + while to_check: + album = to_check.pop() if album.get_children() or album.get_photos(): continue + # This may have been the last child of an otherwise empty parent. + to_check.extend(album.get_parents()) album.delete(commit=False) + if commit: self.log.debug('Committing - purge empty albums') self.commit() @@ -464,25 +470,21 @@ class PDBPhotoMixin: if extension is not None and extension_not is not None: extension = extension.difference(extension_not) - mmf_expression_noconflict = searchhelpers.check_mmf_expression_exclusive( + tags_fixed = searchhelpers.normalize_mmf_vs_expression_conflict( tag_musts, tag_mays, tag_forbids, tag_expression, - warning_bag + warning_bag, ) - if not mmf_expression_noconflict: - tag_musts = None - tag_mays = None - tag_forbids = None - tag_expression = None + (tag_musts, tag_mays, tag_forbids, tag_expression) = tags_fixed + if tag_expression: frozen_children = self.get_cached_frozen_children() tag_expression_tree = searchhelpers.tag_expression_tree_builder( tag_expression=tag_expression, photodb=self, - frozen_children=frozen_children, warning_bag=warning_bag, ) if tag_expression_tree is None: @@ -490,7 +492,6 @@ class PDBPhotoMixin: tag_expression = None else: giveback_tag_expression = str(tag_expression_tree) - print(giveback_tag_expression) tag_match_function = searchhelpers.tag_expression_matcher_builder(frozen_children) else: giveback_tag_expression = None @@ -565,16 +566,16 @@ class PDBPhotoMixin: if '*' in extension: wheres.append('extension != ""') else: - binders = ', '.join('?' * len(extension)) - wheres.append('extension IN (%s)' % binders) + qmarks = ', '.join('?' * len(extension)) + wheres.append('extension IN (%s)' % qmarks) bindings.extend(extension) if extension_not: if '*' in extension_not: wheres.append('extension == ""') else: - binders = ', '.join('?' * len(extension_not)) - wheres.append('extension NOT IN (%s)' % binders) + qmarks = ', '.join('?' * len(extension_not)) + wheres.append('extension NOT IN (%s)' % qmarks) bindings.extend(extension_not) if mimetype: @@ -781,6 +782,16 @@ class PDBTagMixin: else: raise exceptions.TagExists(existing_tag) + def get_all_tag_names(self): + ''' + Return a list containing the names of all tags as strings. + Useful for when you don't want the overhead of actual Tag objects. + ''' + query = 'SELECT name FROM tags' + rows = self.sql_select(query) + names = [row[0] for row in rows] + return names + def get_root_tags(self): ''' Yield Tags that have no parent. @@ -808,9 +819,6 @@ class PDBTagMixin: return tagname tagname = tagname.tagname - tagname = tagname.strip('.+') - tagname = tagname.split('.')[-1].split('+')[0] - try: tagname = self.normalize_tagname(tagname) except (exceptions.TagTooShort, exceptions.TagTooLong): @@ -851,6 +859,7 @@ class PDBTagMixin: ''' tagname = self.normalize_tagname(tagname) self.assert_no_such_tag(name=tagname) + description = objects.Tag.normalize_description(description) self.log.debug('New Tag: %s', tagname) @@ -1002,7 +1011,7 @@ class PDBUserMixin: @decorators.required_feature('user.new') @decorators.transaction - def register_user(self, username, password, *, commit=True): + def register_user(self, username, password, *, display_name=None, commit=True): self.assert_valid_username(username) if not isinstance(password, bytes): @@ -1011,6 +1020,11 @@ class PDBUserMixin: self.assert_valid_password(password) self.assert_no_such_user(username=username) + display_name = objects.User.normalize_display_name( + display_name, + max_length=self.config['user']['max_display_name_length'], + ) + self.log.debug('New User: %s', username) user_id = self.generate_user_id() @@ -1022,6 +1036,7 @@ class PDBUserMixin: 'username': username, 'password': hashed_password, 'created': created, + 'display_name': display_name, } self.sql_insert(table='users', data=data) @@ -1079,8 +1094,14 @@ class PDBUtilMixin: return new_photo_kwargs def _normalize_new_photo_ratelimit(new_photo_ratelimit): - if isinstance(new_photo_ratelimit, (int, float)): + if isinstance(new_photo_ratelimit, ratelimiter.Ratelimiter): + pass + elif new_photo_ratelimit is None: + pass + elif isinstance(new_photo_ratelimit, (int, float)): new_photo_ratelimit = ratelimiter.Ratelimiter(allowance=1, period=new_photo_ratelimit) + else: + raise TypeError(new_photo_ratelimit) return new_photo_ratelimit def create_or_fetch_photos(filepaths, new_photo_kwargs): @@ -1117,7 +1138,11 @@ class PDBUtilMixin: return current_album def orphan_join_parent_album(albums_by_path, current_album, current_directory): - if current_album.get_parent() is None: + ''' + If the current album is an orphan, let's check if there exists an + album for the parent directory. If so, add the current album to it. + ''' + if not current_album.has_any_parent(): parent = albums_by_path.get(current_directory.parent.absolute_path, None) if parent is not None: parent.add_child(current_album, commit=False) @@ -1192,7 +1217,7 @@ class PDBUtilMixin: tags = [create_or_get(t) for t in tag_parts] for (higher, lower) in zip(tags, tags[1:]): try: - lower.join_group(higher, commit=False) + higher.add_child(lower, commit=False) note = ('join_group', f'{higher.name}.{lower.name}') output_notes.append(note) except exceptions.GroupExists: @@ -1286,7 +1311,6 @@ class PhotoDB( # OTHER self._cached_frozen_children = None - self._cached_qualname_map = None self.caches = { 'album': cacheclass.Cache(maxlen=self.config['cache_size']['album']), @@ -1297,6 +1321,10 @@ class PhotoDB( } def _check_version(self): + ''' + Compare database's user_version against constants.DATABASE_VERSION, + raising exceptions.DatabaseOutOfDate if not correct. + ''' existing_version = self.sql_execute('PRAGMA user_version').fetchone()[0] if existing_version != constants.DATABASE_VERSION: exc = exceptions.DatabaseOutOfDate( @@ -1327,7 +1355,6 @@ class PhotoDB( def _uncache(self): self._cached_frozen_children = None - self._cached_qualname_map = None def close(self): # Wrapped in hasattr because if the object fails __init__, Python will @@ -1397,12 +1424,10 @@ class PhotoDB( thing_cache[thing_id] = thing return thing - def get_cached_qualname_map(self): - if self._cached_qualname_map is None: - self._cached_qualname_map = tag_export.qualified_names(self.get_tags()) - return self._cached_qualname_map - def get_root_things(self, thing_type): + ''' + For Groupable types, yield things which have no parent. + ''' thing_map = _THING_CLASSES[thing_type] thing_class = thing_map['class'] @@ -1445,6 +1470,9 @@ class PhotoDB( return thing def get_things(self, thing_type): + ''' + Yield things, unfiltered, in whatever order they appear in the database. + ''' thing_map = _THING_CLASSES[thing_type] query = 'SELECT * FROM %s' % thing_map['table'] diff --git a/etiquette/searchhelpers.py b/etiquette/searchhelpers.py index 7dafd92..4354f1a 100644 --- a/etiquette/searchhelpers.py +++ b/etiquette/searchhelpers.py @@ -12,23 +12,6 @@ from . import objects from voussoirkit import expressionmatch -def check_mmf_expression_exclusive( - tag_musts, - tag_mays, - tag_forbids, - tag_expression, - warning_bag=None - ): - if (tag_musts or tag_mays or tag_forbids) and tag_expression: - exc = exceptions.NotExclusive(['tag_musts+mays+forbids', 'tag_expression']) - if warning_bag: - warning_bag.add(exc.error_message) - else: - raise exc - - return False - return True - def expand_mmf(tag_musts, tag_mays, tag_forbids): def _set(x): if x is None: @@ -245,6 +228,33 @@ def normalize_mimetype(mimetype, warning_bag=None): ''' return normalize_extension(mimetype, warning_bag) +def normalize_mmf_vs_expression_conflict( + tag_musts, + tag_mays, + tag_forbids, + tag_expression, + warning_bag=None, + ): + ''' + The user cannot provide both mmf sets and tag expression at the same time. + If both are provided, nullify everything. + ''' + if (tag_musts or tag_mays or tag_forbids) and tag_expression: + exc = exceptions.NotExclusive(['tag_musts+mays+forbids', 'tag_expression']) + if warning_bag: + warning_bag.add(exc.error_message) + else: + raise exc + conflict = True + conflict = False + + if conflict: + tag_musts = None + tag_mays = None + tag_forbids = None + tag_expression = None + return (tag_musts, tag_mays, tag_forbids, tag_expression) + def normalize_offset(offset, warning_bag=None): ''' Either: @@ -424,7 +434,6 @@ def normalize_tagset(photodb, tags, warning_bag=None): def tag_expression_tree_builder( tag_expression, photodb, - frozen_children, warning_bag=None ): if not tag_expression: @@ -439,9 +448,9 @@ def tag_expression_tree_builder( for node in expression_tree.walk_leaves(): try: - node.token = photodb.normalize_tagname(node.token) - except (exceptions.TagTooShort, exceptions.TagTooLong) as exc: - if warning_bag is not None: + node.token = photodb.get_tag(name=node.token).name + except (exceptions.NoSuchTag) as exc: + if warning_bag: warning_bag.add(exc.error_message) node.token = None else: @@ -450,14 +459,6 @@ def tag_expression_tree_builder( if node.token is None: continue - if node.token not in frozen_children: - exc = exceptions.NoSuchTag(node.token) - if warning_bag is not None: - warning_bag.add(exc.error_message) - node.token = None - else: - raise exc - expression_tree.prune() if expression_tree.token is None: return None diff --git a/etiquette/tag_export.py b/etiquette/tag_export.py index dcdd0ee..8c1359f 100644 --- a/etiquette/tag_export.py +++ b/etiquette/tag_export.py @@ -3,7 +3,7 @@ This file provides a variety of functions for exporting a PDB's tags into other formats. Strings, dicts, etc. ''' -def easybake(tags): +def easybake(tags, include_synonyms=True, with_objects=False): ''' A string where every line is the qualified name of a tag or its synonyms. @@ -13,12 +13,33 @@ def easybake(tags): people.family.mother+mom ''' lines = [] - tags = list(tags) for tag in tags: - qualname = tag.qualified_name() - lines.append(qualname) - lines.extend(qualname + '+' + syn for syn in tag.get_synonyms()) - return '\n'.join(lines) + if with_objects: + my_line = (tag.name, tag) + else: + my_line = tag.name + lines.append(my_line) + + if include_synonyms: + syn_lines = tag.get_synonyms() + syn_lines = [f'{tag.name}+{syn}' for syn in syn_lines] + if with_objects: + syn_lines = [(line, tag) for line in syn_lines] + lines.extend(syn_lines) + + child_lines = easybake( + tag.get_children(), + include_synonyms=include_synonyms, + with_objects=with_objects, + ) + if with_objects: + child_lines = [(f'{tag.name}.{line[0]}', line[1]) for line in child_lines] + else: + child_lines = [f'{tag.name}.{line}' for line in child_lines] + lines.extend(child_lines) + + lines.sort() + return lines def flat_dict(tags): ''' @@ -65,26 +86,6 @@ def nested_dict(tags): return result -def qualified_names(tags): - ''' - A dictionary where keys are string names, values are qualified names. - Synonyms included. - - { - 'people': 'people', - 'family': 'people.family', - 'mother': 'people.family.mother', - 'mom': 'people.family.mother', - } - ''' - results = {} - for tag in tags: - qualname = tag.qualified_name() - results[tag.name] = qualname - for synonym in tag.get_synonyms(): - results[synonym] = qualname - return results - def stdout(tags, depth=0): for tag in tags: children = tag.get_children() @@ -99,5 +100,5 @@ def stdout(tags, depth=0): stdout(children, depth=depth+1) - if tag.get_parent() is None: + if not tag.has_any_parent(): print() diff --git a/frontends/etiquette_flask/etiquette_flask/common.py b/frontends/etiquette_flask/etiquette_flask/common.py index 57de776..8d2123f 100644 --- a/frontends/etiquette_flask/etiquette_flask/common.py +++ b/frontends/etiquette_flask/etiquette_flask/common.py @@ -34,7 +34,9 @@ site.jinja_env.trim_blocks = True site.jinja_env.lstrip_blocks = True site.jinja_env.filters['bytestring'] = jinja_filters.bytestring site.jinja_env.filters['file_link'] = jinja_filters.file_link -site.jinja_env.filters['sort_by_qualname'] = jinja_filters.sort_by_qualname +site.jinja_env.filters['sort_tags'] = jinja_filters.sort_tags +site.jinja_env.filters['timestamp_to_naturaldate'] = jinja_filters.timestamp_to_naturaldate +site.jinja_env.filters['timestamp_to_8601'] = jinja_filters.timestamp_to_8601 site.debug = True P = etiquette.photodb.PhotoDB() @@ -93,6 +95,10 @@ def P_photos(photo_ids): def P_tag(tagname): return P.get_tag(name=tagname) +@P_wrapper +def P_tag_id(tag_id): + return P.get_tag(id=tag_id) + @P_wrapper def P_user(username): return P.get_user(username=username) diff --git a/frontends/etiquette_flask/etiquette_flask/endpoints/photo_endpoints.py b/frontends/etiquette_flask/etiquette_flask/endpoints/photo_endpoints.py index 8a1edd1..5fece5c 100644 --- a/frontends/etiquette_flask/etiquette_flask/endpoints/photo_endpoints.py +++ b/frontends/etiquette_flask/etiquette_flask/endpoints/photo_endpoints.py @@ -312,7 +312,7 @@ def get_search_core(): author_helper = lambda users: ', '.join(user.username for user in users) if users else None search_kwargs['author'] = author_helper(search_kwargs['author']) - tagname_helper = lambda tags: [tag.qualified_name() for tag in tags] if tags else None + tagname_helper = lambda tags: [tag.name for tag in tags] if tags else None search_kwargs['tag_musts'] = tagname_helper(search_kwargs['tag_musts']) search_kwargs['tag_mays'] = tagname_helper(search_kwargs['tag_mays']) search_kwargs['tag_forbids'] = tagname_helper(search_kwargs['tag_forbids']) @@ -331,7 +331,7 @@ def get_search_core(): for photo in photos: for tag in photo.get_tags(): total_tags.add(tag) - total_tags = sorted(total_tags, key=lambda t: t.qualified_name()) + total_tags = sorted(total_tags, key=lambda t: t.name) # PREV-NEXT PAGE URLS offset = search_kwargs['offset'] or 0 @@ -371,14 +371,14 @@ def get_search_core(): def get_search_html(): search_results = get_search_core() search_kwargs = search_results['search_kwargs'] - qualname_map = common.P.get_cached_qualname_map() + all_tags = common.P.get_all_tag_names() session = session_manager.get(request) response = flask.render_template( 'search.html', + all_tags=json.dumps(all_tags), next_page_url=search_results['next_page_url'], prev_page_url=search_results['prev_page_url'], photos=search_results['photos'], - qualname_map=json.dumps(qualname_map), search_kwargs=search_kwargs, session=session, total_tags=search_results['total_tags'], diff --git a/frontends/etiquette_flask/etiquette_flask/endpoints/tag_endpoints.py b/frontends/etiquette_flask/etiquette_flask/endpoints/tag_endpoints.py index ac4af5a..921289f 100644 --- a/frontends/etiquette_flask/etiquette_flask/endpoints/tag_endpoints.py +++ b/frontends/etiquette_flask/etiquette_flask/endpoints/tag_endpoints.py @@ -17,6 +17,18 @@ session_manager = common.session_manager def get_tags_specific_redirect(specific_tag): return flask.redirect(request.url.replace('/tags/', '/tag/')) +@site.route('/tagid/') +@site.route('/tagid/.json') +def get_tag_id_redirect(tag_id): + if request.url.endswith('.json'): + tag = common.P_tag_id(tag_id, response_type='json') + else: + tag = common.P_tag_id(tag_id, response_type='html') + url_from = '/tagid/' + tag_id + url_to = '/tag/' + tag.name + url = request.url.replace(url_from, url_to) + return flask.redirect(url) + # Tag metadata operations ########################################################################## @site.route('/tag//edit', methods=['POST']) @@ -36,15 +48,6 @@ def post_tag_edit(specific_tag): # Tag listings ##################################################################################### -def get_tags_core(specific_tag=None): - if specific_tag is None: - tags = common.P.get_tags() - else: - tags = specific_tag.walk_children() - tags = list(tags) - tags.sort(key=lambda x: x.qualified_name()) - return tags - @site.route('/tag/') @site.route('/tags') @session_manager.give_token @@ -57,10 +60,17 @@ def get_tags_html(specific_tag_name=None): new_url = request.url.replace('/tag/' + specific_tag_name, '/tag/' + specific_tag.name) response = flask.redirect(new_url) return response - tags = get_tags_core(specific_tag) + session = session_manager.get(request) include_synonyms = request.args.get('synonyms') include_synonyms = include_synonyms is None or etiquette.helpers.truthystring(include_synonyms) + + if specific_tag is None: + tags = common.P.get_root_tags() + else: + tags = [specific_tag] + tags = etiquette.tag_export.easybake(tags, include_synonyms=False, with_objects=True) + response = flask.render_template( 'tags.html', include_synonyms=include_synonyms, @@ -81,9 +91,14 @@ def get_tags_json(specific_tag_name=None): if specific_tag.name != specific_tag_name: new_url = request.url.replace('/tag/' + specific_tag_name, '/tag/' + specific_tag.name) return flask.redirect(new_url) - tags = get_tags_core(specific_tag=specific_tag) + include_synonyms = request.args.get('synonyms') include_synonyms = include_synonyms is None or etiquette.helpers.truthystring(include_synonyms) + if specific_tag is None: + tags = list(common.P.get_tags()) + else: + tags = list(specific_tag.walk_children()) + tags = [etiquette.jsonify.tag(tag, include_synonyms=include_synonyms) for tag in tags] return jsonify.make_json_response(tags) @@ -133,9 +148,15 @@ def post_tag_delete(): Delete a tag. ''' tagname = request.form['tagname'] - tagname = tagname.split('.')[-1].split('+')[0] - tag = common.P.get_tag(name=tagname) - - tag.delete() - response = {'action': 'delete_tag', 'tagname': tag.name} + tagname = tagname.split('+')[0] + if '.' in tagname: + (parentname, tagname) = tagname.rsplit('.', 1) + parent = common.P.get_tag(name=parentname) + tag = common.P.get_tag(name=tagname) + parent.remove_child(tag) + response = {'action': 'unlink_tag', 'tagname': f'{parent.name}.{tag.name}'} + else: + tag = common.P.get_tag(name=tagname) + tag.delete() + response = {'action': 'delete_tag', 'tagname': tag.name} return jsonify.make_json_response(response) diff --git a/frontends/etiquette_flask/etiquette_flask/jinja_filters.py b/frontends/etiquette_flask/etiquette_flask/jinja_filters.py index 15d0465..f5e85ad 100644 --- a/frontends/etiquette_flask/etiquette_flask/jinja_filters.py +++ b/frontends/etiquette_flask/etiquette_flask/jinja_filters.py @@ -1,8 +1,11 @@ -import etiquette +import datetime import jinja2.filters +import etiquette + import voussoirkit.bytestring + def bytestring(x): try: return voussoirkit.bytestring.bytestring(x) @@ -15,6 +18,16 @@ def file_link(photo, short=False): basename = jinja2.filters.do_urlencode(photo.basename) return f'/file/{photo.id}/{basename}' -def sort_by_qualname(tags): - tags = sorted(tags, key=lambda x: x.qualified_name()) +def sort_tags(tags): + tags = sorted(tags, key=lambda x: x.name) return tags + +def timestamp_to_8601(timestamp): + return datetime.datetime.utcfromtimestamp(timestamp).isoformat(' ') + ' UTC' + +def timestamp_to_string(timestamp, format): + date = datetime.datetime.utcfromtimestamp(timestamp) + return date.strftime(format) + +def timestamp_to_naturaldate(timestamp): + return timestamp_to_string(timestamp, '%B %d, %Y') diff --git a/frontends/etiquette_flask/etiquette_flask_launch.py b/frontends/etiquette_flask/etiquette_flask_launch.py index 6f25d08..69c337b 100644 --- a/frontends/etiquette_flask/etiquette_flask_launch.py +++ b/frontends/etiquette_flask/etiquette_flask_launch.py @@ -13,7 +13,6 @@ handler.setFormatter(logging.Formatter(log_format, style='{')) logging.getLogger().addHandler(handler) import gevent.pywsgi -import gevent.wsgi import argparse import sys diff --git a/frontends/etiquette_flask/templates/album.html b/frontends/etiquette_flask/templates/album.html index d146b8c..8c80a2b 100644 --- a/frontends/etiquette_flask/templates/album.html +++ b/frontends/etiquette_flask/templates/album.html @@ -63,9 +63,11 @@ p
    {% set viewparam = "?view=list" if view == "list" else "" %} - {% set parent = album.get_parent() %} - {% if parent %} + {% set parents = album.get_parents() %} + {% if parents %} + {% for parent in parents %}
  • {{parent.display_name}}
    • + {% endfor %} {% else %}
  • Albums
    • {% endif %} @@ -155,13 +157,13 @@ function unpaste_photo_clipboard() } var paste_photo_clipboard_button = document.createElement("button"); paste_photo_clipboard_button.classList.add("green_button"); -paste_photo_clipboard_button.innerText = "Add to album"; +paste_photo_clipboard_button.innerText = "Add to this album"; paste_photo_clipboard_button.onclick = paste_photo_clipboard; document.getElementById("clipboard_tray_toolbox").appendChild(paste_photo_clipboard_button); var unpaste_photo_clipboard_button = document.createElement("button"); unpaste_photo_clipboard_button.classList.add("red_button"); -unpaste_photo_clipboard_button.innerText = "Remove from album"; +unpaste_photo_clipboard_button.innerText = "Remove from this album"; unpaste_photo_clipboard_button.onclick = unpaste_photo_clipboard; document.getElementById("clipboard_tray_toolbox").appendChild(unpaste_photo_clipboard_button); diff --git a/frontends/etiquette_flask/templates/clipboard.html b/frontends/etiquette_flask/templates/clipboard.html index c74e4d4..7cb7c92 100644 --- a/frontends/etiquette_flask/templates/clipboard.html +++ b/frontends/etiquette_flask/templates/clipboard.html @@ -311,19 +311,29 @@ function searchhidden_callback(response) } create_message_bubble(message_area, message_positivity, message_text, 8000); } -function submit_set_searchhidden() +function submit_set_searchhidden(callback) { + if (photo_clipboard.size == 0) + {return;} + var url = "/batch/photos/set_searchhidden"; var data = new FormData(); - data.append("photo_ids", Array.from(photo_clipboard).join(",")); - post(url, data, searchhidden_callback); + var photo_ids = Array.from(photo_clipboard).join(","); + + data.append("photo_ids", photo_ids); + post(url, data, callback); } -function submit_unset_searchhidden() +function submit_unset_searchhidden(callback) { + if (photo_clipboard.size == 0) + {return;} + var url = "/batch/photos/unset_searchhidden"; var data = new FormData(); - data.append("photo_ids", Array.from(photo_clipboard).join(",")); - post(url, data, searchhidden_callback); + var photo_ids = Array.from(photo_clipboard).join(","); + + data.append("photo_ids", photo_ids); + post(url, data, callback); } diff --git a/frontends/etiquette_flask/templates/photo.html b/frontends/etiquette_flask/templates/photo.html index 09d8b25..47e44c2 100644 --- a/frontends/etiquette_flask/templates/photo.html +++ b/frontends/etiquette_flask/templates/photo.html @@ -8,6 +8,8 @@ + +