diff --git a/etiquette/objects.py b/etiquette/objects.py index 8a1ed75..d85de8f 100644 --- a/etiquette/objects.py +++ b/etiquette/objects.py @@ -11,10 +11,13 @@ import os import PIL.Image import re import send2trash +import time import traceback import typing from voussoirkit import bytestring +from voussoirkit import dotdict +from voussoirkit import expressionmatch from voussoirkit import gentools from voussoirkit import hms from voussoirkit import pathclass @@ -32,6 +35,7 @@ from . import constants from . import decorators from . import exceptions from . import helpers +from . import searchhelpers BAIL = sentinel.Sentinel('BAIL') @@ -1521,6 +1525,431 @@ class Photo(ObjectBase): self._tagged_at_dt = helpers.utcfromtimestamp(self.tagged_at_unix) return self._tagged_at_dt +class Search: + ''' + FILE METADATA + ============= + area, aspectratio, width, height, bytes, duration, bitrate: + A dotdot_range string representing min and max. Or just a number + for lower bound. + + extension: + A string or list of strings of acceptable file extensions. + + extension_not: + A string or list of strings of unacceptable file extensions. + Including '*' will forbid all extensions, thus returning only + extensionless files. + + filename: + A string or list of strings in the form of an expression. + Match is CASE-INSENSITIVE. + Examples: + '.pdf AND (programming OR "survival guide")' + '.pdf programming python' (implicitly AND each term) + + sha256: + A string or list of strings of exact SHA256 hashes to match. + + within_directory: + A string or list of strings or pathclass Paths of directories. + Photos MUST have a `filepath` that is a child of one of these + directories. + + OBJECT METADATA + =============== + author: + A list of User objects or usernames, or a string of comma-separated + usernames. + + created: + A dotdot_range string respresenting min and max. Or just a number + for lower bound. + + has_albums: + If True, require that the Photo belongs to >=1 album. + If False, require that the Photo belongs to no albums. + If None, either is okay. + + has_tags: + If True, require that the Photo has >=1 tag. + If False, require that the Photo has no tags. + If None, any amount is okay. + + has_thumbnail: + Require a thumbnail? + If None, anything is okay. + + is_searchhidden: + If True, find *only* searchhidden photos. + If False, find *only* nonhidden photos. + If None, either is okay. + + mimetype: + A string or list of strings of acceptable mimetypes. + 'image', 'video', ... + Note we are only interested in the simple "video", "audio" etc. + For exact mimetypes you might as well use an extension search. + + TAGS + ==== + tag_musts: + A list of tag names or Tag objects. + Photos MUST have ALL tags in this list. + + tag_mays: + A list of tag names or Tag objects. + Photos MUST have AT LEAST ONE tag in this list. + + tag_forbids: + A list of tag names or Tag objects. + Photos MUST NOT have ANY tag in the list. + + tag_expression: + A string or list of strings in the form of an expression. + Can NOT be used with the must, may, forbid style search. + Examples: + 'family AND (animals OR vacation)' + 'family vacation outdoors' (implicitly AND each term) + + QUERY OPTIONS + ============= + limit: + The maximum number of *successful* results to yield. + + offset: + How many *successful* results to skip before we start yielding. + + orderby: + A list of strings like ['aspectratio DESC', 'created ASC'] to sort + and subsort the results. + Descending is assumed if not provided. + + yield_albums: + If True, albums which contain photos matching the search + will be yielded. + + yield_photos: + If True, photos matching the search will be yielded. + ''' + def __init__(self, photodb, kwargs, *, raise_errors=True): + self.photodb = photodb + self.created = timetools.now() + self.raise_errors = raise_errors + if isinstance(kwargs, dict): + kwargs = dotdict.DotDict(kwargs, default=None) + self.kwargs = kwargs + self.generator_started = False + self.generator_exhausted = False + self.more_after_limit = None + self.start_time = None + self.end_time = None + self.start_commit_id = None + self.warning_bag = WarningBag() + self.results = self._generator() + self.results_received = 0 + + def atomify(self): + raise NotImplementedError + + def jsonify(self): + # The search has converted many arguments into sets or other types. + # Convert them back into something that will display nicely on the search form. + kwargs = self.kwargs._to_dict() + join_helper = lambda x: ', '.join(x) if x else None + kwargs['extension'] = join_helper(kwargs['extension']) + kwargs['extension_not'] = join_helper(kwargs['extension_not']) + kwargs['mimetype'] = join_helper(kwargs['mimetype']) + kwargs['sha256'] = join_helper(kwargs['sha256']) + + author_helper = lambda users: ', '.join(user.username for user in users) if users else None + kwargs['author'] = author_helper(kwargs['author']) + + tagname_helper = lambda tags: [tag.name for tag in tags] if tags else None + kwargs['tag_musts'] = tagname_helper(kwargs['tag_musts']) + kwargs['tag_mays'] = tagname_helper(kwargs['tag_mays']) + kwargs['tag_forbids'] = tagname_helper(kwargs['tag_forbids']) + + results = [ + result.jsonify(include_albums=False) + if isinstance(result, Photo) else + result.jsonify(minimal=True) + for result in self.results + ] + + j = { + 'kwargs': kwargs, + 'results': results, + 'more_after_limit': self.more_after_limit, + } + return j + + def _generator(self): + self.start_time = time.perf_counter() + self.generator_started = True + self.start_commit_id = self.photodb.last_commit_id + + kwargs = self.kwargs + + maximums = {} + minimums = {} + searchhelpers.minmax('area', kwargs.area, minimums, maximums, warning_bag=self.warning_bag) + searchhelpers.minmax('created', kwargs.created, minimums, maximums, warning_bag=self.warning_bag) + searchhelpers.minmax('width', kwargs.width, minimums, maximums, warning_bag=self.warning_bag) + searchhelpers.minmax('height', kwargs.height, minimums, maximums, warning_bag=self.warning_bag) + searchhelpers.minmax('aspectratio', kwargs.aspectratio, minimums, maximums, warning_bag=self.warning_bag) + searchhelpers.minmax('bytes', kwargs.bytes, minimums, maximums, warning_bag=self.warning_bag) + searchhelpers.minmax('duration', kwargs.duration, minimums, maximums, warning_bag=self.warning_bag) + searchhelpers.minmax('bitrate', kwargs.bitrate, minimums, maximums, warning_bag=self.warning_bag) + + kwargs.author = searchhelpers.normalize_author(kwargs.author, photodb=self.photodb, warning_bag=self.warning_bag) + kwargs.extension = searchhelpers.normalize_extension(kwargs.extension) + kwargs.extension_not = searchhelpers.normalize_extension(kwargs.extension_not) + kwargs.filename = searchhelpers.normalize_filename(kwargs.filename) + kwargs.has_albums = searchhelpers.normalize_has_tags(kwargs.has_albums) + kwargs.has_tags = searchhelpers.normalize_has_tags(kwargs.has_tags) + kwargs.has_thumbnail = searchhelpers.normalize_has_thumbnail(kwargs.has_thumbnail) + kwargs.is_searchhidden = searchhelpers.normalize_is_searchhidden(kwargs.is_searchhidden) + kwargs.sha256 = searchhelpers.normalize_sha256(kwargs.sha256) + kwargs.mimetype = searchhelpers.normalize_extension(kwargs.mimetype) + kwargs.sha256 = searchhelpers.normalize_extension(kwargs.sha256) + kwargs.within_directory = searchhelpers.normalize_within_directory(kwargs.within_directory, warning_bag=self.warning_bag) + kwargs.yield_albums = searchhelpers.normalize_yield_albums(kwargs.yield_albums) + kwargs.yield_photos = searchhelpers.normalize_yield_photos(kwargs.yield_photos) + + if kwargs.has_tags is False: + if (kwargs.tag_musts or kwargs.tag_mays or kwargs.tag_forbids or kwargs.tag_expression): + self.warning_bag.add("has_tags=False so all tag requests are ignored.") + kwargs.tag_musts = None + kwargs.tag_mays = None + kwargs.tag_forbids = None + kwargs.tag_expression = None + else: + kwargs.tag_musts = searchhelpers.normalize_tagset(self.photodb, kwargs.tag_musts, warning_bag=self.warning_bag) + kwargs.tag_mays = searchhelpers.normalize_tagset(self.photodb, kwargs.tag_mays, warning_bag=self.warning_bag) + kwargs.tag_forbids = searchhelpers.normalize_tagset(self.photodb, kwargs.tag_forbids, warning_bag=self.warning_bag) + kwargs.tag_expression = searchhelpers.normalize_tag_expression(kwargs.tag_expression) + + if kwargs.extension is not None and kwargs.extension_not is not None: + kwargs.extension = kwargs.extension.difference(kwargs.extension_not) + + tags_fixed = searchhelpers.normalize_mmf_vs_expression_conflict( + kwargs.tag_musts, + kwargs.tag_mays, + kwargs.tag_forbids, + kwargs.tag_expression, + self.warning_bag, + ) + (kwargs.tag_musts, kwargs.tag_mays, kwargs.tag_forbids, kwargs.tag_expression) = tags_fixed + + if kwargs.tag_expression: + tag_expression_tree = searchhelpers.tag_expression_tree_builder( + tag_expression=kwargs.tag_expression, + photodb=self.photodb, + warning_bag=self.warning_bag, + ) + if tag_expression_tree is None: + kwargs.tag_expression = None + kwargs.tag_expression = None + else: + kwargs.tag_expression = str(tag_expression_tree) + frozen_children = self.photodb.get_cached_tag_export('flat_dict', tags=self.get_root_tags()) + tag_match_function = searchhelpers.tag_expression_matcher_builder(frozen_children) + else: + tag_expression_tree = None + kwargs.tag_expression = None + + if kwargs.has_tags is True and (kwargs.tag_musts or kwargs.tag_mays): + # has_tags check is redundant then, so disable it. + kwargs.has_tags = None + + kwargs.limit = searchhelpers.normalize_limit(kwargs.limit, warning_bag=self.warning_bag) + kwargs.offset = searchhelpers.normalize_offset(kwargs.offset, warning_bag=self.warning_bag) + kwargs.orderby = searchhelpers.normalize_orderby(kwargs.orderby, warning_bag=self.warning_bag) + + if kwargs.filename: + try: + filename_tree = expressionmatch.ExpressionTree.parse(kwargs.filename) + filename_tree.map(lambda x: x.lower()) + except expressionmatch.NoTokens: + filename_tree = None + else: + filename_tree = None + + if kwargs.orderby: + orderby = [(expanded, direction) for (friendly, expanded, direction) in kwargs.orderby] + kwargs.orderby = [ + f'{friendly}-{direction}' + for (friendly, expanded, direction) in kwargs.orderby + ] + else: + orderby = [('created', 'desc')] + kwargs.orderby = None + + if not kwargs.yield_albums and not kwargs.yield_photos: + exc = exceptions.NoYields(['yield_albums', 'yield_photos']) + self.warning_bag.add(exc) + if self.raise_errors: + raise exceptions.NoYields(['yield_albums', 'yield_photos']) + else: + return + + photo_tag_rel_exist_clauses = searchhelpers.photo_tag_rel_exist_clauses( + kwargs.tag_musts, + kwargs.tag_mays, + kwargs.tag_forbids, + ) + + notnulls = set() + yesnulls = set() + wheres = [] + bindings = [] + + if photo_tag_rel_exist_clauses: + wheres.extend(photo_tag_rel_exist_clauses) + + if kwargs.author: + author_ids = [user.id for user in kwargs.author] + wheres.append(f'author_id IN {sqlhelpers.listify(author_ids)}') + + if kwargs.extension: + if '*' in kwargs.extension: + wheres.append('extension != ""') + else: + qmarks = ', '.join('?' * len(kwargs.extension)) + wheres.append(f'extension IN ({qmarks})') + bindings.extend(kwargs.extension) + + if kwargs.extension_not: + if '*' in kwargs.extension_not: + wheres.append('extension == ""') + else: + qmarks = ', '.join('?' * len(kwargs.extension_not)) + wheres.append(f'extension NOT IN ({qmarks})') + bindings.extend(kwargs.extension_not) + + if kwargs.mimetype: + extensions = { + extension + for (extension, (typ, subtyp)) in constants.MIMETYPES.items() + if typ in kwargs.mimetype + } + wheres.append(f'extension IN {sqlhelpers.listify(extensions)} COLLATE NOCASE') + + if kwargs.within_directory: + patterns = {d.absolute_path.rstrip(os.sep) for d in kwargs.within_directory} + patterns = {f'{d}{os.sep}%' for d in patterns} + clauses = ['filepath LIKE ?'] * len(patterns) + if len(clauses) > 1: + clauses = ' OR '.join(clauses) + clauses = f'({clauses})' + else: + clauses = clauses.pop() + wheres.append(clauses) + bindings.extend(patterns) + + if kwargs.has_albums is True or (kwargs.yield_albums and not kwargs.yield_photos): + wheres.append('EXISTS (SELECT 1 FROM album_photo_rel WHERE photoid == photos.id)') + elif kwargs.has_albums is False: + wheres.append('NOT EXISTS (SELECT 1 FROM album_photo_rel WHERE photoid == photos.id)') + + if kwargs.has_tags is True: + wheres.append('EXISTS (SELECT 1 FROM photo_tag_rel WHERE photoid == photos.id)') + elif kwargs.has_tags is False: + wheres.append('NOT EXISTS (SELECT 1 FROM photo_tag_rel WHERE photoid == photos.id)') + + if kwargs.has_thumbnail is True: + notnulls.add('thumbnail') + elif kwargs.has_thumbnail is False: + yesnulls.add('thumbnail') + + for (column, direction) in orderby: + if column != 'RANDOM()': + notnulls.add(column) + + if kwargs.is_searchhidden is True: + wheres.append('searchhidden == 1') + elif kwargs.is_searchhidden is False: + wheres.append('searchhidden == 0') + + if kwargs.sha256: + wheres.append(f'sha256 IN {sqlhelpers.listify(kwargs.sha256)}') + + for column in notnulls: + wheres.append(column + ' IS NOT NULL') + for column in yesnulls: + wheres.append(column + ' IS NULL') + + for (column, value) in minimums.items(): + wheres.append(column + ' >= ' + str(value)) + + for (column, value) in maximums.items(): + wheres.append(column + ' <= ' + str(value)) + + query = ['SELECT * FROM photos'] + + if wheres: + wheres = 'WHERE ' + ' AND '.join(wheres) + query.append(wheres) + + if orderby: + orderby = [f'{column} {direction}' for (column, direction) in orderby] + orderby = ', '.join(orderby) + orderby = 'ORDER BY ' + orderby + query.append(orderby) + + query = ' '.join(query) + + query = f'{"-" * 80}\n{query}\n{"-" * 80}' + + log.debug('\n%s %s', query, bindings) + # print(self.photodb.explain(query, bindings)) + generator = self.photodb.select(query, bindings) + seen_albums = set() + offset = kwargs.offset + for row in generator: + photo = self.photodb.get_cached_instance(Photo, row) + + if filename_tree and not filename_tree.evaluate(photo.basename.lower()): + continue + + if tag_expression_tree: + photo_tags = set(photo.get_tags()) + success = tag_expression_tree.evaluate( + photo_tags, + match_function=tag_match_function, + ) + if not success: + continue + + if offset > 0: + offset -= 1 + continue + + if kwargs.yield_albums: + new_albums = photo.get_containing_albums().difference(seen_albums) + yield from new_albums + self.results_received += len(new_albums) + seen_albums.update(new_albums) + + if kwargs.yield_photos: + yield photo + self.results_received += 1 + + if kwargs.limit is not None and self.results_received >= kwargs.limit: + break + + try: + next(generator) + except StopIteration: + self.more_after_limit = False + else: + self.more_after_limit = True + + self.generator_exhausted = True + self.end_time = time.perf_counter() + log.debug('Search took %s.', self.end_time - self.start_time) + class Tag(ObjectBase, GroupableMixin): ''' A Tag, which can be applied to Photos for organization. @@ -1619,8 +2048,8 @@ class Tag(ObjectBase, GroupableMixin): # only has A because some other photo with A and B thinks A is obsolete. # This technique is nice and simple to understand for now. ancestors = list(member.walk_parents()) - photos = self.photodb.search(tag_musts=[member], is_searchhidden=None, yield_albums=False) - for photo in photos: + photos = self.photodb.search(tag_musts=[member], is_searchhidden=None, yield_photos=True, yield_albums=False) + for photo in photos.results: photo.remove_tags(ancestors) @decorators.required_feature('tag.edit') @@ -2112,3 +2541,6 @@ class WarningBag: def add(self, warning) -> None: self.warnings.add(warning) + + def jsonify(self): + return [getattr(w, 'error_message', str(w)) for w in self.warnings] diff --git a/etiquette/photodb.py b/etiquette/photodb.py index fcd84d8..cdb8efa 100644 --- a/etiquette/photodb.py +++ b/etiquette/photodb.py @@ -3,9 +3,7 @@ import hashlib import json import os import random -import sqlite3 import tempfile -import time import types import typing @@ -14,12 +12,10 @@ from . import decorators from . import exceptions from . import helpers from . import objects -from . import searchhelpers from . import tag_export from voussoirkit import cacheclass from voussoirkit import configlayers -from voussoirkit import expressionmatch from voussoirkit import pathclass from voussoirkit import ratelimiter from voussoirkit import spinal @@ -411,441 +407,8 @@ class PDBPhotoMixin: photo.delete() yield photo - def search( - self, - *, - area=None, - aspectratio=None, - width=None, - height=None, - bytes=None, - duration=None, - bitrate=None, - - author=None, - created=None, - extension=None, - extension_not=None, - filename=None, - has_albums=None, - has_tags=None, - has_thumbnail=None, - is_searchhidden=False, - mimetype=None, - sha256=None, - tag_musts=None, - tag_mays=None, - tag_forbids=None, - tag_expression=None, - within_directory=None, - - limit=None, - offset=None, - orderby=None, - warning_bag=None, - - give_back_parameters=False, - yield_albums=False, - yield_photos=True, - ): - ''' - PHOTO PROPERTIES - area, aspectratio, width, height, bytes, duration, bitrate: - A dotdot_range string representing min and max. Or just a number - for lower bound. - - TAGS AND FILTERS - author: - A list of User objects or usernames, or a string of comma-separated - usernames. - - created: - A dotdot_range string respresenting min and max. Or just a number - for lower bound. - - extension: - A string or list of strings of acceptable file extensions. - - extension_not: - A string or list of strings of unacceptable file extensions. - Including '*' will forbid all extensions - - filename: - A string or list of strings in the form of an expression. - Match is CASE-INSENSITIVE. - Examples: - '.pdf AND (programming OR "survival guide")' - '.pdf programming python' (implicitly AND each term) - - has_albums: - If True, require that the Photo belongs to >=1 album. - If False, require that the Photo belongs to no albums. - If None, either is okay. - - has_tags: - If True, require that the Photo has >=1 tag. - If False, require that the Photo has no tags. - If None, any amount is okay. - - has_thumbnail: - Require a thumbnail? - If None, anything is okay. - - is_searchhidden: - Find photos that are marked as searchhidden? - If True, find *only* searchhidden photos. - If False, find *only* nonhidden photos. - If None, either is okay. - Default False. - - mimetype: - A string or list of strings of acceptable mimetypes. - 'image', 'video', ... - Note we are only interested in the simple "video", "audio" etc. - For exact mimetypes you might as well use an extension search. - - tag_musts: - A list of tag names or Tag objects. - Photos MUST have ALL tags in this list. - - tag_mays: - A list of tag names or Tag objects. - Photos MUST have AT LEAST ONE tag in this list. - - tag_forbids: - A list of tag names or Tag objects. - Photos MUST NOT have ANY tag in the list. - - tag_expression: - A string or list of strings in the form of an expression. - Can NOT be used with the must, may, forbid style search. - Examples: - 'family AND (animals OR vacation)' - 'family vacation outdoors' (implicitly AND each term) - - within_directory: - A string or list of strings or pathclass Paths of directories. - Photos MUST have a `filepath` that is a child of one of these - directories. - - QUERY OPTIONS - limit: - The maximum number of *successful* results to yield. - - offset: - How many *successful* results to skip before we start yielding. - - orderby: - A list of strings like ['aspectratio DESC', 'created ASC'] to sort - and subsort the results. - Descending is assumed if not provided. - - warning_bag: - If provided, invalid search queries will add a warning to the bag - and try their best to continue. The generator will yield the bag - back to you as the final object. - Without the bag, exceptions may be raised. - - YIELD OPTIONS - give_back_parameters: - If True, the generator's first yield will be a dictionary of all the - cleaned up, normalized parameters. The user may have given us loads - of trash, so we should show them the formatting we want. - - yield_albums: - If True, albums which contain photos matching the search - will be yielded. - - yield_photos: - If True, photos matching the search will be yielded. - ''' - start_time = time.perf_counter() - - maximums = {} - minimums = {} - searchhelpers.minmax('area', area, minimums, maximums, warning_bag=warning_bag) - searchhelpers.minmax('created', created, minimums, maximums, warning_bag=warning_bag) - searchhelpers.minmax('width', width, minimums, maximums, warning_bag=warning_bag) - searchhelpers.minmax('height', height, minimums, maximums, warning_bag=warning_bag) - searchhelpers.minmax('aspectratio', aspectratio, minimums, maximums, warning_bag=warning_bag) - searchhelpers.minmax('bytes', bytes, minimums, maximums, warning_bag=warning_bag) - searchhelpers.minmax('duration', duration, minimums, maximums, warning_bag=warning_bag) - searchhelpers.minmax('bitrate', bitrate, minimums, maximums, warning_bag=warning_bag) - - author = searchhelpers.normalize_author(author, photodb=self, warning_bag=warning_bag) - extension = searchhelpers.normalize_extension(extension) - extension_not = searchhelpers.normalize_extension(extension_not) - filename = searchhelpers.normalize_filename(filename) - has_albums = searchhelpers.normalize_has_tags(has_albums) - has_tags = searchhelpers.normalize_has_tags(has_tags) - has_thumbnail = searchhelpers.normalize_has_thumbnail(has_thumbnail) - is_searchhidden = searchhelpers.normalize_is_searchhidden(is_searchhidden) - sha256 = searchhelpers.normalize_sha256(sha256) - mimetype = searchhelpers.normalize_extension(mimetype) - sha256 = searchhelpers.normalize_extension(sha256) - within_directory = searchhelpers.normalize_within_directory(within_directory, warning_bag=warning_bag) - yield_albums = searchhelpers.normalize_yield_albums(yield_albums) - yield_photos = searchhelpers.normalize_yield_photos(yield_photos) - - if has_tags is False: - if (tag_musts or tag_mays or tag_forbids or tag_expression) and warning_bag: - warning_bag.add("has_tags=False so all tag requests are ignored.") - tag_musts = None - tag_mays = None - tag_forbids = None - tag_expression = None - else: - tag_musts = searchhelpers.normalize_tagset(self, tag_musts, warning_bag=warning_bag) - tag_mays = searchhelpers.normalize_tagset(self, tag_mays, warning_bag=warning_bag) - tag_forbids = searchhelpers.normalize_tagset(self, tag_forbids, warning_bag=warning_bag) - tag_expression = searchhelpers.normalize_tag_expression(tag_expression) - - if extension is not None and extension_not is not None: - extension = extension.difference(extension_not) - - tags_fixed = searchhelpers.normalize_mmf_vs_expression_conflict( - tag_musts, - tag_mays, - tag_forbids, - tag_expression, - warning_bag, - ) - (tag_musts, tag_mays, tag_forbids, tag_expression) = tags_fixed - - if tag_expression: - tag_expression_tree = searchhelpers.tag_expression_tree_builder( - tag_expression=tag_expression, - photodb=self, - warning_bag=warning_bag, - ) - if tag_expression_tree is None: - giveback_tag_expression = None - tag_expression = None - else: - giveback_tag_expression = str(tag_expression_tree) - frozen_children = self.get_cached_tag_export('flat_dict', tags=self.get_root_tags()) - tag_match_function = searchhelpers.tag_expression_matcher_builder(frozen_children) - else: - giveback_tag_expression = None - - if has_tags is True and (tag_musts or tag_mays): - # has_tags check is redundant then, so disable it. - has_tags = None - - limit = searchhelpers.normalize_limit(limit, warning_bag=warning_bag) - offset = searchhelpers.normalize_offset(offset, warning_bag=warning_bag) - orderby = searchhelpers.normalize_orderby(orderby, warning_bag=warning_bag) - - if filename: - try: - filename_tree = expressionmatch.ExpressionTree.parse(filename) - filename_tree.map(lambda x: x.lower()) - except expressionmatch.NoTokens: - filename_tree = None - else: - filename_tree = None - - if orderby: - giveback_orderby = [ - f'{friendly}-{direction}' - for (friendly, expanded, direction) in orderby - ] - orderby = [(expanded, direction) for (friendly, expanded, direction) in orderby] - else: - giveback_orderby = None - orderby = [('created', 'desc')] - - if give_back_parameters: - parameters = { - 'area': area, - 'width': width, - 'height': height, - 'aspectratio': aspectratio, - 'bitrate': bitrate, - 'bytes': bytes, - 'duration': duration, - 'author': list(author) or None, - 'created': created, - 'extension': list(extension) or None, - 'extension_not': list(extension_not) or None, - 'filename': filename or None, - 'has_albums': has_albums, - 'has_tags': has_tags, - 'has_thumbnail': has_thumbnail, - 'mimetype': list(mimetype) or None, - 'sha256': list(sha256) or None, - 'tag_musts': tag_musts or None, - 'tag_mays': tag_mays or None, - 'tag_forbids': tag_forbids or None, - 'tag_expression': giveback_tag_expression or None, - 'within_directory': within_directory or None, - 'limit': limit, - 'offset': offset or None, - 'orderby': giveback_orderby, - 'yield_albums': yield_albums, - 'yield_photos': yield_photos, - } - yield parameters - - if not yield_albums and not yield_photos: - exc = exceptions.NoYields(['yield_albums', 'yield_photos']) - if warning_bag: - warning_bag.add(exc) - yield warning_bag - return - else: - raise exceptions.NoYields(['yield_albums', 'yield_photos']) - - photo_tag_rel_exist_clauses = searchhelpers.photo_tag_rel_exist_clauses( - tag_musts, - tag_mays, - tag_forbids, - ) - - notnulls = set() - yesnulls = set() - wheres = [] - bindings = [] - - if author: - author_ids = [user.id for user in author] - wheres.append(f'author_id IN {sqlhelpers.listify(author_ids)}') - - if extension: - if '*' in extension: - wheres.append('extension != ""') - else: - qmarks = ', '.join('?' * len(extension)) - wheres.append(f'extension IN ({qmarks})') - bindings.extend(extension) - - if extension_not: - if '*' in extension_not: - wheres.append('extension == ""') - else: - qmarks = ', '.join('?' * len(extension_not)) - wheres.append(f'extension NOT IN ({qmarks})') - bindings.extend(extension_not) - - if mimetype: - extensions = {extension for (extension, (typ, subtyp)) in constants.MIMETYPES.items() if typ in mimetype} - wheres.append(f'extension IN {sqlhelpers.listify(extensions)} COLLATE NOCASE') - - if within_directory: - patterns = {d.absolute_path.rstrip(os.sep) for d in within_directory} - patterns = {f'{d}{os.sep}%' for d in patterns} - clauses = ['filepath LIKE ?'] * len(patterns) - if len(clauses) > 1: - clauses = ' OR '.join(clauses) - clauses = f'({clauses})' - else: - clauses = clauses.pop() - wheres.append(clauses) - bindings.extend(patterns) - - if has_albums is True: - wheres.append('EXISTS (SELECT 1 FROM album_photo_rel WHERE photoid == photos.id)') - elif has_albums is False: - wheres.append('NOT EXISTS (SELECT 1 FROM album_photo_rel WHERE photoid == photos.id)') - - if has_tags is True: - wheres.append('EXISTS (SELECT 1 FROM photo_tag_rel WHERE photoid == photos.id)') - elif has_tags is False: - wheres.append('NOT EXISTS (SELECT 1 FROM photo_tag_rel WHERE photoid == photos.id)') - - if yield_albums and not yield_photos: - wheres.append('EXISTS (SELECT 1 FROM album_photo_rel WHERE photoid == photos.id)') - - if has_thumbnail is True: - notnulls.add('thumbnail') - elif has_thumbnail is False: - yesnulls.add('thumbnail') - - for (column, direction) in orderby: - if column != 'RANDOM()': - notnulls.add(column) - - if is_searchhidden is True: - wheres.append('searchhidden == 1') - elif is_searchhidden is False: - wheres.append('searchhidden == 0') - - if sha256: - wheres.append(f'sha256 IN {sqlhelpers.listify(sha256)}') - - for column in notnulls: - wheres.append(column + ' IS NOT NULL') - for column in yesnulls: - wheres.append(column + ' IS NULL') - - for (column, value) in minimums.items(): - wheres.append(column + ' >= ' + str(value)) - - for (column, value) in maximums.items(): - wheres.append(column + ' <= ' + str(value)) - - if photo_tag_rel_exist_clauses: - wheres.extend(photo_tag_rel_exist_clauses) - - query = ['SELECT * FROM photos'] - - if wheres: - wheres = 'WHERE ' + ' AND '.join(wheres) - query.append(wheres) - - if orderby: - orderby = [f'{column} {direction}' for (column, direction) in orderby] - orderby = ', '.join(orderby) - orderby = 'ORDER BY ' + orderby - query.append(orderby) - - query = ' '.join(query) - - query = f'{"-" * 80}\n{query}\n{"-" * 80}' - - log.debug('\n%s %s', query, bindings) - # explain = self.execute('EXPLAIN QUERY PLAN ' + query, bindings) - # print('\n'.join(str(x) for x in explain.fetchall())) - generator = self.select(query, bindings) - seen_albums = set() - results_received = 0 - for row in generator: - photo = self.get_cached_instance(objects.Photo, row) - - if filename_tree and not filename_tree.evaluate(photo.basename.lower()): - continue - - if tag_expression: - photo_tags = set(photo.get_tags()) - success = tag_expression_tree.evaluate( - photo_tags, - match_function=tag_match_function, - ) - if not success: - continue - - if offset > 0: - offset -= 1 - continue - - if limit is not None and results_received >= limit: - break - - if yield_albums: - new_albums = photo.get_containing_albums().difference(seen_albums) - yield from new_albums - results_received += len(new_albums) - seen_albums.update(new_albums) - - if yield_photos: - yield photo - results_received += 1 - - if warning_bag and warning_bag.warnings: - yield warning_bag - - end_time = time.perf_counter() - log.debug('Search took %s.', end_time - start_time) + def search(self, **kwargs): + return objects.Search(photodb=self, kwargs=kwargs) #################################################################################################### diff --git a/etiquette/searchhelpers.py b/etiquette/searchhelpers.py index 34fed27..436c26d 100644 --- a/etiquette/searchhelpers.py +++ b/etiquette/searchhelpers.py @@ -183,7 +183,7 @@ def normalize_filename(filename_terms): Returns: A string where terms are separated by spaces. ''' if filename_terms is None: - filename_terms = '' + return None if not isinstance(filename_terms, str): filename_terms = ' '.join(filename_terms) diff --git a/frontends/etiquette_cli.py b/frontends/etiquette_cli.py index b2f13e8..c2f2e73 100644 --- a/frontends/etiquette_cli.py +++ b/frontends/etiquette_cli.py @@ -125,7 +125,7 @@ def search_in_cwd(**kwargs): return photodb.search( within_directory=cwd, **kwargs, - ) + ).results def search_by_argparse(args, yield_albums=False, yield_photos=False): return search_in_cwd( diff --git a/frontends/etiquette_flask/backend/endpoints/photo_endpoints.py b/frontends/etiquette_flask/backend/endpoints/photo_endpoints.py index 2de02a6..4ea5cf1 100644 --- a/frontends/etiquette_flask/backend/endpoints/photo_endpoints.py +++ b/frontends/etiquette_flask/backend/endpoints/photo_endpoints.py @@ -365,130 +365,98 @@ def post_batch_photos_download_zip(): # Search ########################################################################################### def get_search_core(): - warning_bag = etiquette.objects.WarningBag() + search = common.P.search( + area=request.args.get('area'), + width=request.args.get('width'), + height=request.args.get('height'), + aspectratio=request.args.get('aspectratio'), + bytes=request.args.get('bytes'), + duration=request.args.get('duration'), + bitrate=request.args.get('bitrate'), - has_tags = request.args.get('has_tags') - tag_musts = request.args.get('tag_musts') - tag_mays = request.args.get('tag_mays') - tag_forbids = request.args.get('tag_forbids') - tag_expression = request.args.get('tag_expression') + filename=request.args.get('filename'), + extension_not=request.args.get('extension_not'), + extension=request.args.get('extension'), + mimetype=request.args.get('mimetype'), + sha256=request.args.get('sha256'), - filename_terms = request.args.get('filename') - extension = request.args.get('extension') - extension_not = request.args.get('extension_not') - mimetype = request.args.get('mimetype') - sha256 = request.args.get('sha256') - is_searchhidden = request.args.get('is_searchhidden', False) - yield_albums = request.args.get('yield_albums', False) - yield_photos = request.args.get('yield_photos', True) + author=request.args.get('author'), + created=request.args.get('created'), + has_albums=request.args.get('has_albums'), + has_thumbnail=request.args.get('has_thumbnail'), + is_searchhidden=request.args.get('is_searchhidden', False), - limit = request.args.get('limit') - # This is being pre-processed because the site enforces a maximum value - # which the PhotoDB api does not. - limit = etiquette.searchhelpers.normalize_limit(limit, warning_bag=warning_bag) + has_tags=request.args.get('has_tags'), + tag_musts=request.args.get('tag_musts'), + tag_mays=request.args.get('tag_mays'), + tag_forbids=request.args.get('tag_forbids'), + tag_expression=request.args.get('tag_expression'), - if limit is None: - limit = 50 + limit=request.args.get('limit'), + offset=request.args.get('offset'), + orderby=request.args.get('orderby'), + + yield_albums=request.args.get('yield_albums', False), + yield_photos=request.args.get('yield_photos', True), + ) + + # The site enforces a maximum value which the PhotoDB does not. + search.kwargs.limit = etiquette.searchhelpers.normalize_limit(search.kwargs.limit) + if search.kwargs.limit is None: + search.kwargs.limit = 50 else: - limit = min(limit, 1000) - - offset = request.args.get('offset') - - author = request.args.get('author') - - orderby = request.args.get('orderby') - area = request.args.get('area') - width = request.args.get('width') - height = request.args.get('height') - aspectratio = request.args.get('aspectratio') - bytes = request.args.get('bytes') - has_albums = request.args.get('has_albums') - has_thumbnail = request.args.get('has_thumbnail') - duration = request.args.get('duration') - bitrate = request.args.get('bitrate') - created = request.args.get('created') - - # These are in a dictionary so I can pass them to the page template. - search_kwargs = { - 'area': area, - 'width': width, - 'height': height, - 'aspectratio': aspectratio, - 'bytes': bytes, - 'duration': duration, - 'bitrate': bitrate, - - 'author': author, - 'created': created, - 'extension': extension, - 'extension_not': extension_not, - 'filename': filename_terms, - 'has_albums': has_albums, - 'has_tags': has_tags, - 'has_thumbnail': has_thumbnail, - 'is_searchhidden': is_searchhidden, - 'mimetype': mimetype, - 'sha256': sha256, - 'tag_musts': tag_musts, - 'tag_mays': tag_mays, - 'tag_forbids': tag_forbids, - 'tag_expression': tag_expression, - - 'limit': limit, - 'offset': offset, - 'orderby': orderby, - - 'warning_bag': warning_bag, - 'give_back_parameters': True, - - 'yield_albums': yield_albums, - 'yield_photos': yield_photos, - } - # print(search_kwargs) - search_generator = common.P.search(**search_kwargs) - # Because of the giveback, first element is cleaned up kwargs - search_kwargs = next(search_generator) - # Web UI users aren't allowed to use within_directory anyway, so don't - # show it to them. - search_kwargs.pop('within_directory', None) - # print(search_kwargs) - - warnings = set() - search_results = [] - for item in search_generator: - if isinstance(item, etiquette.objects.WarningBag): - warnings.update(item.warnings) - continue - search_results.append(item) + search.kwargs.limit = min(search.kwargs.limit, 1000) + search.results = list(search.results) warnings = [ w.error_message if hasattr(w, 'error_message') else str(w) - for w in warnings + for w in search.warning_bag.warnings ] + # Web UI users aren't allowed to use within_directory anyway, so don't + # show it to them. + del search.kwargs.within_directory + return search + +@site.route('/search_embed') +def get_search_embed(): + search = get_search_core() + response = common.render_template( + request, + 'search_embed.html', + results=search.results, + search_kwargs=search.kwargs, + ) + return response + +@site.route('/search') +def get_search_html(): + search = get_search_core() + search.kwargs.view = request.args.get('view', 'grid') + # TAGS ON THIS PAGE total_tags = set() - for result in search_results: + for result in search.results: if isinstance(result, etiquette.objects.Photo): total_tags.update(result.get_tags()) total_tags = sorted(total_tags, key=lambda t: t.name) # PREV-NEXT PAGE URLS - offset = search_kwargs['offset'] or 0 + offset = search.kwargs.offset or 0 original_params = request.args.to_dict() - original_params['limit'] = limit + original_params['limit'] = search.kwargs.limit - if limit and len(search_results) >= limit: + if search.more_after_limit: next_params = original_params.copy() - next_params['offset'] = offset + limit + next_params['offset'] = offset + search.kwargs.limit next_params = helpers.dict_to_params(next_params) next_page_url = '/search' + next_params else: next_page_url = None - if limit and offset > 0: + if search.kwargs.limit and offset > 0: prev_params = original_params.copy() - prev_offset = max(0, offset - limit) + prev_offset = max(0, offset - search.kwargs.limit) if prev_offset > 0: prev_params['offset'] = prev_offset else: @@ -498,49 +466,23 @@ def get_search_core(): else: prev_page_url = None - search_kwargs['view'] = request.args.get('view', 'grid') - - final_results = { - 'next_page_url': next_page_url, - 'prev_page_url': prev_page_url, - 'results': search_results, - 'total_tags': total_tags, - 'warnings': list(warnings), - 'search_kwargs': search_kwargs, - } - return final_results - -@site.route('/search_embed') -def get_search_embed(): - search_results = get_search_core() - response = common.render_template( - request, - 'search_embed.html', - results=search_results['results'], - search_kwargs=search_results['search_kwargs'], - ) - return response - -@site.route('/search') -def get_search_html(): - search_results = get_search_core() response = common.render_template( request, 'search.html', - next_page_url=search_results['next_page_url'], - prev_page_url=search_results['prev_page_url'], - results=search_results['results'], - search_kwargs=search_results['search_kwargs'], - total_tags=search_results['total_tags'], - warnings=search_results['warnings'], + next_page_url=next_page_url, + prev_page_url=prev_page_url, + results=search.results, + search_kwargs=search.kwargs, + total_tags=total_tags, + warnings=search.warning_bag.jsonify(), ) return response @site.route('/search.atom') def get_search_atom(): - search_results = get_search_core()['results'] + search = get_search_core() soup = etiquette.helpers.make_atom_feed( - search_results, + search.results, feed_id=request.query_string.decode('utf-8'), feed_title='etiquette search', feed_link=request.url.replace('/search.atom', '/search'), @@ -553,34 +495,9 @@ def get_search_atom(): @site.route('/search.json') def get_search_json(): - search_results = get_search_core() - search_kwargs = search_results['search_kwargs'] - - # The search has converted many arguments into sets or other types. - # Convert them back into something that will display nicely on the search form. - join_helper = lambda x: ', '.join(x) if x else None - search_kwargs['extension'] = join_helper(search_kwargs['extension']) - search_kwargs['extension_not'] = join_helper(search_kwargs['extension_not']) - search_kwargs['mimetype'] = join_helper(search_kwargs['mimetype']) - - author_helper = lambda users: ', '.join(user.username for user in users) if users else None - search_kwargs['author'] = author_helper(search_kwargs['author']) - - tagname_helper = lambda tags: [tag.name for tag in tags] if tags else None - search_kwargs['tag_musts'] = tagname_helper(search_kwargs['tag_musts']) - search_kwargs['tag_mays'] = tagname_helper(search_kwargs['tag_mays']) - search_kwargs['tag_forbids'] = tagname_helper(search_kwargs['tag_forbids']) - - search_results['results'] = [ - result.jsonify(include_albums=False) - if isinstance(result, etiquette.objects.Photo) else - result.jsonify(minimal=True) - for result in search_results['results'] - ] - search_results['total_tags'] = [ - tag.jsonify(minimal=True) for tag in search_results['total_tags'] - ] - return flasktools.json_response(search_results) + search = get_search_core() + response = search.jsonify() + return flasktools.json_response(response) # Swipe ############################################################################################