Move search to an entire class of its own.

The initial motivation for this was to make the "more_after_limit"
feature, which would help the UI to not show a next page button when
the number of results was exactly equal to the limit.

However, in order to surface this more_after_limit status using only
the old search generator, it would have to be a special yield at the
end. I was getting tired of the special yields like give_back_params
at the beginning and warning_bag at the end, and this would be worse.

There is a lot of sideband information about the search that is now
more easily accessible when the search is its own object.

etiquette/objects.py

@@ -11,10 +11,13 @@ import os
 import PIL.Image
 import re
 import send2trash
+import time
 import traceback
 import typing
 
 from voussoirkit import bytestring
+from voussoirkit import dotdict
+from voussoirkit import expressionmatch
 from voussoirkit import gentools
 from voussoirkit import hms
 from voussoirkit import pathclass
@@ -32,6 +35,7 @@ from . import constants
 from . import decorators
 from . import exceptions
 from . import helpers
+from . import searchhelpers
 
 BAIL = sentinel.Sentinel('BAIL')
 
@@ -1521,6 +1525,431 @@ class Photo(ObjectBase):
         self._tagged_at_dt = helpers.utcfromtimestamp(self.tagged_at_unix)
         return self._tagged_at_dt
 
+class Search:
+    '''
+    FILE METADATA
+    =============
+    area, aspectratio, width, height, bytes, duration, bitrate:
+        A dotdot_range string representing min and max. Or just a number
+        for lower bound.
+
+    extension:
+        A string or list of strings of acceptable file extensions.
+
+    extension_not:
+        A string or list of strings of unacceptable file extensions.
+        Including '*' will forbid all extensions, thus returning only
+        extensionless files.
+
+    filename:
+        A string or list of strings in the form of an expression.
+        Match is CASE-INSENSITIVE.
+        Examples:
+        '.pdf AND (programming OR "survival guide")'
+        '.pdf programming python' (implicitly AND each term)
+
+    sha256:
+        A string or list of strings of exact SHA256 hashes to match.
+
+    within_directory:
+        A string or list of strings or pathclass Paths of directories.
+        Photos MUST have a `filepath` that is a child of one of these
+        directories.
+
+    OBJECT METADATA
+    ===============
+    author:
+        A list of User objects or usernames, or a string of comma-separated
+        usernames.
+
+    created:
+        A dotdot_range string respresenting min and max. Or just a number
+        for lower bound.
+
+    has_albums:
+        If True, require that the Photo belongs to >=1 album.
+        If False, require that the Photo belongs to no albums.
+        If None, either is okay.
+
+    has_tags:
+        If True, require that the Photo has >=1 tag.
+        If False, require that the Photo has no tags.
+        If None, any amount is okay.
+
+    has_thumbnail:
+        Require a thumbnail?
+        If None, anything is okay.
+
+    is_searchhidden:
+        If True, find *only* searchhidden photos.
+        If False, find *only* nonhidden photos.
+        If None, either is okay.
+
+    mimetype:
+        A string or list of strings of acceptable mimetypes.
+        'image', 'video', ...
+        Note we are only interested in the simple "video", "audio" etc.
+        For exact mimetypes you might as well use an extension search.
+
+    TAGS
+    ====
+    tag_musts:
+        A list of tag names or Tag objects.
+        Photos MUST have ALL tags in this list.
+
+    tag_mays:
+        A list of tag names or Tag objects.
+        Photos MUST have AT LEAST ONE tag in this list.
+
+    tag_forbids:
+        A list of tag names or Tag objects.
+        Photos MUST NOT have ANY tag in the list.
+
+    tag_expression:
+        A string or list of strings in the form of an expression.
+        Can NOT be used with the must, may, forbid style search.
+        Examples:
+        'family AND (animals OR vacation)'
+        'family vacation outdoors' (implicitly AND each term)
+
+    QUERY OPTIONS
+    =============
+    limit:
+        The maximum number of *successful* results to yield.
+
+    offset:
+        How many *successful* results to skip before we start yielding.
+
+    orderby:
+        A list of strings like ['aspectratio DESC', 'created ASC'] to sort
+        and subsort the results.
+        Descending is assumed if not provided.
+
+    yield_albums:
+        If True, albums which contain photos matching the search
+        will be yielded.
+
+    yield_photos:
+        If True, photos matching the search will be yielded.
+    '''
+    def __init__(self, photodb, kwargs, *, raise_errors=True):
+        self.photodb = photodb
+        self.created = timetools.now()
+        self.raise_errors = raise_errors
+        if isinstance(kwargs, dict):
+            kwargs = dotdict.DotDict(kwargs, default=None)
+        self.kwargs = kwargs
+        self.generator_started = False
+        self.generator_exhausted = False
+        self.more_after_limit = None
+        self.start_time = None
+        self.end_time = None
+        self.start_commit_id = None
+        self.warning_bag = WarningBag()
+        self.results = self._generator()
+        self.results_received = 0
+
+    def atomify(self):
+        raise NotImplementedError
+
+    def jsonify(self):
+        # The search has converted many arguments into sets or other types.
+        # Convert them back into something that will display nicely on the search form.
+        kwargs = self.kwargs._to_dict()
+        join_helper = lambda x: ', '.join(x) if x else None
+        kwargs['extension'] = join_helper(kwargs['extension'])
+        kwargs['extension_not'] = join_helper(kwargs['extension_not'])
+        kwargs['mimetype'] = join_helper(kwargs['mimetype'])
+        kwargs['sha256'] = join_helper(kwargs['sha256'])
+
+        author_helper = lambda users: ', '.join(user.username for user in users) if users else None
+        kwargs['author'] = author_helper(kwargs['author'])
+
+        tagname_helper = lambda tags: [tag.name for tag in tags] if tags else None
+        kwargs['tag_musts'] = tagname_helper(kwargs['tag_musts'])
+        kwargs['tag_mays'] = tagname_helper(kwargs['tag_mays'])
+        kwargs['tag_forbids'] = tagname_helper(kwargs['tag_forbids'])
+
+        results = [
+            result.jsonify(include_albums=False)
+            if isinstance(result, Photo) else
+            result.jsonify(minimal=True)
+            for result in self.results
+        ]
+
+        j = {
+            'kwargs': kwargs,
+            'results': results,
+            'more_after_limit': self.more_after_limit,
+        }
+        return j
+
+    def _generator(self):
+        self.start_time = time.perf_counter()
+        self.generator_started = True
+        self.start_commit_id = self.photodb.last_commit_id
+
+        kwargs = self.kwargs
+
+        maximums = {}
+        minimums = {}
+        searchhelpers.minmax('area', kwargs.area, minimums, maximums, warning_bag=self.warning_bag)
+        searchhelpers.minmax('created', kwargs.created, minimums, maximums, warning_bag=self.warning_bag)
+        searchhelpers.minmax('width', kwargs.width, minimums, maximums, warning_bag=self.warning_bag)
+        searchhelpers.minmax('height', kwargs.height, minimums, maximums, warning_bag=self.warning_bag)
+        searchhelpers.minmax('aspectratio', kwargs.aspectratio, minimums, maximums, warning_bag=self.warning_bag)
+        searchhelpers.minmax('bytes', kwargs.bytes, minimums, maximums, warning_bag=self.warning_bag)
+        searchhelpers.minmax('duration', kwargs.duration, minimums, maximums, warning_bag=self.warning_bag)
+        searchhelpers.minmax('bitrate', kwargs.bitrate, minimums, maximums, warning_bag=self.warning_bag)
+
+        kwargs.author = searchhelpers.normalize_author(kwargs.author, photodb=self.photodb, warning_bag=self.warning_bag)
+        kwargs.extension = searchhelpers.normalize_extension(kwargs.extension)
+        kwargs.extension_not = searchhelpers.normalize_extension(kwargs.extension_not)
+        kwargs.filename = searchhelpers.normalize_filename(kwargs.filename)
+        kwargs.has_albums = searchhelpers.normalize_has_tags(kwargs.has_albums)
+        kwargs.has_tags = searchhelpers.normalize_has_tags(kwargs.has_tags)
+        kwargs.has_thumbnail = searchhelpers.normalize_has_thumbnail(kwargs.has_thumbnail)
+        kwargs.is_searchhidden = searchhelpers.normalize_is_searchhidden(kwargs.is_searchhidden)
+        kwargs.sha256 = searchhelpers.normalize_sha256(kwargs.sha256)
+        kwargs.mimetype = searchhelpers.normalize_extension(kwargs.mimetype)
+        kwargs.sha256 = searchhelpers.normalize_extension(kwargs.sha256)
+        kwargs.within_directory = searchhelpers.normalize_within_directory(kwargs.within_directory, warning_bag=self.warning_bag)
+        kwargs.yield_albums = searchhelpers.normalize_yield_albums(kwargs.yield_albums)
+        kwargs.yield_photos = searchhelpers.normalize_yield_photos(kwargs.yield_photos)
+
+        if kwargs.has_tags is False:
+            if (kwargs.tag_musts or kwargs.tag_mays or kwargs.tag_forbids or kwargs.tag_expression):
+                self.warning_bag.add("has_tags=False so all tag requests are ignored.")
+            kwargs.tag_musts = None
+            kwargs.tag_mays = None
+            kwargs.tag_forbids = None
+            kwargs.tag_expression = None
+        else:
+            kwargs.tag_musts = searchhelpers.normalize_tagset(self.photodb, kwargs.tag_musts, warning_bag=self.warning_bag)
+            kwargs.tag_mays = searchhelpers.normalize_tagset(self.photodb, kwargs.tag_mays, warning_bag=self.warning_bag)
+            kwargs.tag_forbids = searchhelpers.normalize_tagset(self.photodb, kwargs.tag_forbids, warning_bag=self.warning_bag)
+            kwargs.tag_expression = searchhelpers.normalize_tag_expression(kwargs.tag_expression)
+
+        if kwargs.extension is not None and kwargs.extension_not is not None:
+            kwargs.extension = kwargs.extension.difference(kwargs.extension_not)
+
+        tags_fixed = searchhelpers.normalize_mmf_vs_expression_conflict(
+            kwargs.tag_musts,
+            kwargs.tag_mays,
+            kwargs.tag_forbids,
+            kwargs.tag_expression,
+            self.warning_bag,
+        )
+        (kwargs.tag_musts, kwargs.tag_mays, kwargs.tag_forbids, kwargs.tag_expression) = tags_fixed
+
+        if kwargs.tag_expression:
+            tag_expression_tree = searchhelpers.tag_expression_tree_builder(
+                tag_expression=kwargs.tag_expression,
+                photodb=self.photodb,
+                warning_bag=self.warning_bag,
+            )
+            if tag_expression_tree is None:
+                kwargs.tag_expression = None
+                kwargs.tag_expression = None
+            else:
+                kwargs.tag_expression = str(tag_expression_tree)
+                frozen_children = self.photodb.get_cached_tag_export('flat_dict', tags=self.get_root_tags())
+                tag_match_function = searchhelpers.tag_expression_matcher_builder(frozen_children)
+        else:
+            tag_expression_tree = None
+            kwargs.tag_expression = None
+
+        if kwargs.has_tags is True and (kwargs.tag_musts or kwargs.tag_mays):
+            # has_tags check is redundant then, so disable it.
+            kwargs.has_tags = None
+
+        kwargs.limit = searchhelpers.normalize_limit(kwargs.limit, warning_bag=self.warning_bag)
+        kwargs.offset = searchhelpers.normalize_offset(kwargs.offset, warning_bag=self.warning_bag)
+        kwargs.orderby = searchhelpers.normalize_orderby(kwargs.orderby, warning_bag=self.warning_bag)
+
+        if kwargs.filename:
+            try:
+                filename_tree = expressionmatch.ExpressionTree.parse(kwargs.filename)
+                filename_tree.map(lambda x: x.lower())
+            except expressionmatch.NoTokens:
+                filename_tree = None
+        else:
+            filename_tree = None
+
+        if kwargs.orderby:
+            orderby = [(expanded, direction) for (friendly, expanded, direction) in kwargs.orderby]
+            kwargs.orderby = [
+                f'{friendly}-{direction}'
+                for (friendly, expanded, direction) in kwargs.orderby
+            ]
+        else:
+            orderby = [('created', 'desc')]
+            kwargs.orderby = None
+
+        if not kwargs.yield_albums and not kwargs.yield_photos:
+            exc = exceptions.NoYields(['yield_albums', 'yield_photos'])
+            self.warning_bag.add(exc)
+            if self.raise_errors:
+                raise exceptions.NoYields(['yield_albums', 'yield_photos'])
+            else:
+                return
+
+        photo_tag_rel_exist_clauses = searchhelpers.photo_tag_rel_exist_clauses(
+            kwargs.tag_musts,
+            kwargs.tag_mays,
+            kwargs.tag_forbids,
+        )
+
+        notnulls = set()
+        yesnulls = set()
+        wheres = []
+        bindings = []
+
+        if photo_tag_rel_exist_clauses:
+            wheres.extend(photo_tag_rel_exist_clauses)
+
+        if kwargs.author:
+            author_ids = [user.id for user in kwargs.author]
+            wheres.append(f'author_id IN {sqlhelpers.listify(author_ids)}')
+
+        if kwargs.extension:
+            if '*' in kwargs.extension:
+                wheres.append('extension != ""')
+            else:
+                qmarks = ', '.join('?' * len(kwargs.extension))
+                wheres.append(f'extension IN ({qmarks})')
+                bindings.extend(kwargs.extension)
+
+        if kwargs.extension_not:
+            if '*' in kwargs.extension_not:
+                wheres.append('extension == ""')
+            else:
+                qmarks = ', '.join('?' * len(kwargs.extension_not))
+                wheres.append(f'extension NOT IN ({qmarks})')
+                bindings.extend(kwargs.extension_not)
+
+        if kwargs.mimetype:
+            extensions = {
+                extension
+                for (extension, (typ, subtyp)) in constants.MIMETYPES.items()
+                if typ in kwargs.mimetype
+            }
+            wheres.append(f'extension IN {sqlhelpers.listify(extensions)} COLLATE NOCASE')
+
+        if kwargs.within_directory:
+            patterns = {d.absolute_path.rstrip(os.sep) for d in kwargs.within_directory}
+            patterns = {f'{d}{os.sep}%' for d in patterns}
+            clauses = ['filepath LIKE ?'] * len(patterns)
+            if len(clauses) > 1:
+                clauses = ' OR '.join(clauses)
+                clauses = f'({clauses})'
+            else:
+                clauses = clauses.pop()
+            wheres.append(clauses)
+            bindings.extend(patterns)
+
+        if kwargs.has_albums is True or (kwargs.yield_albums and not kwargs.yield_photos):
+            wheres.append('EXISTS (SELECT 1 FROM album_photo_rel WHERE photoid == photos.id)')
+        elif kwargs.has_albums is False:
+            wheres.append('NOT EXISTS (SELECT 1 FROM album_photo_rel WHERE photoid == photos.id)')
+
+        if kwargs.has_tags is True:
+            wheres.append('EXISTS (SELECT 1 FROM photo_tag_rel WHERE photoid == photos.id)')
+        elif kwargs.has_tags is False:
+            wheres.append('NOT EXISTS (SELECT 1 FROM photo_tag_rel WHERE photoid == photos.id)')
+
+        if kwargs.has_thumbnail is True:
+            notnulls.add('thumbnail')
+        elif kwargs.has_thumbnail is False:
+            yesnulls.add('thumbnail')
+
+        for (column, direction) in orderby:
+            if column != 'RANDOM()':
+                notnulls.add(column)
+
+        if kwargs.is_searchhidden is True:
+            wheres.append('searchhidden == 1')
+        elif kwargs.is_searchhidden is False:
+            wheres.append('searchhidden == 0')
+
+        if kwargs.sha256:
+            wheres.append(f'sha256 IN {sqlhelpers.listify(kwargs.sha256)}')
+
+        for column in notnulls:
+            wheres.append(column + ' IS NOT NULL')
+        for column in yesnulls:
+            wheres.append(column + ' IS NULL')
+
+        for (column, value) in minimums.items():
+            wheres.append(column + ' >= ' + str(value))
+
+        for (column, value) in maximums.items():
+            wheres.append(column + ' <= ' + str(value))
+
+        query = ['SELECT * FROM photos']
+
+        if wheres:
+            wheres = 'WHERE ' + ' AND '.join(wheres)
+            query.append(wheres)
+
+        if orderby:
+            orderby = [f'{column} {direction}' for (column, direction) in orderby]
+            orderby = ', '.join(orderby)
+            orderby = 'ORDER BY ' + orderby
+            query.append(orderby)
+
+        query = ' '.join(query)
+
+        query = f'{"-" * 80}\n{query}\n{"-" * 80}'
+
+        log.debug('\n%s %s', query, bindings)
+        # print(self.photodb.explain(query, bindings))
+        generator = self.photodb.select(query, bindings)
+        seen_albums = set()
+        offset = kwargs.offset
+        for row in generator:
+            photo = self.photodb.get_cached_instance(Photo, row)
+
+            if filename_tree and not filename_tree.evaluate(photo.basename.lower()):
+                continue
+
+            if tag_expression_tree:
+                photo_tags = set(photo.get_tags())
+                success = tag_expression_tree.evaluate(
+                    photo_tags,
+                    match_function=tag_match_function,
+                )
+                if not success:
+                    continue
+
+            if offset > 0:
+                offset -= 1
+                continue
+
+            if kwargs.yield_albums:
+                new_albums = photo.get_containing_albums().difference(seen_albums)
+                yield from new_albums
+                self.results_received += len(new_albums)
+                seen_albums.update(new_albums)
+
+            if kwargs.yield_photos:
+                yield photo
+                self.results_received += 1
+
+            if kwargs.limit is not None and self.results_received >= kwargs.limit:
+                break
+
+        try:
+            next(generator)
+        except StopIteration:
+            self.more_after_limit = False
+        else:
+            self.more_after_limit = True
+
+        self.generator_exhausted = True
+        self.end_time = time.perf_counter()
+        log.debug('Search took %s.', self.end_time - self.start_time)
+
 class Tag(ObjectBase, GroupableMixin):
     '''
     A Tag, which can be applied to Photos for organization.
@@ -1619,8 +2048,8 @@ class Tag(ObjectBase, GroupableMixin):
         # only has A because some other photo with A and B thinks A is obsolete.
         # This technique is nice and simple to understand for now.
         ancestors = list(member.walk_parents())
-        photos = self.photodb.search(tag_musts=[member], is_searchhidden=None, yield_albums=False)
+        photos = self.photodb.search(tag_musts=[member], is_searchhidden=None, yield_photos=True, yield_albums=False)
-        for photo in photos:
+        for photo in photos.results:
             photo.remove_tags(ancestors)
 
     @decorators.required_feature('tag.edit')
@@ -2112,3 +2541,6 @@ class WarningBag:
 
     def add(self, warning) -> None:
         self.warnings.add(warning)
+
+    def jsonify(self):
+        return [getattr(w, 'error_message', str(w)) for w in self.warnings]

etiquette/photodb.py

@@ -3,9 +3,7 @@ import hashlib
 import json
 import os
 import random
-import sqlite3
 import tempfile
-import time
 import types
 import typing
 
@@ -14,12 +12,10 @@ from . import decorators
 from . import exceptions
 from . import helpers
 from . import objects
-from . import searchhelpers
 from . import tag_export
 
 from voussoirkit import cacheclass
 from voussoirkit import configlayers
-from voussoirkit import expressionmatch
 from voussoirkit import pathclass
 from voussoirkit import ratelimiter
 from voussoirkit import spinal
@@ -411,441 +407,8 @@ class PDBPhotoMixin:
             photo.delete()
             yield photo
 
-    def search(
+    def search(self, **kwargs):
-            self,
+        return objects.Search(photodb=self, kwargs=kwargs)
-            *,
-            area=None,
-            aspectratio=None,
-            width=None,
-            height=None,
-            bytes=None,
-            duration=None,
-            bitrate=None,
-
-            author=None,
-            created=None,
-            extension=None,
-            extension_not=None,
-            filename=None,
-            has_albums=None,
-            has_tags=None,
-            has_thumbnail=None,
-            is_searchhidden=False,
-            mimetype=None,
-            sha256=None,
-            tag_musts=None,
-            tag_mays=None,
-            tag_forbids=None,
-            tag_expression=None,
-            within_directory=None,
-
-            limit=None,
-            offset=None,
-            orderby=None,
-            warning_bag=None,
-
-            give_back_parameters=False,
-            yield_albums=False,
-            yield_photos=True,
-        ):
-        '''
-        PHOTO PROPERTIES
-        area, aspectratio, width, height, bytes, duration, bitrate:
-            A dotdot_range string representing min and max. Or just a number
-            for lower bound.
-
-        TAGS AND FILTERS
-        author:
-            A list of User objects or usernames, or a string of comma-separated
-            usernames.
-
-        created:
-            A dotdot_range string respresenting min and max. Or just a number
-            for lower bound.
-
-        extension:
-            A string or list of strings of acceptable file extensions.
-
-        extension_not:
-            A string or list of strings of unacceptable file extensions.
-            Including '*' will forbid all extensions
-
-        filename:
-            A string or list of strings in the form of an expression.
-            Match is CASE-INSENSITIVE.
-            Examples:
-            '.pdf AND (programming OR "survival guide")'
-            '.pdf programming python' (implicitly AND each term)
-
-        has_albums:
-            If True, require that the Photo belongs to >=1 album.
-            If False, require that the Photo belongs to no albums.
-            If None, either is okay.
-
-        has_tags:
-            If True, require that the Photo has >=1 tag.
-            If False, require that the Photo has no tags.
-            If None, any amount is okay.
-
-        has_thumbnail:
-            Require a thumbnail?
-            If None, anything is okay.
-
-        is_searchhidden:
-            Find photos that are marked as searchhidden?
-            If True, find *only* searchhidden photos.
-            If False, find *only* nonhidden photos.
-            If None, either is okay.
-            Default False.
-
-        mimetype:
-            A string or list of strings of acceptable mimetypes.
-            'image', 'video', ...
-            Note we are only interested in the simple "video", "audio" etc.
-            For exact mimetypes you might as well use an extension search.
-
-        tag_musts:
-            A list of tag names or Tag objects.
-            Photos MUST have ALL tags in this list.
-
-        tag_mays:
-            A list of tag names or Tag objects.
-            Photos MUST have AT LEAST ONE tag in this list.
-
-        tag_forbids:
-            A list of tag names or Tag objects.
-            Photos MUST NOT have ANY tag in the list.
-
-        tag_expression:
-            A string or list of strings in the form of an expression.
-            Can NOT be used with the must, may, forbid style search.
-            Examples:
-            'family AND (animals OR vacation)'
-            'family vacation outdoors' (implicitly AND each term)
-
-        within_directory:
-            A string or list of strings or pathclass Paths of directories.
-            Photos MUST have a `filepath` that is a child of one of these
-            directories.
-
-        QUERY OPTIONS
-        limit:
-            The maximum number of *successful* results to yield.
-
-        offset:
-            How many *successful* results to skip before we start yielding.
-
-        orderby:
-            A list of strings like ['aspectratio DESC', 'created ASC'] to sort
-            and subsort the results.
-            Descending is assumed if not provided.
-
-        warning_bag:
-            If provided, invalid search queries will add a warning to the bag
-            and try their best to continue. The generator will yield the bag
-            back to you as the final object.
-            Without the bag, exceptions may be raised.
-
-        YIELD OPTIONS
-        give_back_parameters:
-            If True, the generator's first yield will be a dictionary of all the
-            cleaned up, normalized parameters. The user may have given us loads
-            of trash, so we should show them the formatting we want.
-
-        yield_albums:
-            If True, albums which contain photos matching the search
-            will be yielded.
-
-        yield_photos:
-            If True, photos matching the search will be yielded.
-        '''
-        start_time = time.perf_counter()
-
-        maximums = {}
-        minimums = {}
-        searchhelpers.minmax('area', area, minimums, maximums, warning_bag=warning_bag)
-        searchhelpers.minmax('created', created, minimums, maximums, warning_bag=warning_bag)
-        searchhelpers.minmax('width', width, minimums, maximums, warning_bag=warning_bag)
-        searchhelpers.minmax('height', height, minimums, maximums, warning_bag=warning_bag)
-        searchhelpers.minmax('aspectratio', aspectratio, minimums, maximums, warning_bag=warning_bag)
-        searchhelpers.minmax('bytes', bytes, minimums, maximums, warning_bag=warning_bag)
-        searchhelpers.minmax('duration', duration, minimums, maximums, warning_bag=warning_bag)
-        searchhelpers.minmax('bitrate', bitrate, minimums, maximums, warning_bag=warning_bag)
-
-        author = searchhelpers.normalize_author(author, photodb=self, warning_bag=warning_bag)
-        extension = searchhelpers.normalize_extension(extension)
-        extension_not = searchhelpers.normalize_extension(extension_not)
-        filename = searchhelpers.normalize_filename(filename)
-        has_albums = searchhelpers.normalize_has_tags(has_albums)
-        has_tags = searchhelpers.normalize_has_tags(has_tags)
-        has_thumbnail = searchhelpers.normalize_has_thumbnail(has_thumbnail)
-        is_searchhidden = searchhelpers.normalize_is_searchhidden(is_searchhidden)
-        sha256 = searchhelpers.normalize_sha256(sha256)
-        mimetype = searchhelpers.normalize_extension(mimetype)
-        sha256 = searchhelpers.normalize_extension(sha256)
-        within_directory = searchhelpers.normalize_within_directory(within_directory, warning_bag=warning_bag)
-        yield_albums = searchhelpers.normalize_yield_albums(yield_albums)
-        yield_photos = searchhelpers.normalize_yield_photos(yield_photos)
-
-        if has_tags is False:
-            if (tag_musts or tag_mays or tag_forbids or tag_expression) and warning_bag:
-                warning_bag.add("has_tags=False so all tag requests are ignored.")
-            tag_musts = None
-            tag_mays = None
-            tag_forbids = None
-            tag_expression = None
-        else:
-            tag_musts = searchhelpers.normalize_tagset(self, tag_musts, warning_bag=warning_bag)
-            tag_mays = searchhelpers.normalize_tagset(self, tag_mays, warning_bag=warning_bag)
-            tag_forbids = searchhelpers.normalize_tagset(self, tag_forbids, warning_bag=warning_bag)
-            tag_expression = searchhelpers.normalize_tag_expression(tag_expression)
-
-        if extension is not None and extension_not is not None:
-            extension = extension.difference(extension_not)
-
-        tags_fixed = searchhelpers.normalize_mmf_vs_expression_conflict(
-            tag_musts,
-            tag_mays,
-            tag_forbids,
-            tag_expression,
-            warning_bag,
-        )
-        (tag_musts, tag_mays, tag_forbids, tag_expression) = tags_fixed
-
-        if tag_expression:
-            tag_expression_tree = searchhelpers.tag_expression_tree_builder(
-                tag_expression=tag_expression,
-                photodb=self,
-                warning_bag=warning_bag,
-            )
-            if tag_expression_tree is None:
-                giveback_tag_expression = None
-                tag_expression = None
-            else:
-                giveback_tag_expression = str(tag_expression_tree)
-                frozen_children = self.get_cached_tag_export('flat_dict', tags=self.get_root_tags())
-                tag_match_function = searchhelpers.tag_expression_matcher_builder(frozen_children)
-        else:
-            giveback_tag_expression = None
-
-        if has_tags is True and (tag_musts or tag_mays):
-            # has_tags check is redundant then, so disable it.
-            has_tags = None
-
-        limit = searchhelpers.normalize_limit(limit, warning_bag=warning_bag)
-        offset = searchhelpers.normalize_offset(offset, warning_bag=warning_bag)
-        orderby = searchhelpers.normalize_orderby(orderby, warning_bag=warning_bag)
-
-        if filename:
-            try:
-                filename_tree = expressionmatch.ExpressionTree.parse(filename)
-                filename_tree.map(lambda x: x.lower())
-            except expressionmatch.NoTokens:
-                filename_tree = None
-        else:
-            filename_tree = None
-
-        if orderby:
-            giveback_orderby = [
-                f'{friendly}-{direction}'
-                for (friendly, expanded, direction) in orderby
-            ]
-            orderby = [(expanded, direction) for (friendly, expanded, direction) in orderby]
-        else:
-            giveback_orderby = None
-            orderby = [('created', 'desc')]
-
-        if give_back_parameters:
-            parameters = {
-                'area': area,
-                'width': width,
-                'height': height,
-                'aspectratio': aspectratio,
-                'bitrate': bitrate,
-                'bytes': bytes,
-                'duration': duration,
-                'author': list(author) or None,
-                'created': created,
-                'extension': list(extension) or None,
-                'extension_not': list(extension_not) or None,
-                'filename': filename or None,
-                'has_albums': has_albums,
-                'has_tags': has_tags,
-                'has_thumbnail': has_thumbnail,
-                'mimetype': list(mimetype) or None,
-                'sha256': list(sha256) or None,
-                'tag_musts': tag_musts or None,
-                'tag_mays': tag_mays or None,
-                'tag_forbids': tag_forbids or None,
-                'tag_expression': giveback_tag_expression or None,
-                'within_directory': within_directory or None,
-                'limit': limit,
-                'offset': offset or None,
-                'orderby': giveback_orderby,
-                'yield_albums': yield_albums,
-                'yield_photos': yield_photos,
-            }
-            yield parameters
-
-        if not yield_albums and not yield_photos:
-            exc = exceptions.NoYields(['yield_albums', 'yield_photos'])
-            if warning_bag:
-                warning_bag.add(exc)
-                yield warning_bag
-                return
-            else:
-                raise exceptions.NoYields(['yield_albums', 'yield_photos'])
-
-        photo_tag_rel_exist_clauses = searchhelpers.photo_tag_rel_exist_clauses(
-            tag_musts,
-            tag_mays,
-            tag_forbids,
-        )
-
-        notnulls = set()
-        yesnulls = set()
-        wheres = []
-        bindings = []
-
-        if author:
-            author_ids = [user.id for user in author]
-            wheres.append(f'author_id IN {sqlhelpers.listify(author_ids)}')
-
-        if extension:
-            if '*' in extension:
-                wheres.append('extension != ""')
-            else:
-                qmarks = ', '.join('?' * len(extension))
-                wheres.append(f'extension IN ({qmarks})')
-                bindings.extend(extension)
-
-        if extension_not:
-            if '*' in extension_not:
-                wheres.append('extension == ""')
-            else:
-                qmarks = ', '.join('?' * len(extension_not))
-                wheres.append(f'extension NOT IN ({qmarks})')
-                bindings.extend(extension_not)
-
-        if mimetype:
-            extensions = {extension for (extension, (typ, subtyp)) in constants.MIMETYPES.items() if typ in mimetype}
-            wheres.append(f'extension IN {sqlhelpers.listify(extensions)} COLLATE NOCASE')
-
-        if within_directory:
-            patterns = {d.absolute_path.rstrip(os.sep) for d in within_directory}
-            patterns = {f'{d}{os.sep}%' for d in patterns}
-            clauses = ['filepath LIKE ?'] * len(patterns)
-            if len(clauses) > 1:
-                clauses = ' OR '.join(clauses)
-                clauses = f'({clauses})'
-            else:
-                clauses = clauses.pop()
-            wheres.append(clauses)
-            bindings.extend(patterns)
-
-        if has_albums is True:
-            wheres.append('EXISTS (SELECT 1 FROM album_photo_rel WHERE photoid == photos.id)')
-        elif has_albums is False:
-            wheres.append('NOT EXISTS (SELECT 1 FROM album_photo_rel WHERE photoid == photos.id)')
-
-        if has_tags is True:
-            wheres.append('EXISTS (SELECT 1 FROM photo_tag_rel WHERE photoid == photos.id)')
-        elif has_tags is False:
-            wheres.append('NOT EXISTS (SELECT 1 FROM photo_tag_rel WHERE photoid == photos.id)')
-
-        if yield_albums and not yield_photos:
-            wheres.append('EXISTS (SELECT 1 FROM album_photo_rel WHERE photoid == photos.id)')
-
-        if has_thumbnail is True:
-            notnulls.add('thumbnail')
-        elif has_thumbnail is False:
-            yesnulls.add('thumbnail')
-
-        for (column, direction) in orderby:
-            if column != 'RANDOM()':
-                notnulls.add(column)
-
-        if is_searchhidden is True:
-            wheres.append('searchhidden == 1')
-        elif is_searchhidden is False:
-            wheres.append('searchhidden == 0')
-
-        if sha256:
-            wheres.append(f'sha256 IN {sqlhelpers.listify(sha256)}')
-
-        for column in notnulls:
-            wheres.append(column + ' IS NOT NULL')
-        for column in yesnulls:
-            wheres.append(column + ' IS NULL')
-
-        for (column, value) in minimums.items():
-            wheres.append(column + ' >= ' + str(value))
-
-        for (column, value) in maximums.items():
-            wheres.append(column + ' <= ' + str(value))
-
-        if photo_tag_rel_exist_clauses:
-            wheres.extend(photo_tag_rel_exist_clauses)
-
-        query = ['SELECT * FROM photos']
-
-        if wheres:
-            wheres = 'WHERE ' + ' AND '.join(wheres)
-            query.append(wheres)
-
-        if orderby:
-            orderby = [f'{column} {direction}' for (column, direction) in orderby]
-            orderby = ', '.join(orderby)
-            orderby = 'ORDER BY ' + orderby
-            query.append(orderby)
-
-        query = ' '.join(query)
-
-        query = f'{"-" * 80}\n{query}\n{"-" * 80}'
-
-        log.debug('\n%s %s', query, bindings)
-        # explain = self.execute('EXPLAIN QUERY PLAN ' + query, bindings)
-        # print('\n'.join(str(x) for x in explain.fetchall()))
-        generator = self.select(query, bindings)
-        seen_albums = set()
-        results_received = 0
-        for row in generator:
-            photo = self.get_cached_instance(objects.Photo, row)
-
-            if filename_tree and not filename_tree.evaluate(photo.basename.lower()):
-                continue
-
-            if tag_expression:
-                photo_tags = set(photo.get_tags())
-                success = tag_expression_tree.evaluate(
-                    photo_tags,
-                    match_function=tag_match_function,
-                )
-                if not success:
-                    continue
-
-            if offset > 0:
-                offset -= 1
-                continue
-
-            if limit is not None and results_received >= limit:
-                break
-
-            if yield_albums:
-                new_albums = photo.get_containing_albums().difference(seen_albums)
-                yield from new_albums
-                results_received += len(new_albums)
-                seen_albums.update(new_albums)
-
-            if yield_photos:
-                yield photo
-                results_received += 1
-
-        if warning_bag and warning_bag.warnings:
-            yield warning_bag
-
-        end_time = time.perf_counter()
-        log.debug('Search took %s.', end_time - start_time)
 
 ####################################################################################################
 

etiquette/searchhelpers.py

@@ -183,7 +183,7 @@ def normalize_filename(filename_terms):
     Returns: A string where terms are separated by spaces.
     '''
     if filename_terms is None:
-        filename_terms = ''
+        return None
 
     if not isinstance(filename_terms, str):
         filename_terms = ' '.join(filename_terms)

frontends/etiquette_cli.py

@@ -125,7 +125,7 @@ def search_in_cwd(**kwargs):
     return photodb.search(
         within_directory=cwd,
         **kwargs,
-    )
+    ).results
 
 def search_by_argparse(args, yield_albums=False, yield_photos=False):
     return search_in_cwd(

frontends/etiquette_flask/backend/endpoints/photo_endpoints.py

@@ -365,130 +365,98 @@ def post_batch_photos_download_zip():
 # Search ###########################################################################################
 
 def get_search_core():
-    warning_bag = etiquette.objects.WarningBag()
+    search = common.P.search(
+        area=request.args.get('area'),
+        width=request.args.get('width'),
+        height=request.args.get('height'),
+        aspectratio=request.args.get('aspectratio'),
+        bytes=request.args.get('bytes'),
+        duration=request.args.get('duration'),
+        bitrate=request.args.get('bitrate'),
 
-    has_tags = request.args.get('has_tags')
+        filename=request.args.get('filename'),
-    tag_musts = request.args.get('tag_musts')
+        extension_not=request.args.get('extension_not'),
-    tag_mays = request.args.get('tag_mays')
+        extension=request.args.get('extension'),
-    tag_forbids = request.args.get('tag_forbids')
+        mimetype=request.args.get('mimetype'),
-    tag_expression = request.args.get('tag_expression')
+        sha256=request.args.get('sha256'),
 
-    filename_terms = request.args.get('filename')
+        author=request.args.get('author'),
-    extension = request.args.get('extension')
+        created=request.args.get('created'),
-    extension_not = request.args.get('extension_not')
+        has_albums=request.args.get('has_albums'),
-    mimetype = request.args.get('mimetype')
+        has_thumbnail=request.args.get('has_thumbnail'),
-    sha256 = request.args.get('sha256')
+        is_searchhidden=request.args.get('is_searchhidden', False),
-    is_searchhidden = request.args.get('is_searchhidden', False)
-    yield_albums = request.args.get('yield_albums', False)
-    yield_photos = request.args.get('yield_photos', True)
 
-    limit = request.args.get('limit')
+        has_tags=request.args.get('has_tags'),
-    # This is being pre-processed because the site enforces a maximum value
+        tag_musts=request.args.get('tag_musts'),
-    # which the PhotoDB api does not.
+        tag_mays=request.args.get('tag_mays'),
-    limit = etiquette.searchhelpers.normalize_limit(limit, warning_bag=warning_bag)
+        tag_forbids=request.args.get('tag_forbids'),
+        tag_expression=request.args.get('tag_expression'),
 
-    if limit is None:
+        limit=request.args.get('limit'),
-        limit = 50
+        offset=request.args.get('offset'),
+        orderby=request.args.get('orderby'),
+
+        yield_albums=request.args.get('yield_albums', False),
+        yield_photos=request.args.get('yield_photos', True),
+    )
+
+    # The site enforces a maximum value which the PhotoDB does not.
+    search.kwargs.limit = etiquette.searchhelpers.normalize_limit(search.kwargs.limit)
+    if search.kwargs.limit is None:
+        search.kwargs.limit = 50
     else:
-        limit = min(limit, 1000)
+        search.kwargs.limit = min(search.kwargs.limit, 1000)
-
-    offset = request.args.get('offset')
-
-    author = request.args.get('author')
-
-    orderby = request.args.get('orderby')
-    area = request.args.get('area')
-    width = request.args.get('width')
-    height = request.args.get('height')
-    aspectratio = request.args.get('aspectratio')
-    bytes = request.args.get('bytes')
-    has_albums = request.args.get('has_albums')
-    has_thumbnail = request.args.get('has_thumbnail')
-    duration = request.args.get('duration')
-    bitrate = request.args.get('bitrate')
-    created = request.args.get('created')
-
-    # These are in a dictionary so I can pass them to the page template.
-    search_kwargs = {
-        'area': area,
-        'width': width,
-        'height': height,
-        'aspectratio': aspectratio,
-        'bytes': bytes,
-        'duration': duration,
-        'bitrate': bitrate,
-
-        'author': author,
-        'created': created,
-        'extension': extension,
-        'extension_not': extension_not,
-        'filename': filename_terms,
-        'has_albums': has_albums,
-        'has_tags': has_tags,
-        'has_thumbnail': has_thumbnail,
-        'is_searchhidden': is_searchhidden,
-        'mimetype': mimetype,
-        'sha256': sha256,
-        'tag_musts': tag_musts,
-        'tag_mays': tag_mays,
-        'tag_forbids': tag_forbids,
-        'tag_expression': tag_expression,
-
-        'limit': limit,
-        'offset': offset,
-        'orderby': orderby,
-
-        'warning_bag': warning_bag,
-        'give_back_parameters': True,
-
-        'yield_albums': yield_albums,
-        'yield_photos': yield_photos,
-    }
-    # print(search_kwargs)
-    search_generator = common.P.search(**search_kwargs)
-    # Because of the giveback, first element is cleaned up kwargs
-    search_kwargs = next(search_generator)
-    # Web UI users aren't allowed to use within_directory anyway, so don't
-    # show it to them.
-    search_kwargs.pop('within_directory', None)
-    # print(search_kwargs)
-
-    warnings = set()
-    search_results = []
-    for item in search_generator:
-        if isinstance(item, etiquette.objects.WarningBag):
-            warnings.update(item.warnings)
-            continue
-        search_results.append(item)
 
+    search.results = list(search.results)
     warnings = [
         w.error_message if hasattr(w, 'error_message') else str(w)
-        for w in warnings
+        for w in search.warning_bag.warnings
     ]
 
+    # Web UI users aren't allowed to use within_directory anyway, so don't
+    # show it to them.
+    del search.kwargs.within_directory
+    return search
+
+@site.route('/search_embed')
+def get_search_embed():
+    search = get_search_core()
+    response = common.render_template(
+        request,
+        'search_embed.html',
+        results=search.results,
+        search_kwargs=search.kwargs,
+    )
+    return response
+
+@site.route('/search')
+def get_search_html():
+    search = get_search_core()
+    search.kwargs.view = request.args.get('view', 'grid')
+
     # TAGS ON THIS PAGE
     total_tags = set()
-    for result in search_results:
+    for result in search.results:
         if isinstance(result, etiquette.objects.Photo):
             total_tags.update(result.get_tags())
     total_tags = sorted(total_tags, key=lambda t: t.name)
 
     # PREV-NEXT PAGE URLS
-    offset = search_kwargs['offset'] or 0
+    offset = search.kwargs.offset or 0
     original_params = request.args.to_dict()
-    original_params['limit'] = limit
+    original_params['limit'] = search.kwargs.limit
 
-    if limit and len(search_results) >= limit:
+    if search.more_after_limit:
         next_params = original_params.copy()
-        next_params['offset'] = offset + limit
+        next_params['offset'] = offset + search.kwargs.limit
         next_params = helpers.dict_to_params(next_params)
         next_page_url = '/search' + next_params
     else:
         next_page_url = None
 
-    if limit and offset > 0:
+    if search.kwargs.limit and offset > 0:
         prev_params = original_params.copy()
-        prev_offset = max(0, offset - limit)
+        prev_offset = max(0, offset - search.kwargs.limit)
         if prev_offset > 0:
             prev_params['offset'] = prev_offset
         else:
@@ -498,49 +466,23 @@ def get_search_core():
     else:
         prev_page_url = None
 
-    search_kwargs['view'] = request.args.get('view', 'grid')
-
-    final_results = {
-        'next_page_url': next_page_url,
-        'prev_page_url': prev_page_url,
-        'results': search_results,
-        'total_tags': total_tags,
-        'warnings': list(warnings),
-        'search_kwargs': search_kwargs,
-    }
-    return final_results
-
-@site.route('/search_embed')
-def get_search_embed():
-    search_results = get_search_core()
-    response = common.render_template(
-        request,
-        'search_embed.html',
-        results=search_results['results'],
-        search_kwargs=search_results['search_kwargs'],
-    )
-    return response
-
-@site.route('/search')
-def get_search_html():
-    search_results = get_search_core()
     response = common.render_template(
         request,
         'search.html',
-        next_page_url=search_results['next_page_url'],
+        next_page_url=next_page_url,
-        prev_page_url=search_results['prev_page_url'],
+        prev_page_url=prev_page_url,
-        results=search_results['results'],
+        results=search.results,
-        search_kwargs=search_results['search_kwargs'],
+        search_kwargs=search.kwargs,
-        total_tags=search_results['total_tags'],
+        total_tags=total_tags,
-        warnings=search_results['warnings'],
+        warnings=search.warning_bag.jsonify(),
     )
     return response
 
 @site.route('/search.atom')
 def get_search_atom():
-    search_results = get_search_core()['results']
+    search = get_search_core()
     soup = etiquette.helpers.make_atom_feed(
-        search_results,
+        search.results,
         feed_id=request.query_string.decode('utf-8'),
         feed_title='etiquette search',
         feed_link=request.url.replace('/search.atom', '/search'),
@@ -553,34 +495,9 @@ def get_search_atom():
 
 @site.route('/search.json')
 def get_search_json():
-    search_results = get_search_core()
+    search = get_search_core()
-    search_kwargs = search_results['search_kwargs']
+    response = search.jsonify()
-
+    return flasktools.json_response(response)
-    # The search has converted many arguments into sets or other types.
-    # Convert them back into something that will display nicely on the search form.
-    join_helper = lambda x: ', '.join(x) if x else None
-    search_kwargs['extension'] = join_helper(search_kwargs['extension'])
-    search_kwargs['extension_not'] = join_helper(search_kwargs['extension_not'])
-    search_kwargs['mimetype'] = join_helper(search_kwargs['mimetype'])
-
-    author_helper = lambda users: ', '.join(user.username for user in users) if users else None
-    search_kwargs['author'] = author_helper(search_kwargs['author'])
-
-    tagname_helper = lambda tags: [tag.name for tag in tags] if tags else None
-    search_kwargs['tag_musts'] = tagname_helper(search_kwargs['tag_musts'])
-    search_kwargs['tag_mays'] = tagname_helper(search_kwargs['tag_mays'])
-    search_kwargs['tag_forbids'] = tagname_helper(search_kwargs['tag_forbids'])
-
-    search_results['results'] = [
-        result.jsonify(include_albums=False)
-        if isinstance(result, etiquette.objects.Photo) else
-        result.jsonify(minimal=True)
-        for result in search_results['results']
-    ]
-    search_results['total_tags'] = [
-        tag.jsonify(minimal=True) for tag in search_results['total_tags']
-    ]
-    return flasktools.json_response(search_results)
 
 # Swipe ############################################################################################