voussoir/etiquette
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Move search to an entire class of its own.

Browse source 
The initial motivation for this was to make the "more_after_limit"
feature, which would help the UI to not show a next page button when
the number of results was exactly equal to the limit.

However, in order to surface this more_after_limit status using only
the old search generator, it would have to be a special yield at the
end. I was getting tired of the special yields like give_back_params
at the beginning and warning_bag at the end, and this would be worse.

There is a lot of sideband information about the search that is now
more easily accessible when the search is its own object.
This commit is contained in:
voussoir 2022-11-06 22:58:36 -08:00
parent 9f8dd057f0
commit b64901105c
5 changed files with 515 additions and 603 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

436
etiquette/objects.py
View file

@ -11,10 +11,13 @@ import os
import PIL.Image
import re
import send2trash
import time
import traceback
import typing
from voussoirkit import bytestring
from voussoirkit import dotdict
from voussoirkit import expressionmatch
from voussoirkit import gentools
from voussoirkit import hms
from voussoirkit import pathclass
@ -32,6 +35,7 @@ from . import constants
from . import decorators
from . import exceptions
from . import helpers
from . import searchhelpers
BAIL = sentinel.Sentinel('BAIL')
@ -1521,6 +1525,431 @@ class Photo(ObjectBase):
self._tagged_at_dt = helpers.utcfromtimestamp(self.tagged_at_unix)
return self._tagged_at_dt
class Search:
'''
FILE METADATA
=============
area, aspectratio, width, height, bytes, duration, bitrate:
A dotdot_range string representing min and max. Or just a number
for lower bound.
extension:
A string or list of strings of acceptable file extensions.
extension_not:
A string or list of strings of unacceptable file extensions.
Including '*' will forbid all extensions, thus returning only
extensionless files.
filename:
A string or list of strings in the form of an expression.
Match is CASE-INSENSITIVE.
Examples:
'.pdf AND (programming OR "survival guide")'
'.pdf programming python' (implicitly AND each term)
sha256:
A string or list of strings of exact SHA256 hashes to match.
within_directory:
A string or list of strings or pathclass Paths of directories.
Photos MUST have a `filepath` that is a child of one of these
directories.
OBJECT METADATA
===============
author:
A list of User objects or usernames, or a string of comma-separated
usernames.
created:
A dotdot_range string respresenting min and max. Or just a number
for lower bound.
has_albums:
If True, require that the Photo belongs to >=1 album.
If False, require that the Photo belongs to no albums.
If None, either is okay.
has_tags:
If True, require that the Photo has >=1 tag.
If False, require that the Photo has no tags.
If None, any amount is okay.
has_thumbnail:
Require a thumbnail?
If None, anything is okay.
is_searchhidden:
If True, find *only* searchhidden photos.
If False, find *only* nonhidden photos.
If None, either is okay.
mimetype:
A string or list of strings of acceptable mimetypes.
'image', 'video', ...
Note we are only interested in the simple "video", "audio" etc.
For exact mimetypes you might as well use an extension search.
TAGS
====
tag_musts:
A list of tag names or Tag objects.
Photos MUST have ALL tags in this list.
tag_mays:
A list of tag names or Tag objects.
Photos MUST have AT LEAST ONE tag in this list.
tag_forbids:
A list of tag names or Tag objects.
Photos MUST NOT have ANY tag in the list.
tag_expression:
A string or list of strings in the form of an expression.
Can NOT be used with the must, may, forbid style search.
Examples:
'family AND (animals OR vacation)'
'family vacation outdoors' (implicitly AND each term)
QUERY OPTIONS
=============
limit:
The maximum number of *successful* results to yield.
offset:
How many *successful* results to skip before we start yielding.
orderby:
A list of strings like ['aspectratio DESC', 'created ASC'] to sort
and subsort the results.
Descending is assumed if not provided.
yield_albums:
If True, albums which contain photos matching the search
will be yielded.
yield_photos:
If True, photos matching the search will be yielded.
'''
def __init__(self, photodb, kwargs, *, raise_errors=True):
self.photodb = photodb
self.created = timetools.now()
self.raise_errors = raise_errors
if isinstance(kwargs, dict):
kwargs = dotdict.DotDict(kwargs, default=None)
self.kwargs = kwargs
self.generator_started = False
self.generator_exhausted = False
self.more_after_limit = None
self.start_time = None
self.end_time = None
self.start_commit_id = None
self.warning_bag = WarningBag()
self.results = self._generator()
self.results_received = 0
def atomify(self):
raise NotImplementedError
def jsonify(self):
# The search has converted many arguments into sets or other types.
# Convert them back into something that will display nicely on the search form.
kwargs = self.kwargs._to_dict()
join_helper = lambda x: ', '.join(x) if x else None
kwargs['extension'] = join_helper(kwargs['extension'])
kwargs['extension_not'] = join_helper(kwargs['extension_not'])
kwargs['mimetype'] = join_helper(kwargs['mimetype'])
kwargs['sha256'] = join_helper(kwargs['sha256'])
author_helper = lambda users: ', '.join(user.username for user in users) if users else None
kwargs['author'] = author_helper(kwargs['author'])
tagname_helper = lambda tags: [tag.name for tag in tags] if tags else None
kwargs['tag_musts'] = tagname_helper(kwargs['tag_musts'])
kwargs['tag_mays'] = tagname_helper(kwargs['tag_mays'])
kwargs['tag_forbids'] = tagname_helper(kwargs['tag_forbids'])
results = [
result.jsonify(include_albums=False)
if isinstance(result, Photo) else
result.jsonify(minimal=True)
for result in self.results
]
j = {
'kwargs': kwargs,
'results': results,
'more_after_limit': self.more_after_limit,
}
return j
def _generator(self):
self.start_time = time.perf_counter()
self.generator_started = True
self.start_commit_id = self.photodb.last_commit_id
kwargs = self.kwargs
maximums = {}
minimums = {}
searchhelpers.minmax('area', kwargs.area, minimums, maximums, warning_bag=self.warning_bag)
searchhelpers.minmax('created', kwargs.created, minimums, maximums, warning_bag=self.warning_bag)
searchhelpers.minmax('width', kwargs.width, minimums, maximums, warning_bag=self.warning_bag)
searchhelpers.minmax('height', kwargs.height, minimums, maximums, warning_bag=self.warning_bag)
searchhelpers.minmax('aspectratio', kwargs.aspectratio, minimums, maximums, warning_bag=self.warning_bag)
searchhelpers.minmax('bytes', kwargs.bytes, minimums, maximums, warning_bag=self.warning_bag)
searchhelpers.minmax('duration', kwargs.duration, minimums, maximums, warning_bag=self.warning_bag)
searchhelpers.minmax('bitrate', kwargs.bitrate, minimums, maximums, warning_bag=self.warning_bag)
kwargs.author = searchhelpers.normalize_author(kwargs.author, photodb=self.photodb, warning_bag=self.warning_bag)
kwargs.extension = searchhelpers.normalize_extension(kwargs.extension)
kwargs.extension_not = searchhelpers.normalize_extension(kwargs.extension_not)
kwargs.filename = searchhelpers.normalize_filename(kwargs.filename)
kwargs.has_albums = searchhelpers.normalize_has_tags(kwargs.has_albums)
kwargs.has_tags = searchhelpers.normalize_has_tags(kwargs.has_tags)
kwargs.has_thumbnail = searchhelpers.normalize_has_thumbnail(kwargs.has_thumbnail)
kwargs.is_searchhidden = searchhelpers.normalize_is_searchhidden(kwargs.is_searchhidden)
kwargs.sha256 = searchhelpers.normalize_sha256(kwargs.sha256)
kwargs.mimetype = searchhelpers.normalize_extension(kwargs.mimetype)
kwargs.sha256 = searchhelpers.normalize_extension(kwargs.sha256)
kwargs.within_directory = searchhelpers.normalize_within_directory(kwargs.within_directory, warning_bag=self.warning_bag)
kwargs.yield_albums = searchhelpers.normalize_yield_albums(kwargs.yield_albums)
kwargs.yield_photos = searchhelpers.normalize_yield_photos(kwargs.yield_photos)
if kwargs.has_tags is False:
if (kwargs.tag_musts or kwargs.tag_mays or kwargs.tag_forbids or kwargs.tag_expression):
self.warning_bag.add("has_tags=False so all tag requests are ignored.")
kwargs.tag_musts = None
kwargs.tag_mays = None
kwargs.tag_forbids = None
kwargs.tag_expression = None
else:
kwargs.tag_musts = searchhelpers.normalize_tagset(self.photodb, kwargs.tag_musts, warning_bag=self.warning_bag)
kwargs.tag_mays = searchhelpers.normalize_tagset(self.photodb, kwargs.tag_mays, warning_bag=self.warning_bag)
kwargs.tag_forbids = searchhelpers.normalize_tagset(self.photodb, kwargs.tag_forbids, warning_bag=self.warning_bag)
kwargs.tag_expression = searchhelpers.normalize_tag_expression(kwargs.tag_expression)
if kwargs.extension is not None and kwargs.extension_not is not None:
kwargs.extension = kwargs.extension.difference(kwargs.extension_not)
tags_fixed = searchhelpers.normalize_mmf_vs_expression_conflict(
kwargs.tag_musts,
kwargs.tag_mays,
kwargs.tag_forbids,
kwargs.tag_expression,
self.warning_bag,
)
(kwargs.tag_musts, kwargs.tag_mays, kwargs.tag_forbids, kwargs.tag_expression) = tags_fixed
if kwargs.tag_expression:
tag_expression_tree = searchhelpers.tag_expression_tree_builder(
tag_expression=kwargs.tag_expression,
photodb=self.photodb,
warning_bag=self.warning_bag,
)
if tag_expression_tree is None:
kwargs.tag_expression = None
kwargs.tag_expression = None
else:
kwargs.tag_expression = str(tag_expression_tree)
frozen_children = self.photodb.get_cached_tag_export('flat_dict', tags=self.get_root_tags())
tag_match_function = searchhelpers.tag_expression_matcher_builder(frozen_children)
else:
tag_expression_tree = None
kwargs.tag_expression = None
if kwargs.has_tags is True and (kwargs.tag_musts or kwargs.tag_mays):
# has_tags check is redundant then, so disable it.
kwargs.has_tags = None
kwargs.limit = searchhelpers.normalize_limit(kwargs.limit, warning_bag=self.warning_bag)
kwargs.offset = searchhelpers.normalize_offset(kwargs.offset, warning_bag=self.warning_bag)
kwargs.orderby = searchhelpers.normalize_orderby(kwargs.orderby, warning_bag=self.warning_bag)
if kwargs.filename:
try:
filename_tree = expressionmatch.ExpressionTree.parse(kwargs.filename)
filename_tree.map(lambda x: x.lower())
except expressionmatch.NoTokens:
filename_tree = None
else:
filename_tree = None
if kwargs.orderby:
orderby = [(expanded, direction) for (friendly, expanded, direction) in kwargs.orderby]
kwargs.orderby = [
f'{friendly}-{direction}'
for (friendly, expanded, direction) in kwargs.orderby
]
else:
orderby = [('created', 'desc')]
kwargs.orderby = None
if not kwargs.yield_albums and not kwargs.yield_photos:
exc = exceptions.NoYields(['yield_albums', 'yield_photos'])
self.warning_bag.add(exc)
if self.raise_errors:
raise exceptions.NoYields(['yield_albums', 'yield_photos'])
else:
return
photo_tag_rel_exist_clauses = searchhelpers.photo_tag_rel_exist_clauses(
kwargs.tag_musts,
kwargs.tag_mays,
kwargs.tag_forbids,
)
notnulls = set()
yesnulls = set()
wheres = []
bindings = []
if photo_tag_rel_exist_clauses:
wheres.extend(photo_tag_rel_exist_clauses)
if kwargs.author:
author_ids = [user.id for user in kwargs.author]
wheres.append(f'author_id IN {sqlhelpers.listify(author_ids)}')
if kwargs.extension:
if '*' in kwargs.extension:
wheres.append('extension != ""')
else:
qmarks = ', '.join('?' * len(kwargs.extension))
wheres.append(f'extension IN ({qmarks})')
bindings.extend(kwargs.extension)
if kwargs.extension_not:
if '*' in kwargs.extension_not:
wheres.append('extension == ""')
else:
qmarks = ', '.join('?' * len(kwargs.extension_not))
wheres.append(f'extension NOT IN ({qmarks})')
bindings.extend(kwargs.extension_not)
if kwargs.mimetype:
extensions = {
extension
for (extension, (typ, subtyp)) in constants.MIMETYPES.items()
if typ in kwargs.mimetype
}
wheres.append(f'extension IN {sqlhelpers.listify(extensions)} COLLATE NOCASE')
if kwargs.within_directory:
patterns = {d.absolute_path.rstrip(os.sep) for d in kwargs.within_directory}
patterns = {f'{d}{os.sep}%' for d in patterns}
clauses = ['filepath LIKE ?'] * len(patterns)
if len(clauses) > 1:
clauses = ' OR '.join(clauses)
clauses = f'({clauses})'
else:
clauses = clauses.pop()
wheres.append(clauses)
bindings.extend(patterns)
if kwargs.has_albums is True or (kwargs.yield_albums and not kwargs.yield_photos):
wheres.append('EXISTS (SELECT 1 FROM album_photo_rel WHERE photoid == photos.id)')
elif kwargs.has_albums is False:
wheres.append('NOT EXISTS (SELECT 1 FROM album_photo_rel WHERE photoid == photos.id)')
if kwargs.has_tags is True:
wheres.append('EXISTS (SELECT 1 FROM photo_tag_rel WHERE photoid == photos.id)')
elif kwargs.has_tags is False:
wheres.append('NOT EXISTS (SELECT 1 FROM photo_tag_rel WHERE photoid == photos.id)')
if kwargs.has_thumbnail is True:
notnulls.add('thumbnail')
elif kwargs.has_thumbnail is False:
yesnulls.add('thumbnail')
for (column, direction) in orderby:
if column != 'RANDOM()':
notnulls.add(column)
if kwargs.is_searchhidden is True:
wheres.append('searchhidden == 1')
elif kwargs.is_searchhidden is False:
wheres.append('searchhidden == 0')
if kwargs.sha256:
wheres.append(f'sha256 IN {sqlhelpers.listify(kwargs.sha256)}')
for column in notnulls:
wheres.append(column + ' IS NOT NULL')
for column in yesnulls:
wheres.append(column + ' IS NULL')
for (column, value) in minimums.items():
wheres.append(column + ' >= ' + str(value))
for (column, value) in maximums.items():
wheres.append(column + ' <= ' + str(value))
query = ['SELECT * FROM photos']
if wheres:
wheres = 'WHERE ' + ' AND '.join(wheres)
query.append(wheres)
if orderby:
orderby = [f'{column} {direction}' for (column, direction) in orderby]
orderby = ', '.join(orderby)
orderby = 'ORDER BY ' + orderby
query.append(orderby)
query = ' '.join(query)
query = f'{"-" * 80}\n{query}\n{"-" * 80}'
log.debug('\n%s %s', query, bindings)
# print(self.photodb.explain(query, bindings))
generator = self.photodb.select(query, bindings)
seen_albums = set()
offset = kwargs.offset
for row in generator:
photo = self.photodb.get_cached_instance(Photo, row)
if filename_tree and not filename_tree.evaluate(photo.basename.lower()):
continue
if tag_expression_tree:
photo_tags = set(photo.get_tags())
success = tag_expression_tree.evaluate(
photo_tags,
match_function=tag_match_function,
)
if not success:
continue
if offset > 0:
offset -= 1
continue
if kwargs.yield_albums:
new_albums = photo.get_containing_albums().difference(seen_albums)
yield from new_albums
self.results_received += len(new_albums)
seen_albums.update(new_albums)
if kwargs.yield_photos:
yield photo
self.results_received += 1
if kwargs.limit is not None and self.results_received >= kwargs.limit:
break
try:
next(generator)
except StopIteration:
self.more_after_limit = False
else:
self.more_after_limit = True
self.generator_exhausted = True
self.end_time = time.perf_counter()
log.debug('Search took %s.', self.end_time - self.start_time)
class Tag(ObjectBase, GroupableMixin):
'''
A Tag, which can be applied to Photos for organization.
@ -1619,8 +2048,8 @@ class Tag(ObjectBase, GroupableMixin):
# only has A because some other photo with A and B thinks A is obsolete.
# This technique is nice and simple to understand for now.
ancestors = list(member.walk_parents())
photos = self.photodb.search(tag_musts=[member], is_searchhidden=None, yield_albums=False)
for photo in photos:
photos = self.photodb.search(tag_musts=[member], is_searchhidden=None, yield_photos=True, yield_albums=False)
for photo in photos.results:
photo.remove_tags(ancestors)
@decorators.required_feature('tag.edit')
@ -2112,3 +2541,6 @@ class WarningBag:
def add(self, warning) -> None:
self.warnings.add(warning)
def jsonify(self):
return [getattr(w, 'error_message', str(w)) for w in self.warnings]

441
etiquette/photodb.py
View file

@ -3,9 +3,7 @@ import hashlib
import json
import os
import random
import sqlite3
import tempfile
import time
import types
import typing
@ -14,12 +12,10 @@ from . import decorators
from . import exceptions
from . import helpers
from . import objects
from . import searchhelpers
from . import tag_export
from voussoirkit import cacheclass
from voussoirkit import configlayers
from voussoirkit import expressionmatch
from voussoirkit import pathclass
from voussoirkit import ratelimiter
from voussoirkit import spinal
@ -411,441 +407,8 @@ class PDBPhotoMixin:
photo.delete()
yield photo
def search(
self,
*,
area=None,
aspectratio=None,
width=None,
height=None,
bytes=None,
duration=None,
bitrate=None,
author=None,
created=None,
extension=None,
extension_not=None,
filename=None,
has_albums=None,
has_tags=None,
has_thumbnail=None,
is_searchhidden=False,
mimetype=None,
sha256=None,
tag_musts=None,
tag_mays=None,
tag_forbids=None,
tag_expression=None,
within_directory=None,
limit=None,
offset=None,
orderby=None,
warning_bag=None,
give_back_parameters=False,
yield_albums=False,
yield_photos=True,
):
'''
PHOTO PROPERTIES
area, aspectratio, width, height, bytes, duration, bitrate:
A dotdot_range string representing min and max. Or just a number
for lower bound.
TAGS AND FILTERS
author:
A list of User objects or usernames, or a string of comma-separated
usernames.
created:
A dotdot_range string respresenting min and max. Or just a number
for lower bound.
extension:
A string or list of strings of acceptable file extensions.
extension_not:
A string or list of strings of unacceptable file extensions.
Including '*' will forbid all extensions
filename:
A string or list of strings in the form of an expression.
Match is CASE-INSENSITIVE.
Examples:
'.pdf AND (programming OR "survival guide")'
'.pdf programming python' (implicitly AND each term)
has_albums:
If True, require that the Photo belongs to >=1 album.
If False, require that the Photo belongs to no albums.
If None, either is okay.
has_tags:
If True, require that the Photo has >=1 tag.
If False, require that the Photo has no tags.
If None, any amount is okay.
has_thumbnail:
Require a thumbnail?
If None, anything is okay.
is_searchhidden:
Find photos that are marked as searchhidden?
If True, find *only* searchhidden photos.
If False, find *only* nonhidden photos.
If None, either is okay.
Default False.
mimetype:
A string or list of strings of acceptable mimetypes.
'image', 'video', ...
Note we are only interested in the simple "video", "audio" etc.
For exact mimetypes you might as well use an extension search.
tag_musts:
A list of tag names or Tag objects.
Photos MUST have ALL tags in this list.
tag_mays:
A list of tag names or Tag objects.
Photos MUST have AT LEAST ONE tag in this list.
tag_forbids:
A list of tag names or Tag objects.
Photos MUST NOT have ANY tag in the list.
tag_expression:
A string or list of strings in the form of an expression.
Can NOT be used with the must, may, forbid style search.
Examples:
'family AND (animals OR vacation)'
'family vacation outdoors' (implicitly AND each term)
within_directory:
A string or list of strings or pathclass Paths of directories.
Photos MUST have a `filepath` that is a child of one of these
directories.
QUERY OPTIONS
limit:
The maximum number of *successful* results to yield.
offset:
How many *successful* results to skip before we start yielding.
orderby:
A list of strings like ['aspectratio DESC', 'created ASC'] to sort
and subsort the results.
Descending is assumed if not provided.
warning_bag:
If provided, invalid search queries will add a warning to the bag
and try their best to continue. The generator will yield the bag
back to you as the final object.
Without the bag, exceptions may be raised.
YIELD OPTIONS
give_back_parameters:
If True, the generator's first yield will be a dictionary of all the
cleaned up, normalized parameters. The user may have given us loads
of trash, so we should show them the formatting we want.
yield_albums:
If True, albums which contain photos matching the search
will be yielded.
yield_photos:
If True, photos matching the search will be yielded.
'''
start_time = time.perf_counter()
maximums = {}
minimums = {}
searchhelpers.minmax('area', area, minimums, maximums, warning_bag=warning_bag)
searchhelpers.minmax('created', created, minimums, maximums, warning_bag=warning_bag)
searchhelpers.minmax('width', width, minimums, maximums, warning_bag=warning_bag)
searchhelpers.minmax('height', height, minimums, maximums, warning_bag=warning_bag)
searchhelpers.minmax('aspectratio', aspectratio, minimums, maximums, warning_bag=warning_bag)
searchhelpers.minmax('bytes', bytes, minimums, maximums, warning_bag=warning_bag)
searchhelpers.minmax('duration', duration, minimums, maximums, warning_bag=warning_bag)
searchhelpers.minmax('bitrate', bitrate, minimums, maximums, warning_bag=warning_bag)
author = searchhelpers.normalize_author(author, photodb=self, warning_bag=warning_bag)
extension = searchhelpers.normalize_extension(extension)
extension_not = searchhelpers.normalize_extension(extension_not)
filename = searchhelpers.normalize_filename(filename)
has_albums = searchhelpers.normalize_has_tags(has_albums)
has_tags = searchhelpers.normalize_has_tags(has_tags)
has_thumbnail = searchhelpers.normalize_has_thumbnail(has_thumbnail)
is_searchhidden = searchhelpers.normalize_is_searchhidden(is_searchhidden)
sha256 = searchhelpers.normalize_sha256(sha256)
mimetype = searchhelpers.normalize_extension(mimetype)
sha256 = searchhelpers.normalize_extension(sha256)
within_directory = searchhelpers.normalize_within_directory(within_directory, warning_bag=warning_bag)
yield_albums = searchhelpers.normalize_yield_albums(yield_albums)
yield_photos = searchhelpers.normalize_yield_photos(yield_photos)
if has_tags is False:
if (tag_musts or tag_mays or tag_forbids or tag_expression) and warning_bag:
warning_bag.add("has_tags=False so all tag requests are ignored.")
tag_musts = None
tag_mays = None
tag_forbids = None
tag_expression = None
else:
tag_musts = searchhelpers.normalize_tagset(self, tag_musts, warning_bag=warning_bag)
tag_mays = searchhelpers.normalize_tagset(self, tag_mays, warning_bag=warning_bag)
tag_forbids = searchhelpers.normalize_tagset(self, tag_forbids, warning_bag=warning_bag)
tag_expression = searchhelpers.normalize_tag_expression(tag_expression)
if extension is not None and extension_not is not None:
extension = extension.difference(extension_not)
tags_fixed = searchhelpers.normalize_mmf_vs_expression_conflict(
tag_musts,
tag_mays,
tag_forbids,
tag_expression,
warning_bag,
)
(tag_musts, tag_mays, tag_forbids, tag_expression) = tags_fixed
if tag_expression:
tag_expression_tree = searchhelpers.tag_expression_tree_builder(
tag_expression=tag_expression,
photodb=self,
warning_bag=warning_bag,
)
if tag_expression_tree is None:
giveback_tag_expression = None
tag_expression = None
else:
giveback_tag_expression = str(tag_expression_tree)
frozen_children = self.get_cached_tag_export('flat_dict', tags=self.get_root_tags())
tag_match_function = searchhelpers.tag_expression_matcher_builder(frozen_children)
else:
giveback_tag_expression = None
if has_tags is True and (tag_musts or tag_mays):
# has_tags check is redundant then, so disable it.
has_tags = None
limit = searchhelpers.normalize_limit(limit, warning_bag=warning_bag)
offset = searchhelpers.normalize_offset(offset, warning_bag=warning_bag)
orderby = searchhelpers.normalize_orderby(orderby, warning_bag=warning_bag)
if filename:
try:
filename_tree = expressionmatch.ExpressionTree.parse(filename)
filename_tree.map(lambda x: x.lower())
except expressionmatch.NoTokens:
filename_tree = None
else:
filename_tree = None
if orderby:
giveback_orderby = [
f'{friendly}-{direction}'
for (friendly, expanded, direction) in orderby
]
orderby = [(expanded, direction) for (friendly, expanded, direction) in orderby]
else:
giveback_orderby = None
orderby = [('created', 'desc')]
if give_back_parameters:
parameters = {
'area': area,
'width': width,
'height': height,
'aspectratio': aspectratio,
'bitrate': bitrate,
'bytes': bytes,
'duration': duration,
'author': list(author) or None,
'created': created,
'extension': list(extension) or None,
'extension_not': list(extension_not) or None,
'filename': filename or None,
'has_albums': has_albums,
'has_tags': has_tags,
'has_thumbnail': has_thumbnail,
'mimetype': list(mimetype) or None,
'sha256': list(sha256) or None,
'tag_musts': tag_musts or None,
'tag_mays': tag_mays or None,
'tag_forbids': tag_forbids or None,
'tag_expression': giveback_tag_expression or None,
'within_directory': within_directory or None,
'limit': limit,
'offset': offset or None,
'orderby': giveback_orderby,
'yield_albums': yield_albums,
'yield_photos': yield_photos,
}
yield parameters
if not yield_albums and not yield_photos:
exc = exceptions.NoYields(['yield_albums', 'yield_photos'])
if warning_bag:
warning_bag.add(exc)
yield warning_bag
return
else:
raise exceptions.NoYields(['yield_albums', 'yield_photos'])
photo_tag_rel_exist_clauses = searchhelpers.photo_tag_rel_exist_clauses(
tag_musts,
tag_mays,
tag_forbids,
)
notnulls = set()
yesnulls = set()
wheres = []
bindings = []
if author:
author_ids = [user.id for user in author]
wheres.append(f'author_id IN {sqlhelpers.listify(author_ids)}')
if extension:
if '*' in extension:
wheres.append('extension != ""')
else:
qmarks = ', '.join('?' * len(extension))
wheres.append(f'extension IN ({qmarks})')
bindings.extend(extension)
if extension_not:
if '*' in extension_not:
wheres.append('extension == ""')
else:
qmarks = ', '.join('?' * len(extension_not))
wheres.append(f'extension NOT IN ({qmarks})')
bindings.extend(extension_not)
if mimetype:
extensions = {extension for (extension, (typ, subtyp)) in constants.MIMETYPES.items() if typ in mimetype}
wheres.append(f'extension IN {sqlhelpers.listify(extensions)} COLLATE NOCASE')
if within_directory:
patterns = {d.absolute_path.rstrip(os.sep) for d in within_directory}
patterns = {f'{d}{os.sep}%' for d in patterns}
clauses = ['filepath LIKE ?'] * len(patterns)
if len(clauses) > 1:
clauses = ' OR '.join(clauses)
clauses = f'({clauses})'
else:
clauses = clauses.pop()
wheres.append(clauses)
bindings.extend(patterns)
if has_albums is True:
wheres.append('EXISTS (SELECT 1 FROM album_photo_rel WHERE photoid == photos.id)')
elif has_albums is False:
wheres.append('NOT EXISTS (SELECT 1 FROM album_photo_rel WHERE photoid == photos.id)')
if has_tags is True:
wheres.append('EXISTS (SELECT 1 FROM photo_tag_rel WHERE photoid == photos.id)')
elif has_tags is False:
wheres.append('NOT EXISTS (SELECT 1 FROM photo_tag_rel WHERE photoid == photos.id)')
if yield_albums and not yield_photos:
wheres.append('EXISTS (SELECT 1 FROM album_photo_rel WHERE photoid == photos.id)')
if has_thumbnail is True:
notnulls.add('thumbnail')
elif has_thumbnail is False:
yesnulls.add('thumbnail')
for (column, direction) in orderby:
if column != 'RANDOM()':
notnulls.add(column)
if is_searchhidden is True:
wheres.append('searchhidden == 1')
elif is_searchhidden is False:
wheres.append('searchhidden == 0')
if sha256:
wheres.append(f'sha256 IN {sqlhelpers.listify(sha256)}')
for column in notnulls:
wheres.append(column + ' IS NOT NULL')
for column in yesnulls:
wheres.append(column + ' IS NULL')
for (column, value) in minimums.items():
wheres.append(column + ' >= ' + str(value))
for (column, value) in maximums.items():
wheres.append(column + ' <= ' + str(value))
if photo_tag_rel_exist_clauses:
wheres.extend(photo_tag_rel_exist_clauses)
query = ['SELECT * FROM photos']
if wheres:
wheres = 'WHERE ' + ' AND '.join(wheres)
query.append(wheres)
if orderby:
orderby = [f'{column} {direction}' for (column, direction) in orderby]
orderby = ', '.join(orderby)
orderby = 'ORDER BY ' + orderby
query.append(orderby)
query = ' '.join(query)
query = f'{"-" * 80}\n{query}\n{"-" * 80}'
log.debug('\n%s %s', query, bindings)
# explain = self.execute('EXPLAIN QUERY PLAN ' + query, bindings)
# print('\n'.join(str(x) for x in explain.fetchall()))
generator = self.select(query, bindings)
seen_albums = set()
results_received = 0
for row in generator:
photo = self.get_cached_instance(objects.Photo, row)
if filename_tree and not filename_tree.evaluate(photo.basename.lower()):
continue
if tag_expression:
photo_tags = set(photo.get_tags())
success = tag_expression_tree.evaluate(
photo_tags,
match_function=tag_match_function,
)
if not success:
continue
if offset > 0:
offset -= 1
continue
if limit is not None and results_received >= limit:
break
if yield_albums:
new_albums = photo.get_containing_albums().difference(seen_albums)
yield from new_albums
results_received += len(new_albums)
seen_albums.update(new_albums)
if yield_photos:
yield photo
results_received += 1
if warning_bag and warning_bag.warnings:
yield warning_bag
end_time = time.perf_counter()
log.debug('Search took %s.', end_time - start_time)
def search(self, **kwargs):
return objects.Search(photodb=self, kwargs=kwargs)
####################################################################################################

2
etiquette/searchhelpers.py
View file

@ -183,7 +183,7 @@ def normalize_filename(filename_terms):
Returns: A string where terms are separated by spaces.
'''
if filename_terms is None:
filename_terms = ''
return None
if not isinstance(filename_terms, str):
filename_terms = ' '.join(filename_terms)

2
frontends/etiquette_cli.py
View file

@ -125,7 +125,7 @@ def search_in_cwd(**kwargs):
return photodb.search(
within_directory=cwd,
**kwargs,
)
).results
def search_by_argparse(args, yield_albums=False, yield_photos=False):
return search_in_cwd(

237
frontends/etiquette_flask/backend/endpoints/photo_endpoints.py
View file

@ -365,130 +365,98 @@ def post_batch_photos_download_zip():
# Search ###########################################################################################
def get_search_core():
warning_bag = etiquette.objects.WarningBag()
search = common.P.search(
area=request.args.get('area'),
width=request.args.get('width'),
height=request.args.get('height'),
aspectratio=request.args.get('aspectratio'),
bytes=request.args.get('bytes'),
duration=request.args.get('duration'),
bitrate=request.args.get('bitrate'),
has_tags = request.args.get('has_tags')
tag_musts = request.args.get('tag_musts')
tag_mays = request.args.get('tag_mays')
tag_forbids = request.args.get('tag_forbids')
tag_expression = request.args.get('tag_expression')
filename=request.args.get('filename'),
extension_not=request.args.get('extension_not'),
extension=request.args.get('extension'),
mimetype=request.args.get('mimetype'),
sha256=request.args.get('sha256'),
filename_terms = request.args.get('filename')
extension = request.args.get('extension')
extension_not = request.args.get('extension_not')
mimetype = request.args.get('mimetype')
sha256 = request.args.get('sha256')
is_searchhidden = request.args.get('is_searchhidden', False)
yield_albums = request.args.get('yield_albums', False)
yield_photos = request.args.get('yield_photos', True)
author=request.args.get('author'),
created=request.args.get('created'),
has_albums=request.args.get('has_albums'),
has_thumbnail=request.args.get('has_thumbnail'),
is_searchhidden=request.args.get('is_searchhidden', False),
limit = request.args.get('limit')
# This is being pre-processed because the site enforces a maximum value
# which the PhotoDB api does not.
limit = etiquette.searchhelpers.normalize_limit(limit, warning_bag=warning_bag)
has_tags=request.args.get('has_tags'),
tag_musts=request.args.get('tag_musts'),
tag_mays=request.args.get('tag_mays'),
tag_forbids=request.args.get('tag_forbids'),
tag_expression=request.args.get('tag_expression'),
if limit is None:
limit = 50
limit=request.args.get('limit'),
offset=request.args.get('offset'),
orderby=request.args.get('orderby'),
yield_albums=request.args.get('yield_albums', False),
yield_photos=request.args.get('yield_photos', True),
)
# The site enforces a maximum value which the PhotoDB does not.
search.kwargs.limit = etiquette.searchhelpers.normalize_limit(search.kwargs.limit)
if search.kwargs.limit is None:
search.kwargs.limit = 50
else:
limit = min(limit, 1000)
offset = request.args.get('offset')
author = request.args.get('author')
orderby = request.args.get('orderby')
area = request.args.get('area')
width = request.args.get('width')
height = request.args.get('height')
aspectratio = request.args.get('aspectratio')
bytes = request.args.get('bytes')
has_albums = request.args.get('has_albums')
has_thumbnail = request.args.get('has_thumbnail')
duration = request.args.get('duration')
bitrate = request.args.get('bitrate')
created = request.args.get('created')
# These are in a dictionary so I can pass them to the page template.
search_kwargs = {
'area': area,
'width': width,
'height': height,
'aspectratio': aspectratio,
'bytes': bytes,
'duration': duration,
'bitrate': bitrate,
'author': author,
'created': created,
'extension': extension,
'extension_not': extension_not,
'filename': filename_terms,
'has_albums': has_albums,
'has_tags': has_tags,
'has_thumbnail': has_thumbnail,
'is_searchhidden': is_searchhidden,
'mimetype': mimetype,
'sha256': sha256,
'tag_musts': tag_musts,
'tag_mays': tag_mays,
'tag_forbids': tag_forbids,
'tag_expression': tag_expression,
'limit': limit,
'offset': offset,
'orderby': orderby,
'warning_bag': warning_bag,
'give_back_parameters': True,
'yield_albums': yield_albums,
'yield_photos': yield_photos,
}
# print(search_kwargs)
search_generator = common.P.search(**search_kwargs)
# Because of the giveback, first element is cleaned up kwargs
search_kwargs = next(search_generator)
# Web UI users aren't allowed to use within_directory anyway, so don't
# show it to them.
search_kwargs.pop('within_directory', None)
# print(search_kwargs)
warnings = set()
search_results = []
for item in search_generator:
if isinstance(item, etiquette.objects.WarningBag):
warnings.update(item.warnings)
continue
search_results.append(item)
search.kwargs.limit = min(search.kwargs.limit, 1000)
search.results = list(search.results)
warnings = [
w.error_message if hasattr(w, 'error_message') else str(w)
for w in warnings
for w in search.warning_bag.warnings
]
# Web UI users aren't allowed to use within_directory anyway, so don't
# show it to them.
del search.kwargs.within_directory
return search
@site.route('/search_embed')
def get_search_embed():
search = get_search_core()
response = common.render_template(
request,
'search_embed.html',
results=search.results,
search_kwargs=search.kwargs,
)
return response
@site.route('/search')
def get_search_html():
search = get_search_core()
search.kwargs.view = request.args.get('view', 'grid')
# TAGS ON THIS PAGE
total_tags = set()
for result in search_results:
for result in search.results:
if isinstance(result, etiquette.objects.Photo):
total_tags.update(result.get_tags())
total_tags = sorted(total_tags, key=lambda t: t.name)
# PREV-NEXT PAGE URLS
offset = search_kwargs['offset'] or 0
offset = search.kwargs.offset or 0
original_params = request.args.to_dict()
original_params['limit'] = limit
original_params['limit'] = search.kwargs.limit
if limit and len(search_results) >= limit:
if search.more_after_limit:
next_params = original_params.copy()
next_params['offset'] = offset + limit
next_params['offset'] = offset + search.kwargs.limit
next_params = helpers.dict_to_params(next_params)
next_page_url = '/search' + next_params
else:
next_page_url = None
if limit and offset > 0:
if search.kwargs.limit and offset > 0:
prev_params = original_params.copy()
prev_offset = max(0, offset - limit)
prev_offset = max(0, offset - search.kwargs.limit)
if prev_offset > 0:
prev_params['offset'] = prev_offset
else:
@ -498,49 +466,23 @@ def get_search_core():
else:
prev_page_url = None
search_kwargs['view'] = request.args.get('view', 'grid')
final_results = {
'next_page_url': next_page_url,
'prev_page_url': prev_page_url,
'results': search_results,
'total_tags': total_tags,
'warnings': list(warnings),
'search_kwargs': search_kwargs,
}
return final_results
@site.route('/search_embed')
def get_search_embed():
search_results = get_search_core()
response = common.render_template(
request,
'search_embed.html',
results=search_results['results'],
search_kwargs=search_results['search_kwargs'],
)
return response
@site.route('/search')
def get_search_html():
search_results = get_search_core()
response = common.render_template(
request,
'search.html',
next_page_url=search_results['next_page_url'],
prev_page_url=search_results['prev_page_url'],
results=search_results['results'],
search_kwargs=search_results['search_kwargs'],
total_tags=search_results['total_tags'],
warnings=search_results['warnings'],
next_page_url=next_page_url,
prev_page_url=prev_page_url,
results=search.results,
search_kwargs=search.kwargs,
total_tags=total_tags,
warnings=search.warning_bag.jsonify(),
)
return response
@site.route('/search.atom')
def get_search_atom():
search_results = get_search_core()['results']
search = get_search_core()
soup = etiquette.helpers.make_atom_feed(
search_results,
search.results,
feed_id=request.query_string.decode('utf-8'),
feed_title='etiquette search',
feed_link=request.url.replace('/search.atom', '/search'),
@ -553,34 +495,9 @@ def get_search_atom():
@site.route('/search.json')
def get_search_json():
search_results = get_search_core()
search_kwargs = search_results['search_kwargs']
# The search has converted many arguments into sets or other types.
# Convert them back into something that will display nicely on the search form.
join_helper = lambda x: ', '.join(x) if x else None
search_kwargs['extension'] = join_helper(search_kwargs['extension'])
search_kwargs['extension_not'] = join_helper(search_kwargs['extension_not'])
search_kwargs['mimetype'] = join_helper(search_kwargs['mimetype'])
author_helper = lambda users: ', '.join(user.username for user in users) if users else None
search_kwargs['author'] = author_helper(search_kwargs['author'])
tagname_helper = lambda tags: [tag.name for tag in tags] if tags else None
search_kwargs['tag_musts'] = tagname_helper(search_kwargs['tag_musts'])
search_kwargs['tag_mays'] = tagname_helper(search_kwargs['tag_mays'])
search_kwargs['tag_forbids'] = tagname_helper(search_kwargs['tag_forbids'])
search_results['results'] = [
result.jsonify(include_albums=False)
if isinstance(result, etiquette.objects.Photo) else
result.jsonify(minimal=True)
for result in search_results['results']
]
search_results['total_tags'] = [
tag.jsonify(minimal=True) for tag in search_results['total_tags']
]
return flasktools.json_response(search_results)
search = get_search_core()
response = search.jsonify()
return flasktools.json_response(response)
# Swipe ############################################################################################