Centralize and unify search parameter normalization

Create searchhelpers.py which normalize incoming search parameters.

Add argument give_back_parameters so we can more effectively return the normalized parameters to the user to learn from their mistakes.

Create objects.WarningBag and stop using the warnings module with the unclear warning-catcher.

Fix handling of photos without extensions (still needs improvement).

Adopt use of pathclass.Path in more places and keep thumbnail paths relative so that the entire _etiquette dir can be moved and still work right away.

Probably some other things
This commit is contained in:
voussoir 2016-12-24 17:13:45 -08:00
parent 564518f4d8
commit 0d0431edff
8 changed files with 518 additions and 268 deletions

View file

@ -102,8 +102,10 @@ ERROR_RECURSIVE_GROUPING = 'Recursive grouping'
WARNING_MINMAX_INVALID = 'Field "{field}": "{value}" is not a valid request. Ignored.'
WARNING_MINMAX_OOO = 'Field "{field}": minimum "{min}" maximum "{max}" are out of order. Ignored.'
WARNING_NO_SUCH_TAG = 'Tag "{tag}" does not exist. Ignored.'
WARNING_NO_SUCH_USER = 'User "{username}" does not exist. Ignored.'
WARNING_ORDERBY_INVALID = 'Invalid orderby request "{request}". Ignored.'
WARNING_ORDERBY_BADCOL = '"{column}" is not a sorting option. Ignored.'
WARNING_ORDERBY_BADSORTER = 'You can\'t order "{column}" by "{sorter}". Defaulting to descending.'
WARNING_ORDERBY_BADDIRECTION = 'You can\'t order "{column}" by "{direction}". Defaulting to descending.'
# Operational info
EXPRESSION_OPERATORS = {'(', ')', 'OR', 'AND', 'NOT'}

View file

@ -12,7 +12,9 @@ import decorators
import exceptions
import helpers
import jsonify
import objects
import phototagger
import searchhelpers
import sessions
site = flask.Flask(__name__)
@ -352,7 +354,9 @@ def get_file(photoid):
if use_original_filename:
download_as = photo.basename
else:
download_as = photo.id + '.' + photo.extension
download_as = photo.id
if photo.extension:
download_as += photo.extension
download_as = download_as.replace('"', '\\"')
response = flask.make_response(send_file(photo.real_filepath))
@ -382,77 +386,41 @@ def get_photo_json(photoid):
return photo
def get_search_core():
#print(request.args)
warning_bag = objects.WarningBag()
# FILENAME & EXTENSION
filename_terms = request.args.get('filename', None)
extension_string = request.args.get('extension', None)
extension_not_string = request.args.get('extension_not', None)
mimetype_string = request.args.get('mimetype', None)
has_tags = request.args.get('has_tags')
tag_musts = request.args.get('tag_musts')
tag_mays = request.args.get('tag_mays')
tag_forbids = request.args.get('tag_forbids')
tag_expression = request.args.get('tag_expression')
extension_list = helpers.comma_split(extension_string)
extension_not_list = helpers.comma_split(extension_not_string)
mimetype_list = helpers.comma_split(mimetype_string)
filename_terms = request.args.get('filename')
extension = request.args.get('extension')
extension_not = request.args.get('extension_not')
mimetype = request.args.get('mimetype')
# LIMIT
limit = request.args.get('limit', '')
if limit.isdigit():
limit = int(limit)
limit = min(100, limit)
else:
# Note to self: also apply to search.html template url builder.
limit = request.args.get('limit')
# This is being pre-processed because the site enforces a maximum value
# which the PhotoDB api does not.
limit = searchhelpers.normalize_limit(limit, warning_bag=warning_bag)
if limit is None:
limit = 50
# OFFSET
offset = request.args.get('offset', '')
if offset.isdigit():
offset = int(offset)
else:
offset = None
limit = min(limit, 100)
# MUSTS, MAYS, FORBIDS
qualname_map = P.export_tags(exporter=phototagger.tag_export_qualname_map)
tag_musts = request.args.get('tag_musts', '').split(',')
tag_mays = request.args.get('tag_mays', '').split(',')
tag_forbids = request.args.get('tag_forbids', '').split(',')
tag_expression = request.args.get('tag_expression', None)
offset = request.args.get('offset')
tag_musts = [qualname_map.get(tag, tag) for tag in tag_musts if tag != '']
tag_mays = [qualname_map.get(tag, tag) for tag in tag_mays if tag != '']
tag_forbids = [qualname_map.get(tag, tag) for tag in tag_forbids if tag != '']
authors = request.args.get('author')
# AUTHOR
authors = request.args.get('author', None)
if authors:
authors = authors.split(',')
authors = [a.strip() for a in authors]
authors = [P.get_user(username=a) for a in authors]
else:
authors = None
# ORDERBY
orderby = request.args.get('orderby', None)
if orderby:
orderby = orderby.replace('-', ' ')
orderby = orderby.split(',')
else:
orderby = None
# HAS_TAGS
has_tags = request.args.get('has_tags', None)
if has_tags:
has_tags = helpers.truthystring(has_tags)
else:
has_tags = None
# MINMAXERS
area = request.args.get('area', None)
width = request.args.get('width', None)
height = request.args.get('height', None)
ratio = request.args.get('ratio', None)
bytes = request.args.get('bytes', None)
duration = request.args.get('duration', None)
created = request.args.get('created', None)
orderby = request.args.get('orderby')
area = request.args.get('area')
width = request.args.get('width')
height = request.args.get('height')
ratio = request.args.get('ratio')
bytes = request.args.get('bytes')
duration = request.args.get('duration')
created = request.args.get('created')
# These are in a dictionary so I can pass them to the page template.
search_kwargs = {
@ -465,11 +433,11 @@ def get_search_core():
'authors': authors,
'created': created,
'extension': extension_list,
'extension_not': extension_not_list,
'extension': extension,
'extension_not': extension_not,
'filename': filename_terms,
'has_tags': has_tags,
'mimetype': mimetype_list,
'mimetype': mimetype,
'tag_musts': tag_musts,
'tag_mays': tag_mays,
'tag_forbids': tag_forbids,
@ -479,13 +447,35 @@ def get_search_core():
'offset': offset,
'orderby': orderby,
'warn_bad_tags': True,
'warning_bag': warning_bag,
'give_back_parameters': True
}
#print(search_kwargs)
with warnings.catch_warnings(record=True) as catcher:
photos = list(P.search(**search_kwargs))
warns = [str(warning.message) for warning in catcher]
#print(warns)
search_generator = P.search(**search_kwargs)
# Because of the giveback, first element is cleaned up kwargs
search_kwargs = next(search_generator)
# The search has converted many arguments into sets or other types.
# Convert them back into something that will display nicely on the search form.
join_helper = lambda x: ', '.join(x) if x else None
tagname_helper = lambda tags: [tag.qualified_name() for tag in tags] if tags else None
filename_helper = lambda fn: ' '.join('"%s"' % part if ' ' in part else part for part in fn) if fn else None
search_kwargs['extension'] = join_helper(search_kwargs['extension'])
search_kwargs['extension_not'] = join_helper(search_kwargs['extension_not'])
search_kwargs['mimetype'] = join_helper(search_kwargs['mimetype'])
search_kwargs['filename'] = filename_helper(search_kwargs['filename'])
search_kwargs['tag_musts'] = tagname_helper(search_kwargs['tag_musts'])
search_kwargs['tag_mays'] = tagname_helper(search_kwargs['tag_mays'])
search_kwargs['tag_forbids'] = tagname_helper(search_kwargs['tag_forbids'])
search_results = list(search_generator)
warnings = set()
photos = []
for item in search_results:
if isinstance(item, objects.WarningBag):
warnings.update(item.warnings)
else:
photos.append(item)
# TAGS ON THIS PAGE
total_tags = set()
@ -510,18 +500,14 @@ def get_search_core():
view = request.args.get('view', 'grid')
search_kwargs['view'] = view
search_kwargs['extension'] = extension_string
search_kwargs['extension_not'] = extension_not_string
search_kwargs['mimetype'] = mimetype_string
final_results = {
'next_page_url': next_page_url,
'prev_page_url': prev_page_url,
'photos': photos,
'total_tags': total_tags,
'warns': warns,
'warnings': list(warnings),
'search_kwargs': search_kwargs,
'qualname_map': qualname_map,
}
return final_results
@ -530,7 +516,7 @@ def get_search_core():
def get_search_html():
search_results = get_search_core()
search_kwargs = search_results['search_kwargs']
qualname_map = search_results['qualname_map']
qualname_map = P.export_tags(exporter=phototagger.tag_export_qualname_map)
session = session_manager.get(request)
response = flask.render_template(
'search.html',
@ -541,7 +527,7 @@ def get_search_html():
search_kwargs=search_kwargs,
session=session,
total_tags=search_results['total_tags'],
warns=search_results['warns'],
warnings=search_results['warnings'],
)
return response
@ -550,17 +536,11 @@ def get_search_html():
def get_search_json():
search_results = get_search_core()
search_results['photos'] = [jsonify.photo(photo, include_albums=False) for photo in search_results['photos']]
#search_kwargs = search_results['search_kwargs']
#qualname_map = search_results['qualname_map']
include_qualname_map = request.args.get('include_map', False)
include_qualname_map = helpers.truthystring(include_qualname_map)
if not include_qualname_map:
search_results.pop('qualname_map')
return jsonify.make_json_response(search_results)
@site.route('/static/<filename>')
def get_static(filename):
def geft_static(filename):
filename = filename.replace('\\', os.sep)
filename = filename.replace('/', os.sep)
filename = os.path.join('static', filename)

View file

@ -2,7 +2,6 @@ import datetime
import math
import mimetypes
import os
import warnings
import constants
import exceptions
@ -192,6 +191,9 @@ def is_xor(*args):
'''
return [bool(a) for a in args].count(True) == 1
def normalize_extension(extension):
pass
def normalize_filepath(filepath, allowed=''):
'''
Remove some bad characters.
@ -283,102 +285,6 @@ def truthystring(s):
return None
return False
#===============================================================================
def _minmax(key, value, minimums, maximums):
'''
When searching, this function dissects a hyphenated range string
and inserts the correct k:v pair into both minimums and maximums.
('area', '100-200', {}, {}) --> {'area': 100}, {'area': 200} (MODIFIED IN PLACE)
'''
if value is None:
return
if isinstance(value, (int, float)):
minimums[key] = value
return
try:
(low, high) = hyphen_range(value)
except ValueError:
warnings.warn(constants.WARNING_MINMAX_INVALID.format(field=key, value=value))
return
except exceptions.OutOfOrder as e:
warnings.warn(constants.WARNING_MINMAX_OOO.format(field=key, min=e.args[1], max=e.args[2]))
return
if low is not None:
minimums[key] = low
if high is not None:
maximums[key] = high
def _normalize_extensions(extensions):
'''
When searching, this function normalizes the list of inputted extensions.
'''
if isinstance(extensions, str):
extensions = extensions.split()
if extensions is None:
return set()
extensions = [e.lower().strip('.').strip() for e in extensions]
extensions = set(e for e in extensions if e)
return extensions
def _orderby(orderby):
'''
When searching, this function ensures that the user has entered a valid orderby
query, and normalizes the query text.
'random asc' --> ('random', 'asc')
'area' --> ('area', 'desc')
'''
orderby = orderby.lower().strip()
if orderby == '':
return None
orderby = orderby.split(' ')
if len(orderby) == 2:
(column, sorter) = orderby
elif len(orderby) == 1:
column = orderby[0]
sorter = 'desc'
else:
return None
#print(column, sorter)
if column not in constants.ALLOWED_ORDERBY_COLUMNS:
warnings.warn(constants.WARNING_ORDERBY_BADCOL.format(column=column))
return None
if column == 'random':
column = 'RANDOM()'
if sorter not in ['desc', 'asc']:
warnings.warn(constants.WARNING_ORDERBY_BADSORTER.format(column=column, sorter=sorter))
sorter = 'desc'
return (column, sorter)
def _setify_tags(photodb, tags, warn_bad_tags=False):
'''
When searching, this function converts the list of tag strings that the user
requested into Tag objects. If a tag doesn't exist we'll either raise an exception
or just issue a warning.
'''
if tags is None:
return set()
tagset = set()
for tag in tags:
tag = tag.strip()
if tag == '':
continue
try:
tag = photodb.get_tag(tag)
tagset.add(tag)
except exceptions.NoSuchTag:
if warn_bad_tags:
warnings.warn(constants.WARNING_NO_SUCH_TAG.format(tag=tag))
continue
else:
raise
return tagset
def _unitconvert(value):
'''

View file

@ -392,6 +392,8 @@ class Photo(ObjectBase):
For videos, you can provide a `timestamp` to take the thumbnail from.
'''
hopeful_filepath = self.make_thumbnail_filepath()
hopeful_filepath = hopeful_filepath.relative_path
#print(hopeful_filepath)
return_filepath = None
if self.mimetype == 'image':
@ -490,10 +492,10 @@ class Photo(ObjectBase):
basename = chunked_id[-1]
folder = chunked_id[:-1]
folder = os.sep.join(folder)
folder = os.path.join(self.photodb.thumbnail_directory, folder)
folder = self.photodb.thumbnail_directory.join(folder)
if folder:
os.makedirs(folder, exist_ok=True)
hopeful_filepath = os.path.join(folder, basename) + '.jpg'
os.makedirs(folder.absolute_path, exist_ok=True)
hopeful_filepath = folder.with_child(basename + '.jpg')
return hopeful_filepath
@decorators.time_me
@ -828,3 +830,10 @@ class User(ObjectBase):
def __str__(self):
rep = 'User:{username}'.format(username=self.username)
return rep
class WarningBag:
def __init__(self):
self.warnings = set()
def add(self, warning):
self.warnings.add(warning)

View file

@ -8,16 +8,18 @@ import random
import sqlite3
import string
import time
import warnings
import constants
import decorators
import exceptions
import helpers
import objects
import searchhelpers
# pip install
# https://raw.githubusercontent.com/voussoir/else/master/_voussoirkit/voussoirkit.zip
from voussoirkit import pathclass
from voussoirkit import safeprint
from voussoirkit import spinal
@ -151,7 +153,7 @@ def raise_no_such_thing(exception_class, thing_id=None, thing_name=None, comment
message = ''
raise exception_class(message)
def searchfilter_expression(photo_tags, expression, frozen_children, token_normalizer, warn_bad_tags):
def searchfilter_expression(photo_tags, expression, frozen_children, token_normalizer, warning_bag=None):
photo_tags = set(tag.name for tag in photo_tags)
operator_stack = collections.deque()
operand_stack = collections.deque()
@ -178,8 +180,8 @@ def searchfilter_expression(photo_tags, expression, frozen_children, token_norma
token = token_normalizer(token)
value = any(option in photo_tags for option in frozen_children[token])
except KeyError:
if warn_bad_tags:
warnings.warn(constants.WARNING_NO_SUCH_TAG.format(tag=token))
if warning_bag:
warning_bag.add(constants.WARNING_NO_SUCH_TAG.format(tag=token))
else:
raise exceptions.NoSuchTag(token)
return False
@ -218,7 +220,8 @@ def searchfilter_expression(photo_tags, expression, frozen_children, token_norma
while len(operand_stack) > 1 or len(operator_stack) > 0:
operate(operand_stack, operator_stack)
#print(operand_stack)
return operand_stack.pop()
success = operand_stack.pop()
return success
def searchfilter_must_may_forbid(photo_tags, tag_musts, tag_mays, tag_forbids, frozen_children):
if tag_musts and not all(any(option in photo_tags for option in frozen_children[must]) for must in tag_musts):
@ -434,7 +437,10 @@ class PDBPhotoMixin:
Returns the Photo object.
'''
filename = os.path.abspath(filename)
assert os.path.isfile(filename)
safeprint.safeprint('Processing %s' % filename)
if not os.path.isfile(filename):
raise FileNotFoundError(filename)
if not allow_duplicates:
try:
existing = self.get_photo_by_path(filename)
@ -457,7 +463,7 @@ class PDBPhotoMixin:
extension = os.path.splitext(filename)[1]
extension = extension.replace('.', '')
extension = self.normalize_tagname(extension)
#extension = self.normalize_tagname(extension)
created = int(helpers.now())
photoid = self.generate_id('photos')
@ -538,10 +544,11 @@ class PDBPhotoMixin:
tag_forbids=None,
tag_expression=None,
warn_bad_tags=False,
limit=None,
offset=None,
orderby=None
orderby=None,
warning_bag=None,
give_back_parameters=False
):
'''
PHOTO PROPERTIES
@ -590,10 +597,6 @@ class PDBPhotoMixin:
Can NOT be used with the must, may, forbid style search.
QUERY OPTIONS
warn_bad_tags:
If a tag is not found, issue a warning but continue the search.
Otherwise, a exceptions.NoSuchTag exception would be raised.
limit:
The maximum number of *successful* results to yield.
@ -604,57 +607,99 @@ class PDBPhotoMixin:
A list of strings like ['ratio DESC', 'created ASC'] to sort
and subsort the results.
Descending is assumed if not provided.
warning_bag:
Invalid search queries will add a warning to the bag and try their best to continue.
Otherwise they may raise exceptions.
give_back_parameters:
If True, the generator's first yield will be a dictionary of all the cleaned up, normalized
parameters. The user may have given us loads of trash, so we should show them the formatting
we want.
'''
start_time = time.time()
# MINMAXERS
has_tags = searchhelpers.normalize_has_tags(has_tags)
if has_tags is False:
tag_musts = None
tag_mays = None
tag_forbids = None
tag_expression = None
else:
tag_musts = searchhelpers.normalize_tag_mmf(photodb=self, tags=tag_musts, warning_bag=warning_bag)
tag_mays = searchhelpers.normalize_tag_mmf(photodb=self, tags=tag_mays, warning_bag=warning_bag)
tag_forbids = searchhelpers.normalize_tag_mmf(photodb=self, tags=tag_forbids, warning_bag=warning_bag)
tag_expression = searchhelpers.normalize_tag_expression(tag_expression)
#print(tag_musts, tag_mays, tag_forbids)
if (tag_musts or tag_mays or tag_forbids) and tag_expression:
message = 'Expression filter cannot be used with musts, mays, forbids'
if warning_bag:
warning_bag.add(message)
tag_musts = None
tag_mays = None
tag_forbids = None
tag_expression = None
else:
raise exceptions.NotExclusive(message)
extension = searchhelpers.normalize_extensions(extension)
extension_not = searchhelpers.normalize_extensions(extension_not)
mimetype = searchhelpers.normalize_extensions(mimetype)
authors = searchhelpers.normalize_authors(authors, photodb=self, warning_bag=warning_bag)
filename = searchhelpers.normalize_filename(filename)
limit = searchhelpers.normalize_limit(limit, warning_bag=warning_bag)
offset = searchhelpers.normalize_offset(offset)
if offset is None:
offset = 0
maximums = {}
minimums = {}
helpers._minmax('area', area, minimums, maximums)
helpers._minmax('created', created, minimums, maximums)
helpers._minmax('width', width, minimums, maximums)
helpers._minmax('height', height, minimums, maximums)
helpers._minmax('ratio', ratio, minimums, maximums)
helpers._minmax('bytes', bytes, minimums, maximums)
helpers._minmax('duration', duration, minimums, maximums)
orderby = orderby or []
searchhelpers.minmax('area', area, minimums, maximums, warning_bag=warning_bag)
searchhelpers.minmax('created', created, minimums, maximums, warning_bag=warning_bag)
searchhelpers.minmax('width', width, minimums, maximums, warning_bag=warning_bag)
searchhelpers.minmax('height', height, minimums, maximums, warning_bag=warning_bag)
searchhelpers.minmax('ratio', ratio, minimums, maximums, warning_bag=warning_bag)
searchhelpers.minmax('bytes', bytes, minimums, maximums, warning_bag=warning_bag)
searchhelpers.minmax('duration', duration, minimums, maximums, warning_bag=warning_bag)
extension = helpers._normalize_extensions(extension)
extension_not = helpers._normalize_extensions(extension_not)
mimetype = helpers._normalize_extensions(mimetype)
if authors is not None:
if isinstance(authors, str):
authors = {authors, }
authors = set(a.id if isinstance(a, objects.User) else a for a in authors)
if filename is not None:
if not isinstance(filename, str):
filename = ' '.join(filename)
filename = set(term.lower() for term in filename.strip().split(' '))
if (tag_musts or tag_mays or tag_forbids) and tag_expression:
raise exceptions.NotExclusive('Expression filter cannot be used with musts, mays, forbids')
tag_musts = helpers._setify_tags(photodb=self, tags=tag_musts, warn_bad_tags=warn_bad_tags)
tag_mays = helpers._setify_tags(photodb=self, tags=tag_mays, warn_bad_tags=warn_bad_tags)
tag_forbids = helpers._setify_tags(photodb=self, tags=tag_forbids, warn_bad_tags=warn_bad_tags)
query = 'SELECT * FROM photos'
orderby = [helpers._orderby(o) for o in orderby]
orderby = [o for o in orderby if o]
if orderby:
whereable_columns = [o[0] for o in orderby if o[0] != 'RANDOM()']
whereable_columns = [column + ' IS NOT NULL' for column in whereable_columns]
if whereable_columns:
query += ' WHERE '
query += ' AND '.join(whereable_columns)
orderby = [' '.join(o) for o in orderby]
orderby = ', '.join(orderby)
query += ' ORDER BY %s' % orderby
else:
query += ' ORDER BY created DESC'
orderby = searchhelpers.normalize_orderby(orderby)
query = searchhelpers.build_query(orderby)
print(query)
generator = helpers.select_generator(self.sql, query)
if give_back_parameters:
parameters = {
'area': area,
'width': width,
'height': height,
'ratio': ratio,
'bytes': bytes,
'duration': duration,
'authors': authors,
'created': created,
'extension': extension,
'extension_not': extension_not,
'filename': filename,
'has_tags': has_tags,
'mimetype': mimetype,
'tag_musts': tag_musts,
'tag_mays': tag_mays,
'tag_forbids': tag_forbids,
'tag_expression': tag_expression,
'limit': limit,
'offset': offset,
'orderby': orderby,
}
yield parameters
# FROZEN CHILDREN
# To lighten the amount of database reading here, `frozen_children` is a dict where
# EVERY tag in the db is a key, and the value is a list of ALL ITS NESTED CHILDREN.
# This representation is memory inefficient, but it is faster than repeated
@ -669,6 +714,7 @@ class PDBPhotoMixin:
self._cached_frozen_children = frozen_children
photos_received = 0
# LET'S GET STARTED
for fetch in generator:
photo = objects.Photo(self, fetch)
@ -685,6 +731,7 @@ class PDBPhotoMixin:
continue
if authors and photo.author_id not in authors:
#print('Failed author')
continue
if filename and not _helper_filenamefilter(subject=photo.basename, terms=filename):
@ -709,9 +756,11 @@ class PDBPhotoMixin:
photo_tags = photo.tags()
if has_tags is False and len(photo_tags) > 0:
#print('Failed has_tags=False')
continue
if has_tags is True and len(photo_tags) == 0:
#print('Failed has_tags=True')
continue
photo_tags = set(photo_tags)
@ -722,9 +771,10 @@ class PDBPhotoMixin:
expression=tag_expression,
frozen_children=frozen_children,
token_normalizer=self.normalize_tagname,
warn_bad_tags=warn_bad_tags,
warning_bag=warning_bag,
)
if not success:
#print('Failed tag expression')
continue
elif is_must_may_forbid:
success = searchfilter_must_may_forbid(
@ -735,18 +785,23 @@ class PDBPhotoMixin:
frozen_children=frozen_children,
)
if not success:
#print('Failed tag mmf')
continue
if offset is not None and offset > 0:
if offset > 0:
offset -= 1
continue
if limit is not None and photos_received >= limit:
break
photos_received += 1
yield photo
if warning_bag.warnings:
yield warning_bag
end_time = time.time()
print(end_time - start_time)
@ -988,13 +1043,13 @@ class PhotoDB(PDBAlbumMixin, PDBPhotoMixin, PDBTagMixin, PDBUserMixin):
# DATA DIR PREP
data_directory = helpers.normalize_filepath(data_directory, allowed='/\\')
self.data_directory = os.path.abspath(data_directory)
os.makedirs(self.data_directory, exist_ok=True)
self.data_directory = pathclass.Path(data_directory)
os.makedirs(self.data_directory.absolute_path, exist_ok=True)
# DATABASE
self.database_abspath = os.path.join(self.data_directory, 'phototagger.db')
existing_database = os.path.exists(self.database_abspath)
self.sql = sqlite3.connect(self.database_abspath)
self.database_file = self.data_directory.with_child('phototagger.db')
existing_database = self.database_file.exists
self.sql = sqlite3.connect(self.database_file.absolute_path)
self.cur = self.sql.cursor()
if existing_database:
@ -1010,21 +1065,20 @@ class PhotoDB(PDBAlbumMixin, PDBPhotoMixin, PDBTagMixin, PDBUserMixin):
self.cur.execute(statement)
# CONFIG
self.config_abspath = os.path.join(self.data_directory, 'config.json')
self.config_file = self.data_directory.with_child('config.json')
self.config = copy.deepcopy(constants.DEFAULT_CONFIGURATION)
if os.path.isfile(self.config_abspath):
with open(self.config_abspath, 'r') as handle:
if self.config_file.is_file:
with open(self.config_file.absolute_path, 'r') as handle:
user_config = json.load(handle)
self.config.update(user_config)
else:
with open(self.config_abspath, 'w') as handle:
with open(self.config_file.absolute_path, 'w') as handle:
handle.write(json.dumps(self.config, indent=4, sort_keys=True))
#print(self.config)
# THUMBNAIL DIRECTORY
self.thumbnail_directory = os.path.join(self.data_directory, 'site_thumbnails')
self.thumbnail_directory = os.path.abspath(self.thumbnail_directory)
os.makedirs(self.thumbnail_directory, exist_ok=True)
self.thumbnail_directory = self.data_directory.with_child('site_thumbnails')
os.makedirs(self.thumbnail_directory.absolute_path, exist_ok=True)
# OTHER
self.log = logging.getLogger(__name__)
@ -1095,14 +1149,16 @@ class PhotoDB(PDBAlbumMixin, PDBPhotoMixin, PDBTagMixin, PDBUserMixin):
commit=False,
title=current_location.basename,
)
print('Created %s' % current_album.title)
safeprint.safeprint('Created %s' % current_album.title)
albums[current_location.absolute_path] = current_album
parent = albums[current_location.parent.absolute_path]
parent = albums.get(current_location.parent.absolute_path, None)
if parent is not None:
try:
parent.add(current_album, commit=False)
#safeprint.safeprint('Added to %s' % parent.title)
except exceptions.GroupExists:
pass
#print('Added to %s' % parent.title)
for filepath in files:
try:
photo = self.new_photo(filepath.absolute_path, commit=False)

292
searchhelpers.py Normal file
View file

@ -0,0 +1,292 @@
import shlex
import constants
import exceptions
import helpers
import objects
def build_query(orderby):
query = 'SELECT * FROM photos'
if not orderby:
query += ' ORDER BY created DESC'
return query
orderby = [o.split('-') for o in orderby]
whereable_columns = [column for (column, sorter) in orderby if column != 'RANDOM()']
if whereable_columns:
query += ' WHERE '
whereable_columns = [column + ' IS NOT NULL' for column in whereable_columns]
query += ' AND '.join(whereable_columns)
# Combine each column+sorter
orderby = [' '.join(o) for o in orderby]
# Combine everything
orderby = ', '.join(orderby)
query += ' ORDER BY %s' % orderby
return query
def minmax(key, value, minimums, maximums, warning_bag=None):
'''
Dissects a hyphenated range string and inserts the correct k:v pair into
both minimums and maximums.
('area', '100-200', {}, {}) --> {'area': 100}, {'area': 200} (MODIFIED IN PLACE)
'''
if value is None:
return
if isinstance(value, str):
value = value.strip()
if value == '':
return
if isinstance(value, (int, float)):
minimums[key] = value
return
try:
(low, high) = helpers.hyphen_range(value)
except ValueError as e:
if warning_bag:
warning_bag.add(constants.WARNING_MINMAX_INVALID.format(field=key, value=value))
return
else:
raise e
except exceptions.OutOfOrder as e:
if warning_bag:
warning_bag.add(constants.WARNING_MINMAX_OOO.format(field=key, min=e.args[1], max=e.args[2]))
return
else:
raise e
if low is not None:
minimums[key] = low
if high is not None:
maximums[key] = high
def normalize_authors(authors, photodb, warning_bag=None):
if not authors:
return None
if isinstance(authors, str):
authors = helpers.comma_split(authors)
user_ids = set()
for requested_author in authors:
if isinstance(requested_author, objects.User):
if requested_author.photodb == photodb:
user_ids.add(requested_author.id)
else:
requested_author = requested_author.username
try:
user = photodb.get_user(username=requested_author)
except exceptions.NoSuchUser:
if warning_bag:
warning_bag.add(constants.WARNING_NO_SUCH_USER.format(username=requested_author))
else:
raise
else:
user_ids.add(user.id)
if len(user_ids) == 0:
return None
return user_ids
def normalize_extensions(extensions):
if not extensions:
return None
if isinstance(extensions, str):
extensions = helpers.comma_split(extensions)
if len(extensions) == 0:
return None
extensions = [e.lower().strip('.').strip() for e in extensions]
extensions = set(extensions)
extensions = {e for e in extensions if e}
if len(extensions) == 0:
return None
return extensions
def normalize_filename(filename_terms):
if not filename_terms:
return None
if not isinstance(filename_terms, str):
filename_terms = ' '.join(filename_terms)
filename_terms = filename_terms.strip()
filename_terms = [term.lower() for term in shlex.split(filename_terms)]
if not filename_terms:
return None
return filename_terms
def normalize_has_tags(has_tags):
if not has_tags:
return None
if isinstance(has_tags, str):
return helpers.truthystring(has_tags)
if isinstance(has_tags, int):
return bool(has_tags)
return None
def normalize_limit(limit, warning_bag=None):
if not limit and limit != 0:
return None
if isinstance(limit, str):
limit = limit.strip()
if limit.isdigit():
limit = int(limit)
if isinstance(limit, float):
limit = int(limit)
if not isinstance(limit, int):
message = 'Invalid limit "%s%"' % limit
if warning_bag:
warning_bag.add(message)
limit = None
else:
raise ValueError(message)
return limit
def normalize_offset(offset, warning_bag=None):
if not offset:
return None
if isinstance(offset, str):
offset = offset.strip()
if offset.isdigit():
offset = int(offset)
if isinstance(offset, float):
offset = int(offset)
if not isinstance(offset, int):
message = 'Invalid offset "%s%"' % offset
if warning_bag:
warning_bag.add(message)
offset = None
else:
raise ValueError(message)
return offset
def normalize_orderby(orderby, warning_bag=None):
if not orderby:
return None
if isinstance(orderby, str):
orderby = orderby.replace('-', ' ')
orderby = orderby.split(',')
if not orderby:
return None
final_orderby = []
for requested_order in orderby:
requested_order = requested_order.lower().strip()
if not requested_order:
continue
split_order = requested_order.split(' ')
if len(split_order) == 2:
(column, direction) = split_order
elif len(split_order) == 1:
column = split_order[0]
direction = 'desc'
else:
message = constants.WARNING_ORDERBY_INVALID.format(requested=requested_order)
if warning_bag:
warning_bag.add(message)
else:
raise ValueError(message)
continue
if column not in constants.ALLOWED_ORDERBY_COLUMNS:
message = constants.WARNING_ORDERBY_BADCOL.format(column=column)
if warning_bag:
warning_bag.add(message)
else:
raise ValueError(message)
continue
if column == 'random':
column = 'RANDOM()'
if direction not in ('asc', 'desc'):
message = constants.WARNING_ORDERBY_BADDIRECTION.format(column=column, direction=direction)
if warning_bag:
warning_bag.add(message)
else:
raise ValueError(message)
direction = 'desc'
requested_order = '%s-%s' % (column, direction)
final_orderby.append(requested_order)
return final_orderby
def normalize_tag_expression(expression):
if not expression:
return None
if not isinstance(expression, str):
expression = ' '.join(expression)
return expression
def normalize_tag_mmf(tags, photodb, warning_bag=None):
if not tags:
return None
if isinstance(tags, str):
tags = helpers.comma_split(tags)
tagset = set()
for tag in tags:
if isinstance(tag, objects.Tag):
if tag.photodb == photodb:
tagset.add(tag)
continue
else:
tag = tag.name
tag = tag.strip()
if tag == '':
continue
tag = tag.split('.')[-1]
try:
tag = photodb.get_tag(name=tag)
except exceptions.NoSuchTag:
if warning_bag:
warning_bag.add(constants.WARNING_NO_SUCH_TAG.format(tag=tag))
continue
else:
raise
tagset.add(tag)
if len(tagset) == 0:
return None
return tagset

View file

@ -122,8 +122,9 @@
{% if photo.duration %}
<li>Duration: {{photo.duration_string()}}</li>
{% endif %}
<li><a href="/file/{{photo.id}}.{{photo.extension}}?download=1">Download as {{photo.id}}.{{photo.extension}}</a></li>
<li><a href="/file/{{photo.id}}.{{photo.extension}}?download=1&original_filename=1">Download as "{{photo.basename}}"</a></li>
{% set extension= "." + photo.extension if photo.extension != "" else "" %}
<li><a href="/file/{{photo.id}}{{extension}}?download=1">Download as {{photo.id}}.{{photo.extension}}</a></li>
<li><a href="/file/{{photo.id}}{{extension}}?download=1&original_filename=1">Download as "{{photo.basename}}"</a></li>
</ul>
<!-- CONTAINING ALBUMS -->

View file

@ -120,8 +120,8 @@ form
<body>
{{header.make_header(session=session)}}
<div id="error_message_area">
{% for warn in warns %}
<span class="search_warning">{{warn}}</span>
{% for warning in warnings %}
<span class="search_warning">{{warning}}</span>
{% endfor %}
</div>
<div id="content_body">
@ -131,6 +131,7 @@ form
<span>Tag {{tagtype}}:</span>
<ul class="search_builder_tagger">
{% set key="tag_" + tagtype %}
{% if search_kwargs[key] %}
{% for tagname in search_kwargs[key] %}
<li class="search_builder_{{tagtype}}_inputted">
<span class="tag_object">{{tagname}}</span>
@ -138,6 +139,7 @@ form
onclick="remove_searchtag(this, '{{tagname}}', inputted_{{tagtype}});"></button>
</li>
{% endfor %}
{% endif %}
<li><input id="search_builder_{{tagtype}}_input" type="text"></li>
</ul>
</div>
@ -155,7 +157,7 @@ form
<ul id="search_builder_orderby_ul">
{% if "orderby" in search_kwargs and search_kwargs["orderby"] %}
{% for orderby in search_kwargs["orderby"] %}
{% set column, sorter=orderby.split(" ") %}
{% set column, sorter=orderby.split("-") %}
{{ create_orderby_li(selected_column=column, selected_sorter=sorter) }}
{% endfor %}
{% else %}
@ -338,9 +340,9 @@ function add_new_orderby()
{
/* Called by the green + button */
var ul = document.getElementById("search_builder_orderby_ul");
if (ul.children.length >= 8)
if (ul.children.length >= 9)
{
/* 8 because there are only 8 sortable properties */
/* 9 because there are only 9 sortable properties */
return;
}
var li = ul.children;
@ -508,9 +510,11 @@ var inputted_mays = [];
var inputted_forbids = [];
{% for tagtype in ["musts", "mays", "forbids"] %}
{% set key="tag_" + tagtype %}
{% if search_kwargs[key] %}
{% for tagname in search_kwargs[key] %}
inputted_{{tagtype}}.push("{{tagname}}");
{% endfor %}
{% endif %}
{% endfor %}
/* Assign the click handler to "Tags on this page" results. */