Centralize and unify search parameter normalization

Create searchhelpers.py which normalize incoming search parameters.

Add argument give_back_parameters so we can more effectively return the normalized parameters to the user to learn from their mistakes.

Create objects.WarningBag and stop using the warnings module with the unclear warning-catcher.

Fix handling of photos without extensions (still needs improvement).

Adopt use of pathclass.Path in more places and keep thumbnail paths relative so that the entire _etiquette dir can be moved and still work right away.

Probably some other things

constants.py

@@ -102,8 +102,10 @@ ERROR_RECURSIVE_GROUPING = 'Recursive grouping'
 WARNING_MINMAX_INVALID = 'Field "{field}": "{value}" is not a valid request. Ignored.'
 WARNING_MINMAX_OOO = 'Field "{field}": minimum "{min}" maximum "{max}" are out of order. Ignored.'
 WARNING_NO_SUCH_TAG = 'Tag "{tag}" does not exist. Ignored.'
+WARNING_NO_SUCH_USER = 'User "{username}" does not exist. Ignored.'
+WARNING_ORDERBY_INVALID = 'Invalid orderby request "{request}". Ignored.'
 WARNING_ORDERBY_BADCOL = '"{column}" is not a sorting option. Ignored.'
-WARNING_ORDERBY_BADSORTER = 'You can\'t order "{column}" by "{sorter}". Defaulting to descending.'
+WARNING_ORDERBY_BADDIRECTION = 'You can\'t order "{column}" by "{direction}". Defaulting to descending.'
 
 # Operational info
 EXPRESSION_OPERATORS = {'(', ')', 'OR', 'AND', 'NOT'}

etiquette.py

@@ -12,7 +12,9 @@ import decorators
 import exceptions
 import helpers
 import jsonify
+import objects
 import phototagger
+import searchhelpers
 import sessions
 
 site = flask.Flask(__name__)
@@ -352,7 +354,9 @@ def get_file(photoid):
         if use_original_filename:
             download_as = photo.basename
         else:
-            download_as = photo.id + '.' + photo.extension
+            download_as = photo.id
+            if photo.extension:
+                download_as += photo.extension
 
         download_as = download_as.replace('"', '\\"')
         response = flask.make_response(send_file(photo.real_filepath))
@@ -382,77 +386,41 @@ def get_photo_json(photoid):
     return photo
 
 def get_search_core():
-    #print(request.args)
+    warning_bag = objects.WarningBag()
 
-    # FILENAME & EXTENSION
+    has_tags = request.args.get('has_tags')
-    filename_terms = request.args.get('filename', None)
+    tag_musts = request.args.get('tag_musts')
-    extension_string = request.args.get('extension', None)
+    tag_mays = request.args.get('tag_mays')
-    extension_not_string = request.args.get('extension_not', None)
+    tag_forbids = request.args.get('tag_forbids')
-    mimetype_string = request.args.get('mimetype', None)
+    tag_expression = request.args.get('tag_expression')
 
-    extension_list = helpers.comma_split(extension_string)
+    filename_terms = request.args.get('filename')
-    extension_not_list = helpers.comma_split(extension_not_string)
+    extension = request.args.get('extension')
-    mimetype_list = helpers.comma_split(mimetype_string)
+    extension_not = request.args.get('extension_not')
+    mimetype = request.args.get('mimetype')
 
-    # LIMIT
+    limit = request.args.get('limit')
-    limit = request.args.get('limit', '')
+    # This is being pre-processed because the site enforces a maximum value
-    if limit.isdigit():
+    # which the PhotoDB api does not.
-        limit = int(limit)
+    limit = searchhelpers.normalize_limit(limit, warning_bag=warning_bag)
-        limit = min(100, limit)
+
-    else:
+    if limit is None:
-        # Note to self: also apply to search.html template url builder.
         limit = 50
-
-    # OFFSET
-    offset = request.args.get('offset', '')
-    if offset.isdigit():
-        offset = int(offset)
     else:
-        offset = None
+        limit = min(limit, 100)
 
-    # MUSTS, MAYS, FORBIDS
+    offset = request.args.get('offset')
-    qualname_map = P.export_tags(exporter=phototagger.tag_export_qualname_map)
-    tag_musts = request.args.get('tag_musts', '').split(',')
-    tag_mays = request.args.get('tag_mays', '').split(',')
-    tag_forbids = request.args.get('tag_forbids', '').split(',')
-    tag_expression = request.args.get('tag_expression', None)
 
-    tag_musts = [qualname_map.get(tag, tag) for tag in tag_musts if tag != '']
+    authors = request.args.get('author')
-    tag_mays = [qualname_map.get(tag, tag) for tag in tag_mays if tag != '']
-    tag_forbids = [qualname_map.get(tag, tag) for tag in tag_forbids if tag != '']
 
-    # AUTHOR
+    orderby = request.args.get('orderby')
-    authors = request.args.get('author', None)
+    area = request.args.get('area')
-    if authors:
+    width = request.args.get('width')
-        authors = authors.split(',')
+    height = request.args.get('height')
-        authors = [a.strip() for a in authors]
+    ratio = request.args.get('ratio')
-        authors = [P.get_user(username=a) for a in authors]
+    bytes = request.args.get('bytes')
-    else:
+    duration = request.args.get('duration')
-        authors = None
+    created = request.args.get('created')
-
-    # ORDERBY
-    orderby = request.args.get('orderby', None)
-    if orderby:
-        orderby = orderby.replace('-', ' ')
-        orderby = orderby.split(',')
-    else:
-        orderby = None
-
-    # HAS_TAGS
-    has_tags = request.args.get('has_tags', None)
-    if has_tags:
-        has_tags = helpers.truthystring(has_tags)
-    else:
-        has_tags = None
-
-    # MINMAXERS
-    area = request.args.get('area', None)
-    width = request.args.get('width', None)
-    height = request.args.get('height', None)
-    ratio = request.args.get('ratio', None)
-    bytes = request.args.get('bytes', None)
-    duration = request.args.get('duration', None)
-    created = request.args.get('created', None)
 
     # These are in a dictionary so I can pass them to the page template.
     search_kwargs = {
@@ -465,11 +433,11 @@ def get_search_core():
 
         'authors': authors,
         'created': created,
-        'extension': extension_list,
+        'extension': extension,
-        'extension_not': extension_not_list,
+        'extension_not': extension_not,
         'filename': filename_terms,
         'has_tags': has_tags,
-        'mimetype': mimetype_list,
+        'mimetype': mimetype,
         'tag_musts': tag_musts,
         'tag_mays': tag_mays,
         'tag_forbids': tag_forbids,
@@ -479,13 +447,35 @@ def get_search_core():
         'offset': offset,
         'orderby': orderby,
 
-        'warn_bad_tags': True,
+        'warning_bag': warning_bag,
+        'give_back_parameters': True
     }
     #print(search_kwargs)
-    with warnings.catch_warnings(record=True) as catcher:
+    search_generator = P.search(**search_kwargs)
-        photos = list(P.search(**search_kwargs))
+    # Because of the giveback, first element is cleaned up kwargs
-        warns = [str(warning.message) for warning in catcher]
+    search_kwargs = next(search_generator)
-    #print(warns)
+
+    # The search has converted many arguments into sets or other types.
+    # Convert them back into something that will display nicely on the search form.
+    join_helper = lambda x: ', '.join(x) if x else None
+    tagname_helper = lambda tags: [tag.qualified_name() for tag in tags] if tags else None
+    filename_helper = lambda fn: ' '.join('"%s"' % part if ' ' in part else part for part in fn) if fn else None
+    search_kwargs['extension'] = join_helper(search_kwargs['extension'])
+    search_kwargs['extension_not'] = join_helper(search_kwargs['extension_not'])
+    search_kwargs['mimetype'] = join_helper(search_kwargs['mimetype'])
+    search_kwargs['filename'] = filename_helper(search_kwargs['filename'])
+    search_kwargs['tag_musts'] = tagname_helper(search_kwargs['tag_musts'])
+    search_kwargs['tag_mays'] = tagname_helper(search_kwargs['tag_mays'])
+    search_kwargs['tag_forbids'] = tagname_helper(search_kwargs['tag_forbids'])
+
+    search_results = list(search_generator)
+    warnings = set()
+    photos = []
+    for item in search_results:
+        if isinstance(item, objects.WarningBag):
+            warnings.update(item.warnings)
+        else:
+            photos.append(item)
 
     # TAGS ON THIS PAGE
     total_tags = set()
@@ -510,18 +500,14 @@ def get_search_core():
 
     view = request.args.get('view', 'grid')
     search_kwargs['view'] = view
-    search_kwargs['extension'] = extension_string
-    search_kwargs['extension_not'] = extension_not_string
-    search_kwargs['mimetype'] = mimetype_string
 
     final_results = {
         'next_page_url': next_page_url,
         'prev_page_url': prev_page_url,
         'photos': photos,
         'total_tags': total_tags,
-        'warns': warns,
+        'warnings': list(warnings),
         'search_kwargs': search_kwargs,
-        'qualname_map': qualname_map,
     }
     return final_results
 
@@ -530,7 +516,7 @@ def get_search_core():
 def get_search_html():
     search_results = get_search_core()
     search_kwargs = search_results['search_kwargs']
-    qualname_map = search_results['qualname_map']
+    qualname_map = P.export_tags(exporter=phototagger.tag_export_qualname_map)
     session = session_manager.get(request)
     response = flask.render_template(
         'search.html',
@@ -541,7 +527,7 @@ def get_search_html():
         search_kwargs=search_kwargs,
         session=session,
         total_tags=search_results['total_tags'],
-        warns=search_results['warns'],
+        warnings=search_results['warnings'],
     )
     return response
 
@@ -550,17 +536,11 @@ def get_search_html():
 def get_search_json():
     search_results = get_search_core()
     search_results['photos'] = [jsonify.photo(photo, include_albums=False) for photo in search_results['photos']]
-    #search_kwargs = search_results['search_kwargs']
-    #qualname_map = search_results['qualname_map']
-    include_qualname_map = request.args.get('include_map', False)
-    include_qualname_map = helpers.truthystring(include_qualname_map)
-    if not include_qualname_map:
-        search_results.pop('qualname_map')
     return jsonify.make_json_response(search_results)
 
 
 @site.route('/static/<filename>')
-def get_static(filename):
+def geft_static(filename):
     filename = filename.replace('\\', os.sep)
     filename = filename.replace('/', os.sep)
     filename = os.path.join('static', filename)

helpers.py

@@ -2,7 +2,6 @@ import datetime
 import math
 import mimetypes
 import os
-import warnings
 
 import constants
 import exceptions
@@ -192,6 +191,9 @@ def is_xor(*args):
     '''
     return [bool(a) for a in args].count(True) == 1
 
+def normalize_extension(extension):
+    pass
+
 def normalize_filepath(filepath, allowed=''):
     '''
     Remove some bad characters.
@@ -283,102 +285,6 @@ def truthystring(s):
         return None
     return False
 
-#===============================================================================
-
-def _minmax(key, value, minimums, maximums):
-    '''
-    When searching, this function dissects a hyphenated range string
-    and inserts the correct k:v pair into both minimums and maximums.
-    ('area', '100-200', {}, {}) --> {'area': 100}, {'area': 200} (MODIFIED IN PLACE)
-    '''
-    if value is None:
-        return
-    if isinstance(value, (int, float)):
-        minimums[key] = value
-        return
-    try:
-        (low, high) = hyphen_range(value)
-    except ValueError:
-        warnings.warn(constants.WARNING_MINMAX_INVALID.format(field=key, value=value))
-        return
-    except exceptions.OutOfOrder as e:
-        warnings.warn(constants.WARNING_MINMAX_OOO.format(field=key, min=e.args[1], max=e.args[2]))
-        return
-    if low is not None:
-        minimums[key] = low
-    if high is not None:
-        maximums[key] = high
-
-def _normalize_extensions(extensions):
-    '''
-    When searching, this function normalizes the list of inputted extensions.
-    '''
-    if isinstance(extensions, str):
-        extensions = extensions.split()
-    if extensions is None:
-        return set()
-    extensions = [e.lower().strip('.').strip() for e in extensions]
-    extensions = set(e for e in extensions if e)
-    return extensions
-
-def _orderby(orderby):
-    '''
-    When searching, this function ensures that the user has entered a valid orderby
-    query, and normalizes the query text.
-
-    'random asc' --> ('random', 'asc')
-    'area' --> ('area', 'desc')
-    '''
-    orderby = orderby.lower().strip()
-    if orderby == '':
-        return None
-
-    orderby = orderby.split(' ')
-    if len(orderby) == 2:
-        (column, sorter) = orderby
-    elif len(orderby) == 1:
-        column = orderby[0]
-        sorter = 'desc'
-    else:
-        return None
-
-    #print(column, sorter)
-    if column not in constants.ALLOWED_ORDERBY_COLUMNS:
-        warnings.warn(constants.WARNING_ORDERBY_BADCOL.format(column=column))
-        return None
-    if column == 'random':
-        column = 'RANDOM()'
-
-    if sorter not in ['desc', 'asc']:
-        warnings.warn(constants.WARNING_ORDERBY_BADSORTER.format(column=column, sorter=sorter))
-        sorter = 'desc'
-    return (column, sorter)
-
-def _setify_tags(photodb, tags, warn_bad_tags=False):
-    '''
-    When searching, this function converts the list of tag strings that the user
-    requested into Tag objects. If a tag doesn't exist we'll either raise an exception
-    or just issue a warning.
-    '''
-    if tags is None:
-        return set()
-
-    tagset = set()
-    for tag in tags:
-        tag = tag.strip()
-        if tag == '':
-            continue
-        try:
-            tag = photodb.get_tag(tag)
-            tagset.add(tag)
-        except exceptions.NoSuchTag:
-            if warn_bad_tags:
-                warnings.warn(constants.WARNING_NO_SUCH_TAG.format(tag=tag))
-                continue
-            else:
-                raise
-
-    return tagset
 
 def _unitconvert(value):
     '''

objects.py

@@ -392,6 +392,8 @@ class Photo(ObjectBase):
             For videos, you can provide a `timestamp` to take the thumbnail from.
         '''
         hopeful_filepath = self.make_thumbnail_filepath()
+        hopeful_filepath = hopeful_filepath.relative_path
+        #print(hopeful_filepath)
         return_filepath = None
 
         if self.mimetype == 'image':
@@ -490,10 +492,10 @@ class Photo(ObjectBase):
         basename = chunked_id[-1]
         folder = chunked_id[:-1]
         folder = os.sep.join(folder)
-        folder = os.path.join(self.photodb.thumbnail_directory, folder)
+        folder = self.photodb.thumbnail_directory.join(folder)
         if folder:
-            os.makedirs(folder, exist_ok=True)
+            os.makedirs(folder.absolute_path, exist_ok=True)
-        hopeful_filepath = os.path.join(folder, basename) + '.jpg'
+        hopeful_filepath = folder.with_child(basename + '.jpg')
         return hopeful_filepath
 
     @decorators.time_me
@@ -828,3 +830,10 @@ class User(ObjectBase):
     def __str__(self):
         rep = 'User:{username}'.format(username=self.username)
         return rep
+
+class WarningBag:
+    def __init__(self):
+        self.warnings = set()
+
+    def add(self, warning):
+        self.warnings.add(warning)

phototagger.py

@@ -8,16 +8,18 @@ import random
 import sqlite3
 import string
 import time
-import warnings
 
 import constants
 import decorators
 import exceptions
 import helpers
 import objects
+import searchhelpers
 
 # pip install
 # https://raw.githubusercontent.com/voussoir/else/master/_voussoirkit/voussoirkit.zip
+from voussoirkit import pathclass
+from voussoirkit import safeprint
 from voussoirkit import spinal
 
 
@@ -151,7 +153,7 @@ def raise_no_such_thing(exception_class, thing_id=None, thing_name=None, comment
         message = ''
     raise exception_class(message)
 
-def searchfilter_expression(photo_tags, expression, frozen_children, token_normalizer, warn_bad_tags):
+def searchfilter_expression(photo_tags, expression, frozen_children, token_normalizer, warning_bag=None):
     photo_tags = set(tag.name for tag in photo_tags)
     operator_stack = collections.deque()
     operand_stack = collections.deque()
@@ -178,8 +180,8 @@ def searchfilter_expression(photo_tags, expression, frozen_children, token_norma
                 token = token_normalizer(token)
                 value = any(option in photo_tags for option in frozen_children[token])
             except KeyError:
-                if warn_bad_tags:
+                if warning_bag:
-                    warnings.warn(constants.WARNING_NO_SUCH_TAG.format(tag=token))
+                    warning_bag.add(constants.WARNING_NO_SUCH_TAG.format(tag=token))
                 else:
                     raise exceptions.NoSuchTag(token)
                 return False
@@ -218,7 +220,8 @@ def searchfilter_expression(photo_tags, expression, frozen_children, token_norma
     while len(operand_stack) > 1 or len(operator_stack) > 0:
         operate(operand_stack, operator_stack)
     #print(operand_stack)
-    return operand_stack.pop()
+    success = operand_stack.pop()
+    return success
 
 def searchfilter_must_may_forbid(photo_tags, tag_musts, tag_mays, tag_forbids, frozen_children):
     if tag_musts and not all(any(option in photo_tags for option in frozen_children[must]) for must in tag_musts):
@@ -434,7 +437,10 @@ class PDBPhotoMixin:
         Returns the Photo object.
         '''
         filename = os.path.abspath(filename)
-        assert os.path.isfile(filename)
+        safeprint.safeprint('Processing %s' % filename)
+        if not os.path.isfile(filename):
+            raise FileNotFoundError(filename)
+
         if not allow_duplicates:
             try:
                 existing = self.get_photo_by_path(filename)
@@ -457,7 +463,7 @@ class PDBPhotoMixin:
 
         extension = os.path.splitext(filename)[1]
         extension = extension.replace('.', '')
-        extension = self.normalize_tagname(extension)
+        #extension = self.normalize_tagname(extension)
         created = int(helpers.now())
         photoid = self.generate_id('photos')
 
@@ -538,10 +544,11 @@ class PDBPhotoMixin:
             tag_forbids=None,
             tag_expression=None,
 
-            warn_bad_tags=False,
             limit=None,
             offset=None,
-            orderby=None
+            orderby=None,
+            warning_bag=None,
+            give_back_parameters=False
         ):
         '''
         PHOTO PROPERTIES
@@ -590,10 +597,6 @@ class PDBPhotoMixin:
             Can NOT be used with the must, may, forbid style search.
 
         QUERY OPTIONS
-        warn_bad_tags:
-            If a tag is not found, issue a warning but continue the search.
-            Otherwise, a exceptions.NoSuchTag exception would be raised.
-
         limit:
             The maximum number of *successful* results to yield.
 
@@ -604,57 +607,99 @@ class PDBPhotoMixin:
             A list of strings like ['ratio DESC', 'created ASC'] to sort
             and subsort the results.
             Descending is assumed if not provided.
+
+        warning_bag:
+            Invalid search queries will add a warning to the bag and try their best to continue.
+            Otherwise they may raise exceptions.
+
+        give_back_parameters:
+            If True, the generator's first yield will be a dictionary of all the cleaned up, normalized
+            parameters. The user may have given us loads of trash, so we should show them the formatting
+            we want.
         '''
         start_time = time.time()
+
+        # MINMAXERS
+
+        has_tags = searchhelpers.normalize_has_tags(has_tags)
+        if has_tags is False:
+            tag_musts = None
+            tag_mays = None
+            tag_forbids = None
+            tag_expression = None    
+        else:
+            tag_musts = searchhelpers.normalize_tag_mmf(photodb=self, tags=tag_musts, warning_bag=warning_bag)
+            tag_mays = searchhelpers.normalize_tag_mmf(photodb=self, tags=tag_mays, warning_bag=warning_bag)
+            tag_forbids = searchhelpers.normalize_tag_mmf(photodb=self, tags=tag_forbids, warning_bag=warning_bag)
+            tag_expression = searchhelpers.normalize_tag_expression(tag_expression)
+
+        #print(tag_musts, tag_mays, tag_forbids)
+        if (tag_musts or tag_mays or tag_forbids) and tag_expression:
+            message = 'Expression filter cannot be used with musts, mays, forbids'
+            if warning_bag:
+                warning_bag.add(message)
+                tag_musts = None
+                tag_mays = None
+                tag_forbids = None
+                tag_expression = None
+            else:
+                raise exceptions.NotExclusive(message)
+
+        extension = searchhelpers.normalize_extensions(extension)
+        extension_not = searchhelpers.normalize_extensions(extension_not)
+        mimetype = searchhelpers.normalize_extensions(mimetype)
+
+        authors = searchhelpers.normalize_authors(authors, photodb=self, warning_bag=warning_bag)
+
+        filename = searchhelpers.normalize_filename(filename)
+
+        limit = searchhelpers.normalize_limit(limit, warning_bag=warning_bag)
+
+        offset = searchhelpers.normalize_offset(offset)
+        if offset is None:
+            offset = 0
+
         maximums = {}
         minimums = {}
-        helpers._minmax('area', area, minimums, maximums)
+        searchhelpers.minmax('area', area, minimums, maximums, warning_bag=warning_bag)
-        helpers._minmax('created', created, minimums, maximums)
+        searchhelpers.minmax('created', created, minimums, maximums, warning_bag=warning_bag)
-        helpers._minmax('width', width, minimums, maximums)
+        searchhelpers.minmax('width', width, minimums, maximums, warning_bag=warning_bag)
-        helpers._minmax('height', height, minimums, maximums)
+        searchhelpers.minmax('height', height, minimums, maximums, warning_bag=warning_bag)
-        helpers._minmax('ratio', ratio, minimums, maximums)
+        searchhelpers.minmax('ratio', ratio, minimums, maximums, warning_bag=warning_bag)
-        helpers._minmax('bytes', bytes, minimums, maximums)
+        searchhelpers.minmax('bytes', bytes, minimums, maximums, warning_bag=warning_bag)
-        helpers._minmax('duration', duration, minimums, maximums)
+        searchhelpers.minmax('duration', duration, minimums, maximums, warning_bag=warning_bag)
-        orderby = orderby or []
 
-        extension = helpers._normalize_extensions(extension)
+        orderby = searchhelpers.normalize_orderby(orderby)
-        extension_not = helpers._normalize_extensions(extension_not)
+        query = searchhelpers.build_query(orderby)
-        mimetype = helpers._normalize_extensions(mimetype)
-
-        if authors is not None:
-            if isinstance(authors, str):
-                authors = {authors, }
-            authors = set(a.id if isinstance(a, objects.User) else a for a in authors)
-
-        if filename is not None:
-            if not isinstance(filename, str):
-                filename = ' '.join(filename)
-            filename = set(term.lower() for term in filename.strip().split(' '))
-
-        if (tag_musts or tag_mays or tag_forbids) and tag_expression:
-            raise exceptions.NotExclusive('Expression filter cannot be used with musts, mays, forbids')
-
-        tag_musts = helpers._setify_tags(photodb=self, tags=tag_musts, warn_bad_tags=warn_bad_tags)
-        tag_mays = helpers._setify_tags(photodb=self, tags=tag_mays, warn_bad_tags=warn_bad_tags)
-        tag_forbids = helpers._setify_tags(photodb=self, tags=tag_forbids, warn_bad_tags=warn_bad_tags)
-
-        query = 'SELECT * FROM photos'
-        orderby = [helpers._orderby(o) for o in orderby]
-        orderby = [o for o in orderby if o]
-        if orderby:
-            whereable_columns = [o[0] for o in orderby if o[0] != 'RANDOM()']
-            whereable_columns = [column + ' IS NOT NULL' for column in whereable_columns]
-            if whereable_columns:
-                query += ' WHERE '
-                query += ' AND '.join(whereable_columns)
-            orderby = [' '.join(o) for o in orderby]
-            orderby = ', '.join(orderby)
-            query += ' ORDER BY %s' % orderby
-        else:
-            query += ' ORDER BY created DESC'
         print(query)
         generator = helpers.select_generator(self.sql, query)
 
+        if give_back_parameters:
+            parameters = {
+                'area': area,
+                'width': width,
+                'height': height,
+                'ratio': ratio,
+                'bytes': bytes,
+                'duration': duration,
+                'authors': authors,
+                'created': created,
+                'extension': extension,
+                'extension_not': extension_not,
+                'filename': filename,
+                'has_tags': has_tags,
+                'mimetype': mimetype,
+                'tag_musts': tag_musts,
+                'tag_mays': tag_mays,
+                'tag_forbids': tag_forbids,
+                'tag_expression': tag_expression,
+                'limit': limit,
+                'offset': offset,
+                'orderby': orderby,
+            }
+            yield parameters
+
+        # FROZEN CHILDREN
         # To lighten the amount of database reading here, `frozen_children` is a dict where
         # EVERY tag in the db is a key, and the value is a list of ALL ITS NESTED CHILDREN.
         # This representation is memory inefficient, but it is faster than repeated
@@ -669,6 +714,7 @@ class PDBPhotoMixin:
                 self._cached_frozen_children = frozen_children
         photos_received = 0
 
+        # LET'S GET STARTED
         for fetch in generator:
             photo = objects.Photo(self, fetch)
 
@@ -685,6 +731,7 @@ class PDBPhotoMixin:
                 continue
 
             if authors and photo.author_id not in authors:
+                #print('Failed author')
                 continue
 
             if filename and not _helper_filenamefilter(subject=photo.basename, terms=filename):
@@ -709,9 +756,11 @@ class PDBPhotoMixin:
                 photo_tags = photo.tags()
 
                 if has_tags is False and len(photo_tags) > 0:
+                    #print('Failed has_tags=False')
                     continue
 
                 if has_tags is True and len(photo_tags) == 0:
+                    #print('Failed has_tags=True')
                     continue
 
                 photo_tags = set(photo_tags)
@@ -722,9 +771,10 @@ class PDBPhotoMixin:
                         expression=tag_expression,
                         frozen_children=frozen_children,
                         token_normalizer=self.normalize_tagname,
-                        warn_bad_tags=warn_bad_tags,
+                        warning_bag=warning_bag,
                     )
                     if not success:
+                        #print('Failed tag expression')
                         continue
                 elif is_must_may_forbid:
                     success = searchfilter_must_may_forbid(
@@ -735,18 +785,23 @@ class PDBPhotoMixin:
                         frozen_children=frozen_children,
                     )
                     if not success:
+                        #print('Failed tag mmf')
                         continue
 
-            if offset is not None and offset > 0:
+            if offset > 0:
                 offset -= 1
                 continue
 
             if limit is not None and photos_received >= limit:
                 break
 
+
             photos_received += 1
             yield photo
 
+        if warning_bag.warnings:
+            yield warning_bag
+
         end_time = time.time()
         print(end_time - start_time)
 
@@ -988,13 +1043,13 @@ class PhotoDB(PDBAlbumMixin, PDBPhotoMixin, PDBTagMixin, PDBUserMixin):
 
         # DATA DIR PREP
         data_directory = helpers.normalize_filepath(data_directory, allowed='/\\')
-        self.data_directory = os.path.abspath(data_directory)
+        self.data_directory = pathclass.Path(data_directory)
-        os.makedirs(self.data_directory, exist_ok=True)
+        os.makedirs(self.data_directory.absolute_path, exist_ok=True)
 
         # DATABASE
-        self.database_abspath = os.path.join(self.data_directory, 'phototagger.db')
+        self.database_file = self.data_directory.with_child('phototagger.db')
-        existing_database = os.path.exists(self.database_abspath)
+        existing_database = self.database_file.exists
-        self.sql = sqlite3.connect(self.database_abspath)
+        self.sql = sqlite3.connect(self.database_file.absolute_path)
         self.cur = self.sql.cursor()
 
         if existing_database:
@@ -1010,21 +1065,20 @@ class PhotoDB(PDBAlbumMixin, PDBPhotoMixin, PDBTagMixin, PDBUserMixin):
             self.cur.execute(statement)
 
         # CONFIG
-        self.config_abspath = os.path.join(self.data_directory, 'config.json')
+        self.config_file = self.data_directory.with_child('config.json')
         self.config = copy.deepcopy(constants.DEFAULT_CONFIGURATION)
-        if os.path.isfile(self.config_abspath):
+        if self.config_file.is_file:
-            with open(self.config_abspath, 'r') as handle:
+            with open(self.config_file.absolute_path, 'r') as handle:
                 user_config = json.load(handle)
             self.config.update(user_config)
         else:
-            with open(self.config_abspath, 'w') as handle:
+            with open(self.config_file.absolute_path, 'w') as handle:
                 handle.write(json.dumps(self.config, indent=4, sort_keys=True))
         #print(self.config)
 
         # THUMBNAIL DIRECTORY
-        self.thumbnail_directory = os.path.join(self.data_directory, 'site_thumbnails')
+        self.thumbnail_directory = self.data_directory.with_child('site_thumbnails')
-        self.thumbnail_directory = os.path.abspath(self.thumbnail_directory)
+        os.makedirs(self.thumbnail_directory.absolute_path, exist_ok=True)
-        os.makedirs(self.thumbnail_directory, exist_ok=True)
 
         # OTHER
         self.log = logging.getLogger(__name__)
@@ -1095,14 +1149,16 @@ class PhotoDB(PDBAlbumMixin, PDBPhotoMixin, PDBTagMixin, PDBUserMixin):
                         commit=False,
                         title=current_location.basename,
                     )
-                    print('Created %s' % current_album.title)
+                    safeprint.safeprint('Created %s' % current_album.title)
                 albums[current_location.absolute_path] = current_album
-                parent = albums[current_location.parent.absolute_path]
+
+            parent = albums.get(current_location.parent.absolute_path, None)
+            if parent is not None:
                 try:
                     parent.add(current_album, commit=False)
+                    #safeprint.safeprint('Added to %s' % parent.title)
                 except exceptions.GroupExists:
                     pass
-                #print('Added to %s' % parent.title)
             for filepath in files:
                 try:
                     photo = self.new_photo(filepath.absolute_path, commit=False)

searchhelpers.py

@@ -0,0 +1,292 @@
+import shlex
+
+import constants
+import exceptions
+import helpers
+import objects
+
+def build_query(orderby):
+    query = 'SELECT * FROM photos'
+    if not orderby:
+        query += ' ORDER BY created DESC'
+        return query
+
+    orderby = [o.split('-') for o in orderby]
+    whereable_columns = [column for (column, sorter) in orderby if column != 'RANDOM()']
+    if whereable_columns:
+        query += ' WHERE '
+        whereable_columns = [column + ' IS NOT NULL' for column in whereable_columns]
+        query += ' AND '.join(whereable_columns)
+
+    # Combine each column+sorter
+    orderby = [' '.join(o) for o in orderby]
+
+    # Combine everything
+    orderby = ', '.join(orderby)
+    query += ' ORDER BY %s' % orderby
+    return query
+
+def minmax(key, value, minimums, maximums, warning_bag=None):
+    '''
+    Dissects a hyphenated range string and inserts the correct k:v pair into
+    both minimums and maximums.
+    ('area', '100-200', {}, {}) --> {'area': 100}, {'area': 200} (MODIFIED IN PLACE)
+    '''
+    if value is None:
+        return
+
+    if isinstance(value, str):
+        value = value.strip()
+
+    if value == '':
+        return
+
+    if isinstance(value, (int, float)):
+        minimums[key] = value
+        return
+
+    try:
+        (low, high) = helpers.hyphen_range(value)
+
+    except ValueError as e:
+        if warning_bag:
+            warning_bag.add(constants.WARNING_MINMAX_INVALID.format(field=key, value=value))
+            return
+        else:
+            raise e
+
+    except exceptions.OutOfOrder as e:
+        if warning_bag:
+            warning_bag.add(constants.WARNING_MINMAX_OOO.format(field=key, min=e.args[1], max=e.args[2]))
+            return
+        else:
+            raise e
+
+    if low is not None:
+        minimums[key] = low
+
+    if high is not None:
+        maximums[key] = high
+
+def normalize_authors(authors, photodb, warning_bag=None):
+    if not authors:
+        return None
+
+    if isinstance(authors, str):
+        authors = helpers.comma_split(authors)
+
+    user_ids = set()
+    for requested_author in authors:
+        if isinstance(requested_author, objects.User):
+            if requested_author.photodb == photodb:
+                user_ids.add(requested_author.id)
+            else:
+                requested_author = requested_author.username
+
+        try:
+            user = photodb.get_user(username=requested_author)
+        except exceptions.NoSuchUser:
+            if warning_bag:
+                warning_bag.add(constants.WARNING_NO_SUCH_USER.format(username=requested_author))
+            else:
+                raise
+        else:
+            user_ids.add(user.id)
+
+    if len(user_ids) == 0:
+        return None
+
+    return user_ids
+
+def normalize_extensions(extensions):
+    if not extensions:
+        return None
+
+    if isinstance(extensions, str):
+        extensions = helpers.comma_split(extensions)
+
+    if len(extensions) == 0:
+        return None
+
+    extensions = [e.lower().strip('.').strip() for e in extensions]
+    extensions = set(extensions)
+    extensions = {e for e in extensions if e}
+
+    if len(extensions) == 0:
+        return None
+
+    return extensions
+
+def normalize_filename(filename_terms):
+    if not filename_terms:
+        return None
+
+    if not isinstance(filename_terms, str):
+        filename_terms = ' '.join(filename_terms)
+
+    filename_terms = filename_terms.strip()
+    filename_terms = [term.lower() for term in shlex.split(filename_terms)]
+
+    if not filename_terms:
+        return None
+
+    return filename_terms
+
+def normalize_has_tags(has_tags):
+    if not has_tags:
+        return None
+
+    if isinstance(has_tags, str):
+        return helpers.truthystring(has_tags)
+
+    if isinstance(has_tags, int):
+        return bool(has_tags)
+
+    return None
+
+def normalize_limit(limit, warning_bag=None):
+    if not limit and limit != 0:
+        return None
+
+    if isinstance(limit, str):
+        limit = limit.strip()
+        if limit.isdigit():
+            limit = int(limit)
+
+    if isinstance(limit, float):
+        limit = int(limit)
+
+    if not isinstance(limit, int):
+        message = 'Invalid limit "%s%"' % limit
+        if warning_bag:
+            warning_bag.add(message)
+            limit = None
+        else:
+            raise ValueError(message)
+
+    return limit
+
+def normalize_offset(offset, warning_bag=None):
+    if not offset:
+        return None
+
+    if isinstance(offset, str):
+        offset = offset.strip()
+        if offset.isdigit():
+            offset = int(offset)
+
+    if isinstance(offset, float):
+        offset = int(offset)
+
+    if not isinstance(offset, int):
+        message = 'Invalid offset "%s%"' % offset
+        if warning_bag:
+            warning_bag.add(message)
+            offset = None
+        else:
+            raise ValueError(message)
+
+    return offset
+
+
+def normalize_orderby(orderby, warning_bag=None):
+    if not orderby:
+        return None
+
+    if isinstance(orderby, str):
+        orderby = orderby.replace('-', ' ')
+        orderby = orderby.split(',')
+
+    if not orderby:
+        return None
+
+    final_orderby = []
+    for requested_order in orderby:
+        requested_order = requested_order.lower().strip()
+        if not requested_order:
+            continue
+
+        split_order = requested_order.split(' ')
+        if len(split_order) == 2:
+            (column, direction) = split_order
+
+        elif len(split_order) == 1:
+            column = split_order[0]
+            direction = 'desc'
+
+        else:
+            message = constants.WARNING_ORDERBY_INVALID.format(requested=requested_order)
+            if warning_bag:
+                warning_bag.add(message)
+            else:
+                raise ValueError(message)
+            continue
+
+        if column not in constants.ALLOWED_ORDERBY_COLUMNS:
+            message = constants.WARNING_ORDERBY_BADCOL.format(column=column)
+            if warning_bag:
+                warning_bag.add(message)
+            else:
+                raise ValueError(message)
+            continue
+
+        if column == 'random':
+            column = 'RANDOM()'
+
+        if direction not in ('asc', 'desc'):
+            message = constants.WARNING_ORDERBY_BADDIRECTION.format(column=column, direction=direction)
+            if warning_bag:
+                warning_bag.add(message)
+            else:
+                raise ValueError(message)            
+            direction = 'desc'
+
+        requested_order = '%s-%s' % (column, direction)
+        final_orderby.append(requested_order)
+
+    return final_orderby
+
+def normalize_tag_expression(expression):
+    if not expression:
+        return None
+
+    if not isinstance(expression, str):
+        expression = ' '.join(expression)
+
+    return expression
+
+def normalize_tag_mmf(tags, photodb, warning_bag=None):
+    if not tags:
+        return None
+
+    if isinstance(tags, str):
+        tags = helpers.comma_split(tags)
+
+    tagset = set()
+    for tag in tags:
+        if isinstance(tag, objects.Tag):
+            if tag.photodb == photodb:
+                tagset.add(tag)
+                continue
+            else:
+                tag = tag.name
+
+        tag = tag.strip()
+        if tag == '':
+            continue
+        tag = tag.split('.')[-1]
+
+        try:
+            tag = photodb.get_tag(name=tag)
+        except exceptions.NoSuchTag:
+            if warning_bag:
+                warning_bag.add(constants.WARNING_NO_SUCH_TAG.format(tag=tag))
+                continue
+            else:
+                raise
+        tagset.add(tag)
+
+    if len(tagset) == 0:
+        return None
+
+    return tagset

templates/photo.html

@@ -122,8 +122,9 @@
         {% if photo.duration %}
             <li>Duration: {{photo.duration_string()}}</li>
         {% endif %}
-            <li><a href="/file/{{photo.id}}.{{photo.extension}}?download=1">Download as {{photo.id}}.{{photo.extension}}</a></li>
+        {% set extension= "." + photo.extension if photo.extension != "" else "" %}
-            <li><a href="/file/{{photo.id}}.{{photo.extension}}?download=1&original_filename=1">Download as "{{photo.basename}}"</a></li>
+            <li><a href="/file/{{photo.id}}{{extension}}?download=1">Download as {{photo.id}}.{{photo.extension}}</a></li>
+            <li><a href="/file/{{photo.id}}{{extension}}?download=1&original_filename=1">Download as "{{photo.basename}}"</a></li>
         </ul>
 
         <!-- CONTAINING ALBUMS -->

templates/search.html

@@ -120,8 +120,8 @@ form
 <body>
     {{header.make_header(session=session)}}
     <div id="error_message_area">
-        {% for warn in warns %}
+        {% for warning in warnings %}
-        <span class="search_warning">{{warn}}</span>
+        <span class="search_warning">{{warning}}</span>
         {% endfor %}
     </div>
     <div id="content_body">
@@ -131,6 +131,7 @@ form
             <span>Tag {{tagtype}}:</span>
             <ul class="search_builder_tagger">
                 {% set key="tag_" + tagtype %}
+                {% if search_kwargs[key] %}
                 {% for tagname in search_kwargs[key] %}
                 <li class="search_builder_{{tagtype}}_inputted">
                     <span class="tag_object">{{tagname}}</span>
@@ -138,6 +139,7 @@ form
                     onclick="remove_searchtag(this, '{{tagname}}', inputted_{{tagtype}});"></button>
                 </li>
                 {% endfor %}
+                {% endif %}
                 <li><input id="search_builder_{{tagtype}}_input" type="text"></li>
             </ul>
         </div>
@@ -155,7 +157,7 @@ form
             <ul id="search_builder_orderby_ul">
                 {% if "orderby" in search_kwargs and search_kwargs["orderby"] %}
                 {% for orderby in search_kwargs["orderby"] %}
-                    {% set column, sorter=orderby.split(" ") %}
+                    {% set column, sorter=orderby.split("-") %}
                     {{ create_orderby_li(selected_column=column, selected_sorter=sorter) }}
                 {% endfor %}
                 {% else %}
@@ -338,9 +340,9 @@ function add_new_orderby()
 {
     /* Called by the green + button */
     var ul = document.getElementById("search_builder_orderby_ul");
-    if (ul.children.length >= 8)
+    if (ul.children.length >= 9)
     {
-        /* 8 because there are only 8 sortable properties */
+        /* 9 because there are only 9 sortable properties */
         return;
     }
     var li = ul.children;
@@ -508,9 +510,11 @@ var inputted_mays = [];
 var inputted_forbids = [];
 {% for tagtype in ["musts", "mays", "forbids"] %}
     {% set key="tag_" + tagtype %}
+    {% if search_kwargs[key] %}
     {% for tagname in search_kwargs[key] %}
         inputted_{{tagtype}}.push("{{tagname}}");
     {% endfor %}
+    {% endif %}
 {% endfor %}
 
 /* Assign the click handler to "Tags on this page" results. */