From 0d0431edff9ff6c1701851098e976fc4cf64aad5 Mon Sep 17 00:00:00 2001 From: Ethan Dalool Date: Sat, 24 Dec 2016 17:13:45 -0800 Subject: [PATCH] Centralize and unify search parameter normalization Create searchhelpers.py which normalize incoming search parameters. Add argument give_back_parameters so we can more effectively return the normalized parameters to the user to learn from their mistakes. Create objects.WarningBag and stop using the warnings module with the unclear warning-catcher. Fix handling of photos without extensions (still needs improvement). Adopt use of pathclass.Path in more places and keep thumbnail paths relative so that the entire _etiquette dir can be moved and still work right away. Probably some other things --- constants.py | 4 +- etiquette.py | 152 ++++++++++------------ helpers.py | 102 +-------------- objects.py | 15 ++- phototagger.py | 202 ++++++++++++++++++----------- searchhelpers.py | 292 ++++++++++++++++++++++++++++++++++++++++++ templates/photo.html | 5 +- templates/search.html | 14 +- 8 files changed, 518 insertions(+), 268 deletions(-) create mode 100644 searchhelpers.py diff --git a/constants.py b/constants.py index e81a5a5..276228f 100644 --- a/constants.py +++ b/constants.py @@ -102,8 +102,10 @@ ERROR_RECURSIVE_GROUPING = 'Recursive grouping' WARNING_MINMAX_INVALID = 'Field "{field}": "{value}" is not a valid request. Ignored.' WARNING_MINMAX_OOO = 'Field "{field}": minimum "{min}" maximum "{max}" are out of order. Ignored.' WARNING_NO_SUCH_TAG = 'Tag "{tag}" does not exist. Ignored.' +WARNING_NO_SUCH_USER = 'User "{username}" does not exist. Ignored.' +WARNING_ORDERBY_INVALID = 'Invalid orderby request "{request}". Ignored.' WARNING_ORDERBY_BADCOL = '"{column}" is not a sorting option. Ignored.' -WARNING_ORDERBY_BADSORTER = 'You can\'t order "{column}" by "{sorter}". Defaulting to descending.' +WARNING_ORDERBY_BADDIRECTION = 'You can\'t order "{column}" by "{direction}". Defaulting to descending.' # Operational info EXPRESSION_OPERATORS = {'(', ')', 'OR', 'AND', 'NOT'} diff --git a/etiquette.py b/etiquette.py index 9b719ba..c7cce00 100644 --- a/etiquette.py +++ b/etiquette.py @@ -12,7 +12,9 @@ import decorators import exceptions import helpers import jsonify +import objects import phototagger +import searchhelpers import sessions site = flask.Flask(__name__) @@ -352,7 +354,9 @@ def get_file(photoid): if use_original_filename: download_as = photo.basename else: - download_as = photo.id + '.' + photo.extension + download_as = photo.id + if photo.extension: + download_as += photo.extension download_as = download_as.replace('"', '\\"') response = flask.make_response(send_file(photo.real_filepath)) @@ -382,77 +386,41 @@ def get_photo_json(photoid): return photo def get_search_core(): - #print(request.args) + warning_bag = objects.WarningBag() - # FILENAME & EXTENSION - filename_terms = request.args.get('filename', None) - extension_string = request.args.get('extension', None) - extension_not_string = request.args.get('extension_not', None) - mimetype_string = request.args.get('mimetype', None) + has_tags = request.args.get('has_tags') + tag_musts = request.args.get('tag_musts') + tag_mays = request.args.get('tag_mays') + tag_forbids = request.args.get('tag_forbids') + tag_expression = request.args.get('tag_expression') - extension_list = helpers.comma_split(extension_string) - extension_not_list = helpers.comma_split(extension_not_string) - mimetype_list = helpers.comma_split(mimetype_string) + filename_terms = request.args.get('filename') + extension = request.args.get('extension') + extension_not = request.args.get('extension_not') + mimetype = request.args.get('mimetype') - # LIMIT - limit = request.args.get('limit', '') - if limit.isdigit(): - limit = int(limit) - limit = min(100, limit) - else: - # Note to self: also apply to search.html template url builder. + limit = request.args.get('limit') + # This is being pre-processed because the site enforces a maximum value + # which the PhotoDB api does not. + limit = searchhelpers.normalize_limit(limit, warning_bag=warning_bag) + + if limit is None: limit = 50 - - # OFFSET - offset = request.args.get('offset', '') - if offset.isdigit(): - offset = int(offset) else: - offset = None + limit = min(limit, 100) - # MUSTS, MAYS, FORBIDS - qualname_map = P.export_tags(exporter=phototagger.tag_export_qualname_map) - tag_musts = request.args.get('tag_musts', '').split(',') - tag_mays = request.args.get('tag_mays', '').split(',') - tag_forbids = request.args.get('tag_forbids', '').split(',') - tag_expression = request.args.get('tag_expression', None) + offset = request.args.get('offset') - tag_musts = [qualname_map.get(tag, tag) for tag in tag_musts if tag != ''] - tag_mays = [qualname_map.get(tag, tag) for tag in tag_mays if tag != ''] - tag_forbids = [qualname_map.get(tag, tag) for tag in tag_forbids if tag != ''] + authors = request.args.get('author') - # AUTHOR - authors = request.args.get('author', None) - if authors: - authors = authors.split(',') - authors = [a.strip() for a in authors] - authors = [P.get_user(username=a) for a in authors] - else: - authors = None - - # ORDERBY - orderby = request.args.get('orderby', None) - if orderby: - orderby = orderby.replace('-', ' ') - orderby = orderby.split(',') - else: - orderby = None - - # HAS_TAGS - has_tags = request.args.get('has_tags', None) - if has_tags: - has_tags = helpers.truthystring(has_tags) - else: - has_tags = None - - # MINMAXERS - area = request.args.get('area', None) - width = request.args.get('width', None) - height = request.args.get('height', None) - ratio = request.args.get('ratio', None) - bytes = request.args.get('bytes', None) - duration = request.args.get('duration', None) - created = request.args.get('created', None) + orderby = request.args.get('orderby') + area = request.args.get('area') + width = request.args.get('width') + height = request.args.get('height') + ratio = request.args.get('ratio') + bytes = request.args.get('bytes') + duration = request.args.get('duration') + created = request.args.get('created') # These are in a dictionary so I can pass them to the page template. search_kwargs = { @@ -465,11 +433,11 @@ def get_search_core(): 'authors': authors, 'created': created, - 'extension': extension_list, - 'extension_not': extension_not_list, + 'extension': extension, + 'extension_not': extension_not, 'filename': filename_terms, 'has_tags': has_tags, - 'mimetype': mimetype_list, + 'mimetype': mimetype, 'tag_musts': tag_musts, 'tag_mays': tag_mays, 'tag_forbids': tag_forbids, @@ -479,13 +447,35 @@ def get_search_core(): 'offset': offset, 'orderby': orderby, - 'warn_bad_tags': True, + 'warning_bag': warning_bag, + 'give_back_parameters': True } #print(search_kwargs) - with warnings.catch_warnings(record=True) as catcher: - photos = list(P.search(**search_kwargs)) - warns = [str(warning.message) for warning in catcher] - #print(warns) + search_generator = P.search(**search_kwargs) + # Because of the giveback, first element is cleaned up kwargs + search_kwargs = next(search_generator) + + # The search has converted many arguments into sets or other types. + # Convert them back into something that will display nicely on the search form. + join_helper = lambda x: ', '.join(x) if x else None + tagname_helper = lambda tags: [tag.qualified_name() for tag in tags] if tags else None + filename_helper = lambda fn: ' '.join('"%s"' % part if ' ' in part else part for part in fn) if fn else None + search_kwargs['extension'] = join_helper(search_kwargs['extension']) + search_kwargs['extension_not'] = join_helper(search_kwargs['extension_not']) + search_kwargs['mimetype'] = join_helper(search_kwargs['mimetype']) + search_kwargs['filename'] = filename_helper(search_kwargs['filename']) + search_kwargs['tag_musts'] = tagname_helper(search_kwargs['tag_musts']) + search_kwargs['tag_mays'] = tagname_helper(search_kwargs['tag_mays']) + search_kwargs['tag_forbids'] = tagname_helper(search_kwargs['tag_forbids']) + + search_results = list(search_generator) + warnings = set() + photos = [] + for item in search_results: + if isinstance(item, objects.WarningBag): + warnings.update(item.warnings) + else: + photos.append(item) # TAGS ON THIS PAGE total_tags = set() @@ -510,18 +500,14 @@ def get_search_core(): view = request.args.get('view', 'grid') search_kwargs['view'] = view - search_kwargs['extension'] = extension_string - search_kwargs['extension_not'] = extension_not_string - search_kwargs['mimetype'] = mimetype_string final_results = { 'next_page_url': next_page_url, 'prev_page_url': prev_page_url, 'photos': photos, 'total_tags': total_tags, - 'warns': warns, + 'warnings': list(warnings), 'search_kwargs': search_kwargs, - 'qualname_map': qualname_map, } return final_results @@ -530,7 +516,7 @@ def get_search_core(): def get_search_html(): search_results = get_search_core() search_kwargs = search_results['search_kwargs'] - qualname_map = search_results['qualname_map'] + qualname_map = P.export_tags(exporter=phototagger.tag_export_qualname_map) session = session_manager.get(request) response = flask.render_template( 'search.html', @@ -541,7 +527,7 @@ def get_search_html(): search_kwargs=search_kwargs, session=session, total_tags=search_results['total_tags'], - warns=search_results['warns'], + warnings=search_results['warnings'], ) return response @@ -550,17 +536,11 @@ def get_search_html(): def get_search_json(): search_results = get_search_core() search_results['photos'] = [jsonify.photo(photo, include_albums=False) for photo in search_results['photos']] - #search_kwargs = search_results['search_kwargs'] - #qualname_map = search_results['qualname_map'] - include_qualname_map = request.args.get('include_map', False) - include_qualname_map = helpers.truthystring(include_qualname_map) - if not include_qualname_map: - search_results.pop('qualname_map') return jsonify.make_json_response(search_results) @site.route('/static/') -def get_static(filename): +def geft_static(filename): filename = filename.replace('\\', os.sep) filename = filename.replace('/', os.sep) filename = os.path.join('static', filename) diff --git a/helpers.py b/helpers.py index 99d114e..9c1da2a 100644 --- a/helpers.py +++ b/helpers.py @@ -2,7 +2,6 @@ import datetime import math import mimetypes import os -import warnings import constants import exceptions @@ -192,6 +191,9 @@ def is_xor(*args): ''' return [bool(a) for a in args].count(True) == 1 +def normalize_extension(extension): + pass + def normalize_filepath(filepath, allowed=''): ''' Remove some bad characters. @@ -283,102 +285,6 @@ def truthystring(s): return None return False -#=============================================================================== - -def _minmax(key, value, minimums, maximums): - ''' - When searching, this function dissects a hyphenated range string - and inserts the correct k:v pair into both minimums and maximums. - ('area', '100-200', {}, {}) --> {'area': 100}, {'area': 200} (MODIFIED IN PLACE) - ''' - if value is None: - return - if isinstance(value, (int, float)): - minimums[key] = value - return - try: - (low, high) = hyphen_range(value) - except ValueError: - warnings.warn(constants.WARNING_MINMAX_INVALID.format(field=key, value=value)) - return - except exceptions.OutOfOrder as e: - warnings.warn(constants.WARNING_MINMAX_OOO.format(field=key, min=e.args[1], max=e.args[2])) - return - if low is not None: - minimums[key] = low - if high is not None: - maximums[key] = high - -def _normalize_extensions(extensions): - ''' - When searching, this function normalizes the list of inputted extensions. - ''' - if isinstance(extensions, str): - extensions = extensions.split() - if extensions is None: - return set() - extensions = [e.lower().strip('.').strip() for e in extensions] - extensions = set(e for e in extensions if e) - return extensions - -def _orderby(orderby): - ''' - When searching, this function ensures that the user has entered a valid orderby - query, and normalizes the query text. - - 'random asc' --> ('random', 'asc') - 'area' --> ('area', 'desc') - ''' - orderby = orderby.lower().strip() - if orderby == '': - return None - - orderby = orderby.split(' ') - if len(orderby) == 2: - (column, sorter) = orderby - elif len(orderby) == 1: - column = orderby[0] - sorter = 'desc' - else: - return None - - #print(column, sorter) - if column not in constants.ALLOWED_ORDERBY_COLUMNS: - warnings.warn(constants.WARNING_ORDERBY_BADCOL.format(column=column)) - return None - if column == 'random': - column = 'RANDOM()' - - if sorter not in ['desc', 'asc']: - warnings.warn(constants.WARNING_ORDERBY_BADSORTER.format(column=column, sorter=sorter)) - sorter = 'desc' - return (column, sorter) - -def _setify_tags(photodb, tags, warn_bad_tags=False): - ''' - When searching, this function converts the list of tag strings that the user - requested into Tag objects. If a tag doesn't exist we'll either raise an exception - or just issue a warning. - ''' - if tags is None: - return set() - - tagset = set() - for tag in tags: - tag = tag.strip() - if tag == '': - continue - try: - tag = photodb.get_tag(tag) - tagset.add(tag) - except exceptions.NoSuchTag: - if warn_bad_tags: - warnings.warn(constants.WARNING_NO_SUCH_TAG.format(tag=tag)) - continue - else: - raise - - return tagset def _unitconvert(value): ''' @@ -392,4 +298,4 @@ def _unitconvert(value): elif all(c in '0123456789.' for c in value): return float(value) else: - return bytestring.parsebytes(value) + return bytestring.parsebytes(value) \ No newline at end of file diff --git a/objects.py b/objects.py index ebb509a..818bc80 100644 --- a/objects.py +++ b/objects.py @@ -392,6 +392,8 @@ class Photo(ObjectBase): For videos, you can provide a `timestamp` to take the thumbnail from. ''' hopeful_filepath = self.make_thumbnail_filepath() + hopeful_filepath = hopeful_filepath.relative_path + #print(hopeful_filepath) return_filepath = None if self.mimetype == 'image': @@ -490,10 +492,10 @@ class Photo(ObjectBase): basename = chunked_id[-1] folder = chunked_id[:-1] folder = os.sep.join(folder) - folder = os.path.join(self.photodb.thumbnail_directory, folder) + folder = self.photodb.thumbnail_directory.join(folder) if folder: - os.makedirs(folder, exist_ok=True) - hopeful_filepath = os.path.join(folder, basename) + '.jpg' + os.makedirs(folder.absolute_path, exist_ok=True) + hopeful_filepath = folder.with_child(basename + '.jpg') return hopeful_filepath @decorators.time_me @@ -828,3 +830,10 @@ class User(ObjectBase): def __str__(self): rep = 'User:{username}'.format(username=self.username) return rep + +class WarningBag: + def __init__(self): + self.warnings = set() + + def add(self, warning): + self.warnings.add(warning) diff --git a/phototagger.py b/phototagger.py index 8c0442c..bf98ee6 100644 --- a/phototagger.py +++ b/phototagger.py @@ -8,16 +8,18 @@ import random import sqlite3 import string import time -import warnings import constants import decorators import exceptions import helpers import objects +import searchhelpers # pip install # https://raw.githubusercontent.com/voussoir/else/master/_voussoirkit/voussoirkit.zip +from voussoirkit import pathclass +from voussoirkit import safeprint from voussoirkit import spinal @@ -151,7 +153,7 @@ def raise_no_such_thing(exception_class, thing_id=None, thing_name=None, comment message = '' raise exception_class(message) -def searchfilter_expression(photo_tags, expression, frozen_children, token_normalizer, warn_bad_tags): +def searchfilter_expression(photo_tags, expression, frozen_children, token_normalizer, warning_bag=None): photo_tags = set(tag.name for tag in photo_tags) operator_stack = collections.deque() operand_stack = collections.deque() @@ -178,8 +180,8 @@ def searchfilter_expression(photo_tags, expression, frozen_children, token_norma token = token_normalizer(token) value = any(option in photo_tags for option in frozen_children[token]) except KeyError: - if warn_bad_tags: - warnings.warn(constants.WARNING_NO_SUCH_TAG.format(tag=token)) + if warning_bag: + warning_bag.add(constants.WARNING_NO_SUCH_TAG.format(tag=token)) else: raise exceptions.NoSuchTag(token) return False @@ -218,7 +220,8 @@ def searchfilter_expression(photo_tags, expression, frozen_children, token_norma while len(operand_stack) > 1 or len(operator_stack) > 0: operate(operand_stack, operator_stack) #print(operand_stack) - return operand_stack.pop() + success = operand_stack.pop() + return success def searchfilter_must_may_forbid(photo_tags, tag_musts, tag_mays, tag_forbids, frozen_children): if tag_musts and not all(any(option in photo_tags for option in frozen_children[must]) for must in tag_musts): @@ -434,7 +437,10 @@ class PDBPhotoMixin: Returns the Photo object. ''' filename = os.path.abspath(filename) - assert os.path.isfile(filename) + safeprint.safeprint('Processing %s' % filename) + if not os.path.isfile(filename): + raise FileNotFoundError(filename) + if not allow_duplicates: try: existing = self.get_photo_by_path(filename) @@ -457,7 +463,7 @@ class PDBPhotoMixin: extension = os.path.splitext(filename)[1] extension = extension.replace('.', '') - extension = self.normalize_tagname(extension) + #extension = self.normalize_tagname(extension) created = int(helpers.now()) photoid = self.generate_id('photos') @@ -538,10 +544,11 @@ class PDBPhotoMixin: tag_forbids=None, tag_expression=None, - warn_bad_tags=False, limit=None, offset=None, - orderby=None + orderby=None, + warning_bag=None, + give_back_parameters=False ): ''' PHOTO PROPERTIES @@ -590,10 +597,6 @@ class PDBPhotoMixin: Can NOT be used with the must, may, forbid style search. QUERY OPTIONS - warn_bad_tags: - If a tag is not found, issue a warning but continue the search. - Otherwise, a exceptions.NoSuchTag exception would be raised. - limit: The maximum number of *successful* results to yield. @@ -604,57 +607,99 @@ class PDBPhotoMixin: A list of strings like ['ratio DESC', 'created ASC'] to sort and subsort the results. Descending is assumed if not provided. + + warning_bag: + Invalid search queries will add a warning to the bag and try their best to continue. + Otherwise they may raise exceptions. + + give_back_parameters: + If True, the generator's first yield will be a dictionary of all the cleaned up, normalized + parameters. The user may have given us loads of trash, so we should show them the formatting + we want. ''' start_time = time.time() + + # MINMAXERS + + has_tags = searchhelpers.normalize_has_tags(has_tags) + if has_tags is False: + tag_musts = None + tag_mays = None + tag_forbids = None + tag_expression = None + else: + tag_musts = searchhelpers.normalize_tag_mmf(photodb=self, tags=tag_musts, warning_bag=warning_bag) + tag_mays = searchhelpers.normalize_tag_mmf(photodb=self, tags=tag_mays, warning_bag=warning_bag) + tag_forbids = searchhelpers.normalize_tag_mmf(photodb=self, tags=tag_forbids, warning_bag=warning_bag) + tag_expression = searchhelpers.normalize_tag_expression(tag_expression) + + #print(tag_musts, tag_mays, tag_forbids) + if (tag_musts or tag_mays or tag_forbids) and tag_expression: + message = 'Expression filter cannot be used with musts, mays, forbids' + if warning_bag: + warning_bag.add(message) + tag_musts = None + tag_mays = None + tag_forbids = None + tag_expression = None + else: + raise exceptions.NotExclusive(message) + + extension = searchhelpers.normalize_extensions(extension) + extension_not = searchhelpers.normalize_extensions(extension_not) + mimetype = searchhelpers.normalize_extensions(mimetype) + + authors = searchhelpers.normalize_authors(authors, photodb=self, warning_bag=warning_bag) + + filename = searchhelpers.normalize_filename(filename) + + limit = searchhelpers.normalize_limit(limit, warning_bag=warning_bag) + + offset = searchhelpers.normalize_offset(offset) + if offset is None: + offset = 0 + maximums = {} minimums = {} - helpers._minmax('area', area, minimums, maximums) - helpers._minmax('created', created, minimums, maximums) - helpers._minmax('width', width, minimums, maximums) - helpers._minmax('height', height, minimums, maximums) - helpers._minmax('ratio', ratio, minimums, maximums) - helpers._minmax('bytes', bytes, minimums, maximums) - helpers._minmax('duration', duration, minimums, maximums) - orderby = orderby or [] + searchhelpers.minmax('area', area, minimums, maximums, warning_bag=warning_bag) + searchhelpers.minmax('created', created, minimums, maximums, warning_bag=warning_bag) + searchhelpers.minmax('width', width, minimums, maximums, warning_bag=warning_bag) + searchhelpers.minmax('height', height, minimums, maximums, warning_bag=warning_bag) + searchhelpers.minmax('ratio', ratio, minimums, maximums, warning_bag=warning_bag) + searchhelpers.minmax('bytes', bytes, minimums, maximums, warning_bag=warning_bag) + searchhelpers.minmax('duration', duration, minimums, maximums, warning_bag=warning_bag) - extension = helpers._normalize_extensions(extension) - extension_not = helpers._normalize_extensions(extension_not) - mimetype = helpers._normalize_extensions(mimetype) - - if authors is not None: - if isinstance(authors, str): - authors = {authors, } - authors = set(a.id if isinstance(a, objects.User) else a for a in authors) - - if filename is not None: - if not isinstance(filename, str): - filename = ' '.join(filename) - filename = set(term.lower() for term in filename.strip().split(' ')) - - if (tag_musts or tag_mays or tag_forbids) and tag_expression: - raise exceptions.NotExclusive('Expression filter cannot be used with musts, mays, forbids') - - tag_musts = helpers._setify_tags(photodb=self, tags=tag_musts, warn_bad_tags=warn_bad_tags) - tag_mays = helpers._setify_tags(photodb=self, tags=tag_mays, warn_bad_tags=warn_bad_tags) - tag_forbids = helpers._setify_tags(photodb=self, tags=tag_forbids, warn_bad_tags=warn_bad_tags) - - query = 'SELECT * FROM photos' - orderby = [helpers._orderby(o) for o in orderby] - orderby = [o for o in orderby if o] - if orderby: - whereable_columns = [o[0] for o in orderby if o[0] != 'RANDOM()'] - whereable_columns = [column + ' IS NOT NULL' for column in whereable_columns] - if whereable_columns: - query += ' WHERE ' - query += ' AND '.join(whereable_columns) - orderby = [' '.join(o) for o in orderby] - orderby = ', '.join(orderby) - query += ' ORDER BY %s' % orderby - else: - query += ' ORDER BY created DESC' + orderby = searchhelpers.normalize_orderby(orderby) + query = searchhelpers.build_query(orderby) print(query) generator = helpers.select_generator(self.sql, query) + if give_back_parameters: + parameters = { + 'area': area, + 'width': width, + 'height': height, + 'ratio': ratio, + 'bytes': bytes, + 'duration': duration, + 'authors': authors, + 'created': created, + 'extension': extension, + 'extension_not': extension_not, + 'filename': filename, + 'has_tags': has_tags, + 'mimetype': mimetype, + 'tag_musts': tag_musts, + 'tag_mays': tag_mays, + 'tag_forbids': tag_forbids, + 'tag_expression': tag_expression, + 'limit': limit, + 'offset': offset, + 'orderby': orderby, + } + yield parameters + + # FROZEN CHILDREN # To lighten the amount of database reading here, `frozen_children` is a dict where # EVERY tag in the db is a key, and the value is a list of ALL ITS NESTED CHILDREN. # This representation is memory inefficient, but it is faster than repeated @@ -669,6 +714,7 @@ class PDBPhotoMixin: self._cached_frozen_children = frozen_children photos_received = 0 + # LET'S GET STARTED for fetch in generator: photo = objects.Photo(self, fetch) @@ -685,6 +731,7 @@ class PDBPhotoMixin: continue if authors and photo.author_id not in authors: + #print('Failed author') continue if filename and not _helper_filenamefilter(subject=photo.basename, terms=filename): @@ -709,9 +756,11 @@ class PDBPhotoMixin: photo_tags = photo.tags() if has_tags is False and len(photo_tags) > 0: + #print('Failed has_tags=False') continue if has_tags is True and len(photo_tags) == 0: + #print('Failed has_tags=True') continue photo_tags = set(photo_tags) @@ -722,9 +771,10 @@ class PDBPhotoMixin: expression=tag_expression, frozen_children=frozen_children, token_normalizer=self.normalize_tagname, - warn_bad_tags=warn_bad_tags, + warning_bag=warning_bag, ) if not success: + #print('Failed tag expression') continue elif is_must_may_forbid: success = searchfilter_must_may_forbid( @@ -735,18 +785,23 @@ class PDBPhotoMixin: frozen_children=frozen_children, ) if not success: + #print('Failed tag mmf') continue - if offset is not None and offset > 0: + if offset > 0: offset -= 1 continue if limit is not None and photos_received >= limit: break + photos_received += 1 yield photo + if warning_bag.warnings: + yield warning_bag + end_time = time.time() print(end_time - start_time) @@ -988,13 +1043,13 @@ class PhotoDB(PDBAlbumMixin, PDBPhotoMixin, PDBTagMixin, PDBUserMixin): # DATA DIR PREP data_directory = helpers.normalize_filepath(data_directory, allowed='/\\') - self.data_directory = os.path.abspath(data_directory) - os.makedirs(self.data_directory, exist_ok=True) + self.data_directory = pathclass.Path(data_directory) + os.makedirs(self.data_directory.absolute_path, exist_ok=True) # DATABASE - self.database_abspath = os.path.join(self.data_directory, 'phototagger.db') - existing_database = os.path.exists(self.database_abspath) - self.sql = sqlite3.connect(self.database_abspath) + self.database_file = self.data_directory.with_child('phototagger.db') + existing_database = self.database_file.exists + self.sql = sqlite3.connect(self.database_file.absolute_path) self.cur = self.sql.cursor() if existing_database: @@ -1010,21 +1065,20 @@ class PhotoDB(PDBAlbumMixin, PDBPhotoMixin, PDBTagMixin, PDBUserMixin): self.cur.execute(statement) # CONFIG - self.config_abspath = os.path.join(self.data_directory, 'config.json') + self.config_file = self.data_directory.with_child('config.json') self.config = copy.deepcopy(constants.DEFAULT_CONFIGURATION) - if os.path.isfile(self.config_abspath): - with open(self.config_abspath, 'r') as handle: + if self.config_file.is_file: + with open(self.config_file.absolute_path, 'r') as handle: user_config = json.load(handle) self.config.update(user_config) else: - with open(self.config_abspath, 'w') as handle: + with open(self.config_file.absolute_path, 'w') as handle: handle.write(json.dumps(self.config, indent=4, sort_keys=True)) #print(self.config) # THUMBNAIL DIRECTORY - self.thumbnail_directory = os.path.join(self.data_directory, 'site_thumbnails') - self.thumbnail_directory = os.path.abspath(self.thumbnail_directory) - os.makedirs(self.thumbnail_directory, exist_ok=True) + self.thumbnail_directory = self.data_directory.with_child('site_thumbnails') + os.makedirs(self.thumbnail_directory.absolute_path, exist_ok=True) # OTHER self.log = logging.getLogger(__name__) @@ -1095,14 +1149,16 @@ class PhotoDB(PDBAlbumMixin, PDBPhotoMixin, PDBTagMixin, PDBUserMixin): commit=False, title=current_location.basename, ) - print('Created %s' % current_album.title) + safeprint.safeprint('Created %s' % current_album.title) albums[current_location.absolute_path] = current_album - parent = albums[current_location.parent.absolute_path] + + parent = albums.get(current_location.parent.absolute_path, None) + if parent is not None: try: parent.add(current_album, commit=False) + #safeprint.safeprint('Added to %s' % parent.title) except exceptions.GroupExists: pass - #print('Added to %s' % parent.title) for filepath in files: try: photo = self.new_photo(filepath.absolute_path, commit=False) diff --git a/searchhelpers.py b/searchhelpers.py new file mode 100644 index 0000000..a1bcf62 --- /dev/null +++ b/searchhelpers.py @@ -0,0 +1,292 @@ +import shlex + +import constants +import exceptions +import helpers +import objects + +def build_query(orderby): + query = 'SELECT * FROM photos' + if not orderby: + query += ' ORDER BY created DESC' + return query + + orderby = [o.split('-') for o in orderby] + whereable_columns = [column for (column, sorter) in orderby if column != 'RANDOM()'] + if whereable_columns: + query += ' WHERE ' + whereable_columns = [column + ' IS NOT NULL' for column in whereable_columns] + query += ' AND '.join(whereable_columns) + + # Combine each column+sorter + orderby = [' '.join(o) for o in orderby] + + # Combine everything + orderby = ', '.join(orderby) + query += ' ORDER BY %s' % orderby + return query + +def minmax(key, value, minimums, maximums, warning_bag=None): + ''' + Dissects a hyphenated range string and inserts the correct k:v pair into + both minimums and maximums. + ('area', '100-200', {}, {}) --> {'area': 100}, {'area': 200} (MODIFIED IN PLACE) + ''' + if value is None: + return + + if isinstance(value, str): + value = value.strip() + + if value == '': + return + + if isinstance(value, (int, float)): + minimums[key] = value + return + + try: + (low, high) = helpers.hyphen_range(value) + + except ValueError as e: + if warning_bag: + warning_bag.add(constants.WARNING_MINMAX_INVALID.format(field=key, value=value)) + return + else: + raise e + + except exceptions.OutOfOrder as e: + if warning_bag: + warning_bag.add(constants.WARNING_MINMAX_OOO.format(field=key, min=e.args[1], max=e.args[2])) + return + else: + raise e + + if low is not None: + minimums[key] = low + + if high is not None: + maximums[key] = high + +def normalize_authors(authors, photodb, warning_bag=None): + if not authors: + return None + + if isinstance(authors, str): + authors = helpers.comma_split(authors) + + user_ids = set() + for requested_author in authors: + if isinstance(requested_author, objects.User): + if requested_author.photodb == photodb: + user_ids.add(requested_author.id) + else: + requested_author = requested_author.username + + try: + user = photodb.get_user(username=requested_author) + except exceptions.NoSuchUser: + if warning_bag: + warning_bag.add(constants.WARNING_NO_SUCH_USER.format(username=requested_author)) + else: + raise + else: + user_ids.add(user.id) + + if len(user_ids) == 0: + return None + + return user_ids + +def normalize_extensions(extensions): + if not extensions: + return None + + if isinstance(extensions, str): + extensions = helpers.comma_split(extensions) + + if len(extensions) == 0: + return None + + extensions = [e.lower().strip('.').strip() for e in extensions] + extensions = set(extensions) + extensions = {e for e in extensions if e} + + if len(extensions) == 0: + return None + + return extensions + +def normalize_filename(filename_terms): + if not filename_terms: + return None + + if not isinstance(filename_terms, str): + filename_terms = ' '.join(filename_terms) + + filename_terms = filename_terms.strip() + filename_terms = [term.lower() for term in shlex.split(filename_terms)] + + if not filename_terms: + return None + + return filename_terms + +def normalize_has_tags(has_tags): + if not has_tags: + return None + + if isinstance(has_tags, str): + return helpers.truthystring(has_tags) + + if isinstance(has_tags, int): + return bool(has_tags) + + return None + +def normalize_limit(limit, warning_bag=None): + if not limit and limit != 0: + return None + + if isinstance(limit, str): + limit = limit.strip() + if limit.isdigit(): + limit = int(limit) + + if isinstance(limit, float): + limit = int(limit) + + if not isinstance(limit, int): + message = 'Invalid limit "%s%"' % limit + if warning_bag: + warning_bag.add(message) + limit = None + else: + raise ValueError(message) + + return limit + +def normalize_offset(offset, warning_bag=None): + if not offset: + return None + + if isinstance(offset, str): + offset = offset.strip() + if offset.isdigit(): + offset = int(offset) + + if isinstance(offset, float): + offset = int(offset) + + if not isinstance(offset, int): + message = 'Invalid offset "%s%"' % offset + if warning_bag: + warning_bag.add(message) + offset = None + else: + raise ValueError(message) + + return offset + + +def normalize_orderby(orderby, warning_bag=None): + if not orderby: + return None + + if isinstance(orderby, str): + orderby = orderby.replace('-', ' ') + orderby = orderby.split(',') + + if not orderby: + return None + + final_orderby = [] + for requested_order in orderby: + requested_order = requested_order.lower().strip() + if not requested_order: + continue + + split_order = requested_order.split(' ') + if len(split_order) == 2: + (column, direction) = split_order + + elif len(split_order) == 1: + column = split_order[0] + direction = 'desc' + + else: + message = constants.WARNING_ORDERBY_INVALID.format(requested=requested_order) + if warning_bag: + warning_bag.add(message) + else: + raise ValueError(message) + continue + + if column not in constants.ALLOWED_ORDERBY_COLUMNS: + message = constants.WARNING_ORDERBY_BADCOL.format(column=column) + if warning_bag: + warning_bag.add(message) + else: + raise ValueError(message) + continue + + if column == 'random': + column = 'RANDOM()' + + if direction not in ('asc', 'desc'): + message = constants.WARNING_ORDERBY_BADDIRECTION.format(column=column, direction=direction) + if warning_bag: + warning_bag.add(message) + else: + raise ValueError(message) + direction = 'desc' + + requested_order = '%s-%s' % (column, direction) + final_orderby.append(requested_order) + + return final_orderby + +def normalize_tag_expression(expression): + if not expression: + return None + + if not isinstance(expression, str): + expression = ' '.join(expression) + + return expression + +def normalize_tag_mmf(tags, photodb, warning_bag=None): + if not tags: + return None + + if isinstance(tags, str): + tags = helpers.comma_split(tags) + + tagset = set() + for tag in tags: + if isinstance(tag, objects.Tag): + if tag.photodb == photodb: + tagset.add(tag) + continue + else: + tag = tag.name + + tag = tag.strip() + if tag == '': + continue + tag = tag.split('.')[-1] + + try: + tag = photodb.get_tag(name=tag) + except exceptions.NoSuchTag: + if warning_bag: + warning_bag.add(constants.WARNING_NO_SUCH_TAG.format(tag=tag)) + continue + else: + raise + tagset.add(tag) + + if len(tagset) == 0: + return None + + return tagset diff --git a/templates/photo.html b/templates/photo.html index 708df23..5222937 100644 --- a/templates/photo.html +++ b/templates/photo.html @@ -122,8 +122,9 @@ {% if photo.duration %}
  • Duration: {{photo.duration_string()}}
    • {% endif %} -
  • Download as {{photo.id}}.{{photo.extension}}
    • -
  • Download as "{{photo.basename}}"
    • + {% set extension= "." + photo.extension if photo.extension != "" else "" %} +
  • Download as {{photo.id}}.{{photo.extension}}
    • +
  • Download as "{{photo.basename}}"
    • diff --git a/templates/search.html b/templates/search.html index 3278801..3872f6d 100644 --- a/templates/search.html +++ b/templates/search.html @@ -120,8 +120,8 @@ form {{header.make_header(session=session)}}
    - {% for warn in warns %} - {{warn}} + {% for warning in warnings %} + {{warning}} {% endfor %}
    @@ -131,6 +131,7 @@ form Tag {{tagtype}}:
      {% set key="tag_" + tagtype %} + {% if search_kwargs[key] %} {% for tagname in search_kwargs[key] %}
    • {{tagname}} @@ -138,6 +139,7 @@ form onclick="remove_searchtag(this, '{{tagname}}', inputted_{{tagtype}});">
      • {% endfor %} + {% endif %}
    @@ -155,7 +157,7 @@ form