From 57f1b8044204f653fb4d5ae8b35c259b0bce8d74 Mon Sep 17 00:00:00 2001 From: Ethan Dalool Date: Sat, 13 Aug 2022 18:08:45 -0700 Subject: [PATCH] Use SQL generated columns for area, aspectratio, basename, bitrate. --- etiquette/constants.py | 26 +++++--- etiquette/objects.py | 33 ++-------- etiquette/photodb.py | 19 +++--- etiquette/searchhelpers.py | 4 -- frontends/etiquette_cli.py | 4 +- .../backend/endpoints/photo_endpoints.py | 6 +- .../etiquette_flask/templates/photo.html | 3 +- .../etiquette_flask/templates/search.html | 2 +- utilities/database_upgrader.py | 61 +++++++++++++++++++ 9 files changed, 102 insertions(+), 56 deletions(-) diff --git a/etiquette/constants.py b/etiquette/constants.py index 20200f8..a70b32a 100644 --- a/etiquette/constants.py +++ b/etiquette/constants.py @@ -41,7 +41,7 @@ ffmpeg = _load_ffmpeg() # Database ######################################################################################### -DATABASE_VERSION = 21 +DATABASE_VERSION = 22 DB_INIT = f''' CREATE TABLE IF NOT EXISTS albums( @@ -71,15 +71,11 @@ CREATE INDEX IF NOT EXISTS index_bookmarks_author_id on bookmarks(author_id); CREATE TABLE IF NOT EXISTS photos( id INT PRIMARY KEY NOT NULL, filepath TEXT COLLATE NOCASE, - basename TEXT COLLATE NOCASE, override_filename TEXT COLLATE NOCASE, - extension TEXT COLLATE NOCASE, mtime INT, sha256 TEXT, width INT, height INT, - ratio REAL, - area INT, duration INT, bytes INT, created INT, @@ -87,12 +83,26 @@ CREATE TABLE IF NOT EXISTS photos( tagged_at INT, author_id INT, searchhidden INT, + -- GENERATED COLUMNS + area INT GENERATED ALWAYS AS (width * height) VIRTUAL, + aspectratio REAL GENERATED ALWAYS AS (1.0 * width / height) VIRTUAL, + -- Thank you ungalcrys + -- https://stackoverflow.com/a/38330814/5430534 + basename TEXT GENERATED ALWAYS AS ( + COALESCE( + override_filename, + replace(filepath, rtrim(filepath, replace(replace(filepath, '\\', '/'), '/', '')), '') + ) + ) STORED COLLATE NOCASE, + extension TEXT GENERATED ALWAYS AS ( + replace(basename, rtrim(basename, replace(basename, '.', '')), '') + ) VIRTUAL COLLATE NOCASE, + bitrate REAL GENERATED ALWAYS AS ((bytes / 128) / duration) VIRTUAL, FOREIGN KEY(author_id) REFERENCES users(id) ); CREATE INDEX IF NOT EXISTS index_photos_id on photos(id); CREATE INDEX IF NOT EXISTS index_photos_filepath on photos(filepath COLLATE NOCASE); -CREATE INDEX IF NOT EXISTS index_photos_override_filename on - photos(override_filename COLLATE NOCASE); +CREATE INDEX IF NOT EXISTS index_photos_basename on photos(basename COLLATE NOCASE); CREATE INDEX IF NOT EXISTS index_photos_created on photos(created); CREATE INDEX IF NOT EXISTS index_photos_extension on photos(extension); CREATE INDEX IF NOT EXISTS index_photos_author_id on photos(author_id); @@ -194,7 +204,7 @@ ALLOWED_ORDERBY_COLUMNS = { 'extension', 'height', 'random', - 'ratio', + 'aspectratio', 'tagged_at', 'width', } diff --git a/etiquette/objects.py b/etiquette/objects.py index 00db82b..10fa967 100644 --- a/etiquette/objects.py +++ b/etiquette/objects.py @@ -850,11 +850,11 @@ class Photo(ObjectBase): self.real_path = db_row['filepath'] self.real_path = pathclass.Path(self.real_path) + self.basename = db_row['basename'] self.id = db_row['id'] self.created_unix = db_row['created'] self._author_id = self.normalize_author_id(db_row['author_id']) - self.override_filename = db_row['override_filename'] self.extension = self.real_path.extension.no_dot self.mtime = db_row['mtime'] self.sha256 = db_row['sha256'] @@ -864,12 +864,13 @@ class Photo(ObjectBase): else: self.dot_extension = '.' + self.extension - self.area = db_row['area'] self.bytes = db_row['bytes'] self.duration = db_row['duration'] self.width = db_row['width'] self.height = db_row['height'] - self.ratio = db_row['ratio'] + self.area = db_row['area'] + self.aspectratio = db_row['aspectratio'] + self.bitrate = db_row['bitrate'] self.thumbnail = self.normalize_thumbnail(db_row['thumbnail']) self.tagged_at_unix = db_row['tagged_at'] @@ -1004,17 +1005,6 @@ class Photo(ObjectBase): return soup - @property - def basename(self) -> str: - return self.override_filename or self.real_path.basename - - @property - def bitrate(self) -> typing.Optional[float]: - if self.duration and self.bytes is not None: - return (self.bytes / 128) / self.duration - else: - return None - @property def bytes_string(self) -> str: if self.bytes is not None: @@ -1181,11 +1171,11 @@ class Photo(ObjectBase): j = { 'type': 'photo', 'id': self.id, + 'aspectratio': self.aspectratio, 'author': self.author.jsonify() if self._author_id else None, 'extension': self.extension, 'width': self.width, 'height': self.height, - 'ratio': self.ratio, 'area': self.area, 'bytes': self.bytes, 'duration_string': self.duration_string, @@ -1281,8 +1271,6 @@ class Photo(ObjectBase): self.bytes = None self.width = None self.height = None - self.area = None - self.ratio = None self.duration = None if self.real_path.is_file: @@ -1302,10 +1290,6 @@ class Photo(ObjectBase): elif self.simple_mimetype == 'audio': self._reload_audio_metadata() - if self.width and self.height: - self.area = self.width * self.height - self.ratio = round(self.width / self.height, 2) - hash_kwargs = hash_kwargs or {} sha256 = spinal.hash_file(self.real_path, hash_class=hashlib.sha256, **hash_kwargs) self.sha256 = sha256.hexdigest() @@ -1316,8 +1300,6 @@ class Photo(ObjectBase): 'sha256': self.sha256, 'width': self.width, 'height': self.height, - 'area': self.area, - 'ratio': self.ratio, 'duration': self.duration, 'bytes': self.bytes, } @@ -1352,8 +1334,6 @@ class Photo(ObjectBase): data = { 'id': self.id, 'filepath': new_filepath.absolute_path, - 'basename': new_filepath.basename, - 'extension': new_filepath.extension.no_dot, } self.photodb.update(table=Photo, pairs=data, where_key='id') self.real_path = new_filepath @@ -1456,8 +1436,6 @@ class Photo(ObjectBase): data = { 'id': self.id, 'filepath': new_path.absolute_path, - 'basename': new_path.basename, - 'extension': new_path.extension.no_dot, } self.photodb.update(table=Photo, pairs=data, where_key='id') self.real_path = new_path @@ -1494,7 +1472,6 @@ class Photo(ObjectBase): 'override_filename': new_filename, } self.photodb.update(table=Photo, pairs=data, where_key='id') - self.override_filename = new_filename self.__reinit__() diff --git a/etiquette/photodb.py b/etiquette/photodb.py index ed72351..4f2631b 100644 --- a/etiquette/photodb.py +++ b/etiquette/photodb.py @@ -357,9 +357,7 @@ class PDBPhotoMixin: data = { 'id': photo_id, 'filepath': filepath.absolute_path, - 'basename': filepath.basename, 'override_filename': None, - 'extension': filepath.extension.no_dot, 'created': helpers.now().timestamp(), 'tagged_at': None, 'author_id': author_id, @@ -370,14 +368,12 @@ class PDBPhotoMixin: 'bytes': None, 'width': None, 'height': None, - 'area': None, - 'ratio': None, 'duration': None, 'thumbnail': None, } self.insert(table=objects.Photo, pairs=data) - photo = self.get_cached_instance(objects.Photo, data) + photo = self.get_photo(photo_id) if do_metadata: hash_kwargs = hash_kwargs or {} @@ -417,11 +413,12 @@ class PDBPhotoMixin: self, *, area=None, + aspectratio=None, width=None, height=None, - ratio=None, bytes=None, duration=None, + bitrate=None, author=None, created=None, @@ -450,7 +447,7 @@ class PDBPhotoMixin: ): ''' PHOTO PROPERTIES - area, width, height, ratio, bytes, duration: + area, aspectratio, width, height, bytes, duration, bitrate: A dotdot_range string representing min and max. Or just a number for lower bound. @@ -531,7 +528,7 @@ class PDBPhotoMixin: How many *successful* results to skip before we start yielding. orderby: - A list of strings like ['ratio DESC', 'created ASC'] to sort + A list of strings like ['aspectratio DESC', 'created ASC'] to sort and subsort the results. Descending is assumed if not provided. @@ -562,9 +559,10 @@ class PDBPhotoMixin: searchhelpers.minmax('created', created, minimums, maximums, warning_bag=warning_bag) searchhelpers.minmax('width', width, minimums, maximums, warning_bag=warning_bag) searchhelpers.minmax('height', height, minimums, maximums, warning_bag=warning_bag) - searchhelpers.minmax('ratio', ratio, minimums, maximums, warning_bag=warning_bag) + searchhelpers.minmax('aspectratio', aspectratio, minimums, maximums, warning_bag=warning_bag) searchhelpers.minmax('bytes', bytes, minimums, maximums, warning_bag=warning_bag) searchhelpers.minmax('duration', duration, minimums, maximums, warning_bag=warning_bag) + searchhelpers.minmax('bitrate', bitrate, minimums, maximums, warning_bag=warning_bag) author = searchhelpers.normalize_author(author, photodb=self, warning_bag=warning_bag) extension = searchhelpers.normalize_extension(extension) @@ -652,7 +650,8 @@ class PDBPhotoMixin: 'area': area, 'width': width, 'height': height, - 'ratio': ratio, + 'aspectratio': aspectratio, + 'bitrate': bitrate, 'bytes': bytes, 'duration': duration, 'author': list(author) or None, diff --git a/etiquette/searchhelpers.py b/etiquette/searchhelpers.py index 20029aa..cdcbe92 100644 --- a/etiquette/searchhelpers.py +++ b/etiquette/searchhelpers.py @@ -347,10 +347,6 @@ def normalize_orderby(orderby, warning_bag=None): column_friendly = column column_expanded = { 'random': 'RANDOM()', - 'area': '(width * height)', - 'basename': 'COALESCE(override_filename, basename)', - 'bitrate': '((bytes / 128) / duration)', - 'ratio': '(width / height)', }.get(column, column) final_orderby.append( (column_friendly, column_expanded, direction) ) diff --git a/frontends/etiquette_cli.py b/frontends/etiquette_cli.py index 61bd1ba..9165feb 100644 --- a/frontends/etiquette_cli.py +++ b/frontends/etiquette_cli.py @@ -129,7 +129,7 @@ def search_by_argparse(args, yield_albums=False, yield_photos=False): area=args.area, width=args.width, height=args.height, - ratio=args.ratio, + aspectratio=args.aspectratio, bytes=args.bytes, duration=args.duration, author=args.author, @@ -1272,7 +1272,7 @@ def main(argv): ''', ) p_search.add_argument( - '--ratio', + '--aspectratio', metavar='X-Y', default=None, help=''' diff --git a/frontends/etiquette_flask/backend/endpoints/photo_endpoints.py b/frontends/etiquette_flask/backend/endpoints/photo_endpoints.py index c13bf49..2042021 100644 --- a/frontends/etiquette_flask/backend/endpoints/photo_endpoints.py +++ b/frontends/etiquette_flask/backend/endpoints/photo_endpoints.py @@ -399,10 +399,11 @@ def get_search_core(): area = request.args.get('area') width = request.args.get('width') height = request.args.get('height') - ratio = request.args.get('ratio') + aspectratio = request.args.get('aspectratio') bytes = request.args.get('bytes') has_thumbnail = request.args.get('has_thumbnail') duration = request.args.get('duration') + bitrate = request.args.get('bitrate') created = request.args.get('created') # These are in a dictionary so I can pass them to the page template. @@ -410,9 +411,10 @@ def get_search_core(): 'area': area, 'width': width, 'height': height, - 'ratio': ratio, + 'aspectratio': aspectratio, 'bytes': bytes, 'duration': duration, + 'bitrate': bitrate, 'author': author, 'created': created, diff --git a/frontends/etiquette_flask/templates/photo.html b/frontends/etiquette_flask/templates/photo.html index b43e406..52d6536 100644 --- a/frontends/etiquette_flask/templates/photo.html +++ b/frontends/etiquette_flask/templates/photo.html @@ -59,6 +59,7 @@ .photo_viewer_application, .photo_viewer_text { + display: flex; justify-items: center; align-items: center; } @@ -183,7 +184,7 @@ {% endif %} {% if photo.width %}
  • Dimensions: {{photo.width}}x{{photo.height}} px
    • -
  • Aspect ratio: {{photo.ratio}}
    • +
  • Aspect ratio: {{photo.aspectratio|round(2)}}
    • {% endif %}
  • Size: {{photo.bytes|bytestring}}
    • {% if photo.duration %} diff --git a/frontends/etiquette_flask/templates/search.html b/frontends/etiquette_flask/templates/search.html index 2c513b8..656be1c 100644 --- a/frontends/etiquette_flask/templates/search.html +++ b/frontends/etiquette_flask/templates/search.html @@ -171,7 +171,7 @@ - + diff --git a/utilities/database_upgrader.py b/utilities/database_upgrader.py index d4fdac3..6232f71 100644 --- a/utilities/database_upgrader.py +++ b/utilities/database_upgrader.py @@ -823,6 +823,67 @@ def upgrade_20_to_21(photodb): photodb.update(table=etiquette.objects.Photo, pairs={'id': photo.id, 'thumbnail': store_as}, where_key='id') photo.thumbnail = new_thumbnail +def upgrade_21_to_22(photodb): + m = Migrator(photodb) + + m.tables['photos']['create'] = ''' + CREATE TABLE IF NOT EXISTS photos( + id INT PRIMARY KEY NOT NULL, + filepath TEXT COLLATE NOCASE, + override_filename TEXT COLLATE NOCASE, + mtime INT, + sha256 TEXT, + width INT, + height INT, + duration INT, + bytes INT, + created INT, + thumbnail TEXT, + tagged_at INT, + author_id INT, + searchhidden INT, + -- GENERATED COLUMNS + area INT GENERATED ALWAYS AS (width * height) VIRTUAL, + aspectratio REAL GENERATED ALWAYS AS (1.0 * width / height) VIRTUAL, + -- Thank you ungalcrys + -- https://stackoverflow.com/a/38330814/5430534 + basename TEXT GENERATED ALWAYS AS ( + COALESCE( + override_filename, + replace(filepath, rtrim(filepath, replace(replace(filepath, '\\', '/'), '/', '')), '') + ) + ) STORED COLLATE NOCASE, + extension TEXT GENERATED ALWAYS AS ( + replace(basename, rtrim(basename, replace(basename, '.', '')), '') + ) VIRTUAL COLLATE NOCASE, + bitrate REAL GENERATED ALWAYS AS ((bytes / 128) / duration) VIRTUAL, + FOREIGN KEY(author_id) REFERENCES users(id) + ); + ''' + m.tables['photos']['transfer'] = ''' + INSERT INTO photos SELECT + id, + filepath, + override_filename, + mtime, + sha256, + width, + height, + duration, + bytes, + created, + thumbnail, + tagged_at, + author_id, + searchhidden + FROM photos_old; + ''' + + m.go() + + photodb.execute('DROP INDEX index_photos_override_filename') + photodb.execute('CREATE INDEX IF NOT EXISTS index_photos_basename on photos(basename COLLATE NOCASE)') + def upgrade_all(data_directory): ''' Given the directory containing a phototagger database, apply all of the