voussoir/etiquette
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Migrate all IDs from strings to ints. Random 32 bit IDs in future.

Browse source
This commit is contained in:
voussoir 2022-07-15 23:47:15 -07:00
parent 49992f59aa
commit cb43b5d9e0
No known key found for this signature in database
GPG key ID: 5F7554F8C26DACCB
5 changed files with 215 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
etiquette/constants.py
View file

@ -41,17 +41,16 @@ ffmpeg = _load_ffmpeg()
# Database #########################################################################################
DATABASE_VERSION = 20
DATABASE_VERSION = 21
DB_INIT = f'''
----------------------------------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS albums(
id TEXT PRIMARY KEY NOT NULL,
id INT PRIMARY KEY NOT NULL,
title TEXT,
description TEXT,
created INT,
thumbnail_photo TEXT,
author_id TEXT,
thumbnail_photo INT,
author_id INT,
FOREIGN KEY(author_id) REFERENCES users(id),
FOREIGN KEY(thumbnail_photo) REFERENCES photos(id)
);
@ -59,18 +58,18 @@ CREATE INDEX IF NOT EXISTS index_albums_id on albums(id);
CREATE INDEX IF NOT EXISTS index_albums_author_id on albums(author_id);
----------------------------------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS bookmarks(
id TEXT PRIMARY KEY NOT NULL,
id INT PRIMARY KEY NOT NULL,
title TEXT,
url TEXT,
created INT,
author_id TEXT,
author_id INT,
FOREIGN KEY(author_id) REFERENCES users(id)
);
CREATE INDEX IF NOT EXISTS index_bookmarks_id on bookmarks(id);
CREATE INDEX IF NOT EXISTS index_bookmarks_author_id on bookmarks(author_id);
----------------------------------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS photos(
id TEXT PRIMARY KEY NOT NULL,
id INT PRIMARY KEY NOT NULL,
filepath TEXT COLLATE NOCASE,
basename TEXT COLLATE NOCASE,
override_filename TEXT COLLATE NOCASE,
@ -86,7 +85,7 @@ CREATE TABLE IF NOT EXISTS photos(
created INT,
thumbnail TEXT,
tagged_at INT,
author_id TEXT,
author_id INT,
searchhidden INT,
FOREIGN KEY(author_id) REFERENCES users(id)
);
@ -100,11 +99,11 @@ CREATE INDEX IF NOT EXISTS index_photos_author_id on photos(author_id);
CREATE INDEX IF NOT EXISTS index_photos_searchhidden on photos(searchhidden);
----------------------------------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS tags(
id TEXT PRIMARY KEY NOT NULL,
id INT PRIMARY KEY NOT NULL,
name TEXT NOT NULL,
description TEXT,
created INT,
author_id TEXT,
author_id INT,
FOREIGN KEY(author_id) REFERENCES users(id)
);
CREATE INDEX IF NOT EXISTS index_tags_id on tags(id);
@ -112,8 +111,8 @@ CREATE INDEX IF NOT EXISTS index_tags_name on tags(name);
CREATE INDEX IF NOT EXISTS index_tags_author_id on tags(author_id);
----------------------------------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS users(
id TEXT PRIMARY KEY NOT NULL,
username TEXT NOT NULL COLLATE NOCASE,
id INT PRIMARY KEY NOT NULL,
username TEXT UNIQUE NOT NULL COLLATE NOCASE,
password BLOB NOT NULL,
display_name TEXT,
created INT
@ -124,7 +123,7 @@ CREATE INDEX IF NOT EXISTS index_users_username on users(username COLLATE NOCASE
----------------------------------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS album_associated_directories(
albumid TEXT NOT NULL,
albumid INT NOT NULL,
directory TEXT NOT NULL COLLATE NOCASE,
FOREIGN KEY(albumid) REFERENCES albums(id)
);
@ -134,8 +133,8 @@ CREATE INDEX IF NOT EXISTS index_album_associated_directories_directory on
album_associated_directories(directory);
----------------------------------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS album_group_rel(
parentid TEXT NOT NULL,
memberid TEXT NOT NULL,
parentid INT NOT NULL,
memberid INT NOT NULL,
FOREIGN KEY(parentid) REFERENCES albums(id),
FOREIGN KEY(memberid) REFERENCES albums(id)
);
@ -143,8 +142,8 @@ CREATE INDEX IF NOT EXISTS index_album_group_rel_parentid on album_group_rel(par
CREATE INDEX IF NOT EXISTS index_album_group_rel_memberid on album_group_rel(memberid);
----------------------------------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS album_photo_rel(
albumid TEXT NOT NULL,
photoid TEXT NOT NULL,
albumid INT NOT NULL,
photoid INT NOT NULL,
FOREIGN KEY(albumid) REFERENCES albums(id),
FOREIGN KEY(photoid) REFERENCES photos(id)
);
@ -157,8 +156,8 @@ CREATE TABLE IF NOT EXISTS id_numbers(
);
----------------------------------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS photo_tag_rel(
photoid TEXT NOT NULL,
tagid TEXT NOT NULL,
photoid INT NOT NULL,
tagid INT NOT NULL,
FOREIGN KEY(photoid) REFERENCES photos(id),
FOREIGN KEY(tagid) REFERENCES tags(id)
);
@ -167,8 +166,8 @@ CREATE INDEX IF NOT EXISTS index_photo_tag_rel_tagid on photo_tag_rel(tagid);
CREATE INDEX IF NOT EXISTS index_photo_tag_rel_photoid_tagid on photo_tag_rel(photoid, tagid);
----------------------------------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS tag_group_rel(
parentid TEXT NOT NULL,
memberid TEXT NOT NULL,
parentid INT NOT NULL,
memberid INT NOT NULL,
FOREIGN KEY(parentid) REFERENCES tags(id),
FOREIGN KEY(memberid) REFERENCES tags(id)
);
@ -296,7 +295,7 @@ DEFAULT_CONFIGURATION = {
],
'file_read_chunk': 2 ** 20,
'id_length': 12,
'id_bits': 32,
'thumbnail_width': 400,
'thumbnail_height': 400,

26
etiquette/objects.py
View file

@ -42,7 +42,7 @@ class ObjectBase(worms.Object):
self._author = None
@staticmethod
def normalize_author_id(author_id) -> typing.Optional[str]:
def normalize_author_id(author_id) -> typing.Optional[int]:
'''
Raises TypeError if author_id is not the right type.
@ -51,15 +51,11 @@ class ObjectBase(worms.Object):
if author_id is None:
return None
if not isinstance(author_id, str):
raise TypeError(f'Author ID must be {str}, not {type(author_id)}.')
if not isinstance(author_id, int):
raise TypeError(f'Author ID must be {int}, not {type(author_id)}.')
author_id = author_id.strip()
if author_id == '':
return None
if not all(c in constants.USER_ID_CHARACTERS for c in author_id):
raise ValueError(f'Author ID must consist only of {constants.USER_ID_CHARACTERS}.')
if author_id < 1:
raise ValueError(f'Author ID should be positive, not {author_id}.')
return author_id
@ -428,7 +424,7 @@ class Album(ObjectBase, GroupableMixin):
if self.title:
return self.title
else:
return self.id
return str(self.id)
@decorators.required_feature('album.edit')
@worms.atomic
@ -1015,12 +1011,14 @@ class Photo(ObjectBase):
except (OSError, ValueError):
traceback.print_exc()
else:
hopeful_filepath.parent.makedirs(exist_ok=True)
image.save(hopeful_filepath.absolute_path, quality=50)
return_filepath = hopeful_filepath
elif self.simple_mimetype == 'video' and constants.ffmpeg:
log.info('Thumbnailing %s.', self.real_path.absolute_path)
try:
hopeful_filepath.parent.makedirs(exist_ok=True)
success = helpers.generate_video_thumbnail(
self.real_path.absolute_path,
outfile=hopeful_filepath.absolute_path,
@ -1134,13 +1132,11 @@ class Photo(ObjectBase):
'''
Create the filepath that should be the location of our thumbnail.
'''
chunked_id = [''.join(chunk) for chunk in gentools.chunk_generator(self.id, 3)]
(folder, basename) = (chunked_id[:-1], chunked_id[-1])
chunked_id = [''.join(chunk) for chunk in gentools.chunk_generator(str(self.id), 3)]
folder = chunked_id[:-1]
folder = os.sep.join(folder)
folder = self.photodb.thumbnail_directory.join(folder)
if folder:
folder.makedirs(exist_ok=True)
hopeful_filepath = folder.with_child(basename + '.jpg')
hopeful_filepath = folder.with_child(f'{self.id}.jpg')
return hopeful_filepath
# Photo.rename_file already has @required_feature

57
etiquette/photodb.py
View file

@ -30,6 +30,7 @@ from voussoirkit import worms
log = vlogging.getLogger(__name__)
RNG = random.SystemRandom()
####################################################################################################
@ -1038,23 +1039,6 @@ class PDBUserMixin:
if badchars:
raise exceptions.InvalidUsernameChars(username=username, badchars=badchars)
def generate_user_id(self) -> str:
'''
User IDs are randomized instead of integers like the other objects,
so they get their own method.
'''
length = self.config['id_length']
for retry in range(20):
user_id = ''.join(random.choices(constants.USER_ID_CHARACTERS, k=length))
user_exists = self.select_one_value('SELECT 1 FROM users WHERE id == ?', [user_id])
if user_exists is None:
break
else:
raise Exception('Failed to create user id after 20 tries.')
return user_id
def get_user(self, username=None, id=None) -> objects.User:
'''
Redirect to get_user_by_id or get_user_by_username.
@ -1156,7 +1140,7 @@ class PDBUserMixin:
)
# Ok.
user_id = self.generate_user_id()
user_id = self.generate_id(objects.User)
log.info('New User: %s %s.', user_id, username)
hashed_password = bcrypt.hashpw(password, bcrypt.gensalt())
@ -1677,43 +1661,20 @@ class PhotoDB(
if getattr(self, 'ephemeral', False):
self.ephemeral_directory.cleanup()
def generate_id(self, thing_class) -> str:
def generate_id(self, thing_class) -> int:
'''
Create a new ID number that is unique to the given table.
Note that while this method may INSERT / UPDATE, it does not commit.
We'll wait for that to happen in whoever is calling us, so we know the
ID is actually used.
'''
if not (isinstance(thing_class, type) and issubclass(thing_class, objects.ObjectBase)):
if not issubclass(thing_class, objects.ObjectBase):
raise TypeError(thing_class)
table = thing_class.table
table = table.lower()
if table not in ['photos', 'tags', 'albums', 'bookmarks']:
raise ValueError(f'Invalid table requested: {table}.')
last_id = self.select_one_value('SELECT last_id FROM id_numbers WHERE tab == ?', [table])
if last_id is None:
# Register new value
new_id_int = 1
do_insert = True
else:
# Use database value
new_id_int = int(last_id) + 1
do_insert = False
new_id = str(new_id_int).rjust(self.config['id_length'], '0')
pairs = {
'tab': table,
'last_id': new_id,
}
if do_insert:
self.insert(table='id_numbers', data=pairs)
else:
self.update(table='id_numbers', pairs=pairs, where_key='tab')
return new_id
length = self.config['id_bits']
while True:
id = RNG.getrandbits(length)
if not self.exists(f'SELECT 1 FROM {table} WHERE id == ?', [id]):
return id
def load_config(self) -> None:
log.debug('Loading config file.')

20
frontends/etiquette_flask/templates/cards.html
View file

@ -1,7 +1,7 @@
{# ALBUM ######################################################################}
{% macro create_album_card(album, view="grid", unlink_parent=none, draggable=false) %}
{% set id = "album_card_root" if album == "root" else "album_card_" + album.id %}
{% set id = "album_card_root" if album == "root" else "album_card_" ~ album.id %}
{% set view = (view if view in ("list", "grid") else "grid") %}
{% set viewparam = "?view=list" if view == "list" else "" %}
<div
@ -22,7 +22,7 @@ draggable=true
<a class="album_card_thumbnail" href="/album/{{album.id}}{{viewparam}}" draggable="false">
{% endif %}
{% if album.thumbnail_photo %}
{% set thumbnail_src = "/thumbnail/" + album.thumbnail_photo.id + ".jpg" %}
{% set thumbnail_src = "/thumbnail/" ~ album.thumbnail_photo.id ~ ".jpg" %}
{% else %}
{% set thumbnail_src = "/static/basic_thumbnails/album.png" %}
{% endif %}
@ -160,7 +160,7 @@ draggable="true"
{% if view == "grid" %}
{% if photo.thumbnail %}
{% set thumbnail_src = "/thumbnail/" + photo.id + ".jpg" %}
{% set thumbnail_src = "/thumbnail/" ~ photo.id ~ ".jpg" %}
{% else %}
{% set thumbnail_src =
thumbnails.get(photo.extension, "") or
@ -168,7 +168,7 @@ draggable="true"
thumbnails.get(photo.simple_mimetype, "") or
"other"
%}
{% set thumbnail_src = "/static/basic_thumbnails/" + thumbnail_src + ".png" %}
{% set thumbnail_src = "/static/basic_thumbnails/" ~ thumbnail_src ~ ".png" %}
{% endif -%}{# if thumbnail #}
<a class="photo_card_thumbnail" target="_blank" href="/photo/{{photo.id}}" draggable="false">
@ -216,15 +216,15 @@ draggable="true"
) -%}
{%- set href = {
"search": "/search?tag_musts=" + (tag.name|urlencode),
"search_musts": "/search?tag_musts=" + (tag.name|urlencode),
"search_mays": "/search?tag_mays=" + (tag.name|urlencode),
"search_forbids": "/search?tag_forbids=" + (tag.name|urlencode),
"info": "/tag/" + tag.name,
"search": "/search?tag_musts=" ~ (tag.name|urlencode),
"search_musts": "/search?tag_musts=" ~ (tag.name|urlencode),
"search_mays": "/search?tag_mays=" ~ (tag.name|urlencode),
"search_forbids": "/search?tag_forbids=" ~ (tag.name|urlencode),
"info": "/tag/" ~ tag.name,
None: None,
}.get(link, link)
-%}
{%- set class = ("tag_card" + " " + extra_classes).strip() -%}
{%- set class = ("tag_card" ~ " " ~ extra_classes).strip() -%}
{%- set title = (with_alt_description and tag.description) or None -%}
{%- set innertext = innertext_safe or (innertext or tag.name)|e -%}
{%- set element = "a" if (link or onclick) else "span" -%}

163
utilities/database_upgrader.py
View file

@ -1,3 +1,4 @@
import time
import argparse
import os
import sys
@ -50,6 +51,10 @@ class Migrator:
# which is about to get renamed to B_old and then A's reference will be
# broken.
self.photodb.pragma_write('foreign_keys', 'OFF')
for (name, query) in self.indices:
self.photodb.execute(f'DROP INDEX {name}')
for (name, table) in self.tables.items():
if name not in self.existing_tables:
continue
@ -660,6 +665,164 @@ def upgrade_19_to_20(photodb):
photodb.execute('UPDATE photos SET thumbnail = REPLACE(thumbnail, "\\site_thumbnails\\", "\\thumbnails\\")')
photodb.on_commit_queue.append({'action': os.rename, 'args': (old, new)})
def upgrade_20_to_21(photodb):
'''
In this version, the object IDs were migrated from string to int.
'''
m = Migrator(photodb)
m.tables['albums']['create'] = '''
CREATE TABLE IF NOT EXISTS albums(
id INT PRIMARY KEY NOT NULL,
title TEXT,
description TEXT,
created INT,
thumbnail_photo INT,
author_id INT,
FOREIGN KEY(author_id) REFERENCES users(id),
FOREIGN KEY(thumbnail_photo) REFERENCES photos(id)
);
'''
m.tables['albums']['transfer'] = 'INSERT INTO albums SELECT * FROM albums_old'
m.tables['bookmarks']['create'] = '''
CREATE TABLE IF NOT EXISTS bookmarks(
id INT PRIMARY KEY NOT NULL,
title TEXT,
url TEXT,
created INT,
author_id INT,
FOREIGN KEY(author_id) REFERENCES users(id)
);
'''
m.tables['bookmarks']['transfer'] = 'INSERT INTO bookmarks SELECT * FROM bookmarks_old'
m.tables['photos']['create'] = '''
CREATE TABLE IF NOT EXISTS photos(
id INT PRIMARY KEY NOT NULL,
filepath TEXT COLLATE NOCASE,
basename TEXT COLLATE NOCASE,
override_filename TEXT COLLATE NOCASE,
extension TEXT COLLATE NOCASE,
mtime INT,
sha256 TEXT,
width INT,
height INT,
ratio REAL,
area INT,
duration INT,
bytes INT,
created INT,
thumbnail TEXT,
tagged_at INT,
author_id INT,
searchhidden INT,
FOREIGN KEY(author_id) REFERENCES users(id)
);
'''
m.tables['photos']['transfer'] = 'INSERT INTO photos SELECT * FROM photos_old'
m.tables['tags']['create'] = '''
CREATE TABLE IF NOT EXISTS tags(
id INT PRIMARY KEY NOT NULL,
name TEXT NOT NULL,
description TEXT,
created INT,
author_id INT,
FOREIGN KEY(author_id) REFERENCES users(id)
);
'''
m.tables['tags']['transfer'] = 'INSERT INTO tags SELECT * FROM tags_old'
m.tables['users']['create'] = '''
CREATE TABLE IF NOT EXISTS users(
id INT PRIMARY KEY NOT NULL,
username TEXT UNIQUE NOT NULL COLLATE NOCASE,
password BLOB NOT NULL,
display_name TEXT,
created INT
);
'''
m.tables['users']['transfer'] = 'INSERT INTO users SELECT * FROM users_old'
m.tables['album_associated_directories']['create'] = '''
CREATE TABLE IF NOT EXISTS album_associated_directories(
albumid INT NOT NULL,
directory TEXT NOT NULL COLLATE NOCASE,
FOREIGN KEY(albumid) REFERENCES albums(id)
);
'''
m.tables['album_associated_directories']['transfer'] = 'INSERT INTO album_associated_directories SELECT * FROM album_associated_directories_old'
m.tables['album_group_rel']['create'] = '''
CREATE TABLE IF NOT EXISTS album_group_rel(
parentid INT NOT NULL,
memberid INT NOT NULL,
FOREIGN KEY(parentid) REFERENCES albums(id),
FOREIGN KEY(memberid) REFERENCES albums(id)
);
'''
m.tables['album_group_rel']['transfer'] = 'INSERT INTO album_group_rel SELECT * FROM album_group_rel_old'
m.tables['album_photo_rel']['create'] = '''
CREATE TABLE IF NOT EXISTS album_photo_rel(
albumid INT NOT NULL,
photoid INT NOT NULL,
FOREIGN KEY(albumid) REFERENCES albums(id),
FOREIGN KEY(photoid) REFERENCES photos(id)
);
'''
m.tables['album_photo_rel']['transfer'] = 'INSERT INTO album_photo_rel SELECT * FROM album_photo_rel_old'
m.tables['photo_tag_rel']['create'] = '''
CREATE TABLE IF NOT EXISTS photo_tag_rel(
photoid INT NOT NULL,
tagid INT NOT NULL,
FOREIGN KEY(photoid) REFERENCES photos(id),
FOREIGN KEY(tagid) REFERENCES tags(id)
);
'''
m.tables['photo_tag_rel']['transfer'] = 'INSERT INTO photo_tag_rel SELECT * FROM photo_tag_rel_old'
m.tables['tag_group_rel']['create'] = '''
CREATE TABLE IF NOT EXISTS tag_group_rel(
parentid INT NOT NULL,
memberid INT NOT NULL,
FOREIGN KEY(parentid) REFERENCES tags(id),
FOREIGN KEY(memberid) REFERENCES tags(id)
);
'''
m.tables['tag_group_rel']['transfer'] = 'INSERT INTO tag_group_rel SELECT * FROM tag_group_rel_old'
m.go()
users = list(photodb.get_users())
for user in users:
old_id = user.id
new_id = photodb.generate_id(etiquette.objects.User)
photodb.execute('UPDATE users SET id = ? WHERE id = ?', [new_id, old_id])
photodb.execute('UPDATE albums SET author_id = ? WHERE author_id = ?', [new_id, old_id])
photodb.execute('UPDATE bookmarks SET author_id = ? WHERE author_id = ?', [new_id, old_id])
photodb.execute('UPDATE photos SET author_id = ? WHERE author_id = ?', [new_id, old_id])
photodb.execute('UPDATE tags SET author_id = ? WHERE author_id = ?', [new_id, old_id])
def movethumbnail(old_thumbnail, new_thumbnail):
new_thumbnail.parent.makedirs(exist_ok=True)
shutil.move(old_thumbnail.absolute_path, new_thumbnail.absolute_path)
photos = photodb.get_photos()
import shutil
for photo in photos:
if photo.thumbnail is None:
continue
old_thumbnail = photo.thumbnail
new_thumbnail = photo.make_thumbnail_filepath()
print(old_thumbnail, new_thumbnail)
photodb.on_commit_queue.append({'action': movethumbnail, 'args': (old_thumbnail.absolute_path, new_thumbnail.absolute_path)})
store_as = new_thumbnail.relative_to(photodb.thumbnail_directory)
photodb.update(table=etiquette.objects.Photo, pairs={'id': photo.id, 'thumbnail': store_as}, where_key='id')
photo.thumbnail = new_thumbnail
def upgrade_all(data_directory):
'''
Given the directory containing a phototagger database, apply all of the