Migrate all IDs from strings to ints. Random 32 bit IDs in future.

etiquette/constants.py

@@ -41,17 +41,16 @@ ffmpeg = _load_ffmpeg()
 
 # Database #########################################################################################
 
-DATABASE_VERSION = 20
+DATABASE_VERSION = 21
 
 DB_INIT = f'''
-----------------------------------------------------------------------------------------------------
 CREATE TABLE IF NOT EXISTS albums(
-    id TEXT PRIMARY KEY NOT NULL,
+    id INT PRIMARY KEY NOT NULL,
     title TEXT,
     description TEXT,
     created INT,
-    thumbnail_photo TEXT,
+    thumbnail_photo INT,
-    author_id TEXT,
+    author_id INT,
     FOREIGN KEY(author_id) REFERENCES users(id),
     FOREIGN KEY(thumbnail_photo) REFERENCES photos(id)
 );
@@ -59,18 +58,18 @@ CREATE INDEX IF NOT EXISTS index_albums_id on albums(id);
 CREATE INDEX IF NOT EXISTS index_albums_author_id on albums(author_id);
 ----------------------------------------------------------------------------------------------------
 CREATE TABLE IF NOT EXISTS bookmarks(
-    id TEXT PRIMARY KEY NOT NULL,
+    id INT PRIMARY KEY NOT NULL,
     title TEXT,
     url TEXT,
     created INT,
-    author_id TEXT,
+    author_id INT,
     FOREIGN KEY(author_id) REFERENCES users(id)
 );
 CREATE INDEX IF NOT EXISTS index_bookmarks_id on bookmarks(id);
 CREATE INDEX IF NOT EXISTS index_bookmarks_author_id on bookmarks(author_id);
 ----------------------------------------------------------------------------------------------------
 CREATE TABLE IF NOT EXISTS photos(
-    id TEXT PRIMARY KEY NOT NULL,
+    id INT PRIMARY KEY NOT NULL,
     filepath TEXT COLLATE NOCASE,
     basename TEXT COLLATE NOCASE,
     override_filename TEXT COLLATE NOCASE,
@@ -86,7 +85,7 @@ CREATE TABLE IF NOT EXISTS photos(
     created INT,
     thumbnail TEXT,
     tagged_at INT,
-    author_id TEXT,
+    author_id INT,
     searchhidden INT,
     FOREIGN KEY(author_id) REFERENCES users(id)
 );
@@ -100,11 +99,11 @@ CREATE INDEX IF NOT EXISTS index_photos_author_id on photos(author_id);
 CREATE INDEX IF NOT EXISTS index_photos_searchhidden on photos(searchhidden);
 ----------------------------------------------------------------------------------------------------
 CREATE TABLE IF NOT EXISTS tags(
-    id TEXT PRIMARY KEY NOT NULL,
+    id INT PRIMARY KEY NOT NULL,
     name TEXT NOT NULL,
     description TEXT,
     created INT,
-    author_id TEXT,
+    author_id INT,
     FOREIGN KEY(author_id) REFERENCES users(id)
 );
 CREATE INDEX IF NOT EXISTS index_tags_id on tags(id);
@@ -112,8 +111,8 @@ CREATE INDEX IF NOT EXISTS index_tags_name on tags(name);
 CREATE INDEX IF NOT EXISTS index_tags_author_id on tags(author_id);
 ----------------------------------------------------------------------------------------------------
 CREATE TABLE IF NOT EXISTS users(
-    id TEXT PRIMARY KEY NOT NULL,
+    id INT PRIMARY KEY NOT NULL,
-    username TEXT NOT NULL COLLATE NOCASE,
+    username TEXT UNIQUE NOT NULL COLLATE NOCASE,
     password BLOB NOT NULL,
     display_name TEXT,
     created INT
@@ -124,7 +123,7 @@ CREATE INDEX IF NOT EXISTS index_users_username on users(username COLLATE NOCASE
 
 ----------------------------------------------------------------------------------------------------
 CREATE TABLE IF NOT EXISTS album_associated_directories(
-    albumid TEXT NOT NULL,
+    albumid INT NOT NULL,
     directory TEXT NOT NULL COLLATE NOCASE,
     FOREIGN KEY(albumid) REFERENCES albums(id)
 );
@@ -134,8 +133,8 @@ CREATE INDEX IF NOT EXISTS index_album_associated_directories_directory on
     album_associated_directories(directory);
 ----------------------------------------------------------------------------------------------------
 CREATE TABLE IF NOT EXISTS album_group_rel(
-    parentid TEXT NOT NULL,
+    parentid INT NOT NULL,
-    memberid TEXT NOT NULL,
+    memberid INT NOT NULL,
     FOREIGN KEY(parentid) REFERENCES albums(id),
     FOREIGN KEY(memberid) REFERENCES albums(id)
 );
@@ -143,8 +142,8 @@ CREATE INDEX IF NOT EXISTS index_album_group_rel_parentid on album_group_rel(par
 CREATE INDEX IF NOT EXISTS index_album_group_rel_memberid on album_group_rel(memberid);
 ----------------------------------------------------------------------------------------------------
 CREATE TABLE IF NOT EXISTS album_photo_rel(
-    albumid TEXT NOT NULL,
+    albumid INT NOT NULL,
-    photoid TEXT NOT NULL,
+    photoid INT NOT NULL,
     FOREIGN KEY(albumid) REFERENCES albums(id),
     FOREIGN KEY(photoid) REFERENCES photos(id)
 );
@@ -157,8 +156,8 @@ CREATE TABLE IF NOT EXISTS id_numbers(
 );
 ----------------------------------------------------------------------------------------------------
 CREATE TABLE IF NOT EXISTS photo_tag_rel(
-    photoid TEXT NOT NULL,
+    photoid INT NOT NULL,
-    tagid TEXT NOT NULL,
+    tagid INT NOT NULL,
     FOREIGN KEY(photoid) REFERENCES photos(id),
     FOREIGN KEY(tagid) REFERENCES tags(id)
 );
@@ -167,8 +166,8 @@ CREATE INDEX IF NOT EXISTS index_photo_tag_rel_tagid on photo_tag_rel(tagid);
 CREATE INDEX IF NOT EXISTS index_photo_tag_rel_photoid_tagid on photo_tag_rel(photoid, tagid);
 ----------------------------------------------------------------------------------------------------
 CREATE TABLE IF NOT EXISTS tag_group_rel(
-    parentid TEXT NOT NULL,
+    parentid INT NOT NULL,
-    memberid TEXT NOT NULL,
+    memberid INT NOT NULL,
     FOREIGN KEY(parentid) REFERENCES tags(id),
     FOREIGN KEY(memberid) REFERENCES tags(id)
 );
@@ -296,7 +295,7 @@ DEFAULT_CONFIGURATION = {
     ],
 
     'file_read_chunk': 2 ** 20,
-    'id_length': 12,
+    'id_bits': 32,
     'thumbnail_width': 400,
     'thumbnail_height': 400,
 

etiquette/objects.py

@@ -42,7 +42,7 @@ class ObjectBase(worms.Object):
         self._author = None
 
     @staticmethod
-    def normalize_author_id(author_id) -> typing.Optional[str]:
+    def normalize_author_id(author_id) -> typing.Optional[int]:
         '''
         Raises TypeError if author_id is not the right type.
 
@@ -51,15 +51,11 @@ class ObjectBase(worms.Object):
         if author_id is None:
             return None
 
-        if not isinstance(author_id, str):
+        if not isinstance(author_id, int):
-            raise TypeError(f'Author ID must be {str}, not {type(author_id)}.')
+            raise TypeError(f'Author ID must be {int}, not {type(author_id)}.')
 
-        author_id = author_id.strip()
+        if author_id < 1:
-        if author_id == '':
+            raise ValueError(f'Author ID should be positive, not {author_id}.')
-            return None
-
-        if not all(c in constants.USER_ID_CHARACTERS for c in author_id):
-            raise ValueError(f'Author ID must consist only of {constants.USER_ID_CHARACTERS}.')
 
         return author_id
 
@@ -428,7 +424,7 @@ class Album(ObjectBase, GroupableMixin):
         if self.title:
             return self.title
         else:
-            return self.id
+            return str(self.id)
 
     @decorators.required_feature('album.edit')
     @worms.atomic
@@ -1015,12 +1011,14 @@ class Photo(ObjectBase):
             except (OSError, ValueError):
                 traceback.print_exc()
             else:
+                hopeful_filepath.parent.makedirs(exist_ok=True)
                 image.save(hopeful_filepath.absolute_path, quality=50)
                 return_filepath = hopeful_filepath
 
         elif self.simple_mimetype == 'video' and constants.ffmpeg:
             log.info('Thumbnailing %s.', self.real_path.absolute_path)
             try:
+                hopeful_filepath.parent.makedirs(exist_ok=True)
                 success = helpers.generate_video_thumbnail(
                     self.real_path.absolute_path,
                     outfile=hopeful_filepath.absolute_path,
@@ -1134,13 +1132,11 @@ class Photo(ObjectBase):
         '''
         Create the filepath that should be the location of our thumbnail.
         '''
-        chunked_id = [''.join(chunk) for chunk in gentools.chunk_generator(self.id, 3)]
+        chunked_id = [''.join(chunk) for chunk in gentools.chunk_generator(str(self.id), 3)]
-        (folder, basename) = (chunked_id[:-1], chunked_id[-1])
+        folder = chunked_id[:-1]
         folder = os.sep.join(folder)
         folder = self.photodb.thumbnail_directory.join(folder)
-        if folder:
+        hopeful_filepath = folder.with_child(f'{self.id}.jpg')
-            folder.makedirs(exist_ok=True)
-        hopeful_filepath = folder.with_child(basename + '.jpg')
         return hopeful_filepath
 
     # Photo.rename_file already has @required_feature

etiquette/photodb.py

@@ -30,6 +30,7 @@ from voussoirkit import worms
 
 log = vlogging.getLogger(__name__)
 
+RNG = random.SystemRandom()
 
 ####################################################################################################
 
@@ -1038,23 +1039,6 @@ class PDBUserMixin:
         if badchars:
             raise exceptions.InvalidUsernameChars(username=username, badchars=badchars)
 
-    def generate_user_id(self) -> str:
-        '''
-        User IDs are randomized instead of integers like the other objects,
-        so they get their own method.
-        '''
-        length = self.config['id_length']
-        for retry in range(20):
-            user_id = ''.join(random.choices(constants.USER_ID_CHARACTERS, k=length))
-
-            user_exists = self.select_one_value('SELECT 1 FROM users WHERE id == ?', [user_id])
-            if user_exists is None:
-                break
-        else:
-            raise Exception('Failed to create user id after 20 tries.')
-
-        return user_id
-
     def get_user(self, username=None, id=None) -> objects.User:
         '''
         Redirect to get_user_by_id or get_user_by_username.
@@ -1156,7 +1140,7 @@ class PDBUserMixin:
         )
 
         # Ok.
-        user_id = self.generate_user_id()
+        user_id = self.generate_id(objects.User)
         log.info('New User: %s %s.', user_id, username)
 
         hashed_password = bcrypt.hashpw(password, bcrypt.gensalt())
@@ -1677,43 +1661,20 @@ class PhotoDB(
         if getattr(self, 'ephemeral', False):
             self.ephemeral_directory.cleanup()
 
-    def generate_id(self, thing_class) -> str:
+    def generate_id(self, thing_class) -> int:
         '''
         Create a new ID number that is unique to the given table.
-        Note that while this method may INSERT / UPDATE, it does not commit.
-        We'll wait for that to happen in whoever is calling us, so we know the
-        ID is actually used.
         '''
-        if not (isinstance(thing_class, type) and issubclass(thing_class, objects.ObjectBase)):
+        if not issubclass(thing_class, objects.ObjectBase):
             raise TypeError(thing_class)
 
         table = thing_class.table
 
-        table = table.lower()
+        length = self.config['id_bits']
-        if table not in ['photos', 'tags', 'albums', 'bookmarks']:
+        while True:
-            raise ValueError(f'Invalid table requested: {table}.')
+            id = RNG.getrandbits(length)
-
+            if not self.exists(f'SELECT 1 FROM {table} WHERE id == ?', [id]):
-        last_id = self.select_one_value('SELECT last_id FROM id_numbers WHERE tab == ?', [table])
+                return id
-        if last_id is None:
-            # Register new value
-            new_id_int = 1
-            do_insert = True
-        else:
-            # Use database value
-            new_id_int = int(last_id) + 1
-            do_insert = False
-
-        new_id = str(new_id_int).rjust(self.config['id_length'], '0')
-
-        pairs = {
-            'tab': table,
-            'last_id': new_id,
-        }
-        if do_insert:
-            self.insert(table='id_numbers', data=pairs)
-        else:
-            self.update(table='id_numbers', pairs=pairs, where_key='tab')
-        return new_id
 
     def load_config(self) -> None:
         log.debug('Loading config file.')

frontends/etiquette_flask/templates/cards.html

@@ -1,7 +1,7 @@
 {# ALBUM ######################################################################}
 
 {% macro create_album_card(album, view="grid", unlink_parent=none, draggable=false) %}
-{% set id = "album_card_root" if album == "root" else "album_card_" + album.id %}
+{% set id = "album_card_root" if album == "root" else "album_card_" ~ album.id %}
 {% set view = (view if view in ("list", "grid") else "grid") %}
 {% set viewparam = "?view=list" if view == "list" else "" %}
 <div
@@ -22,7 +22,7 @@ draggable=true
     <a class="album_card_thumbnail" href="/album/{{album.id}}{{viewparam}}" draggable="false">
     {% endif %}
         {% if album.thumbnail_photo %}
-        {% set thumbnail_src = "/thumbnail/" + album.thumbnail_photo.id + ".jpg" %}
+        {% set thumbnail_src = "/thumbnail/" ~ album.thumbnail_photo.id ~ ".jpg" %}
         {% else %}
         {% set thumbnail_src = "/static/basic_thumbnails/album.png" %}
         {% endif %}
@@ -160,7 +160,7 @@ draggable="true"
 
     {% if view == "grid" %}
     {% if photo.thumbnail %}
-        {% set thumbnail_src = "/thumbnail/" + photo.id + ".jpg" %}
+        {% set thumbnail_src = "/thumbnail/" ~ photo.id ~ ".jpg" %}
     {% else %}
         {% set thumbnail_src =
             thumbnails.get(photo.extension, "") or
@@ -168,7 +168,7 @@ draggable="true"
             thumbnails.get(photo.simple_mimetype, "") or
             "other"
         %}
-        {% set thumbnail_src = "/static/basic_thumbnails/" + thumbnail_src + ".png" %}
+        {% set thumbnail_src = "/static/basic_thumbnails/" ~ thumbnail_src ~ ".png" %}
     {% endif -%}{# if thumbnail #}
 
     <a class="photo_card_thumbnail" target="_blank" href="/photo/{{photo.id}}" draggable="false">
@@ -216,15 +216,15 @@ draggable="true"
 ) -%}
 
 {%- set href = {
-        "search": "/search?tag_musts=" + (tag.name|urlencode),
+        "search": "/search?tag_musts=" ~ (tag.name|urlencode),
-        "search_musts": "/search?tag_musts=" + (tag.name|urlencode),
+        "search_musts": "/search?tag_musts=" ~ (tag.name|urlencode),
-        "search_mays": "/search?tag_mays=" + (tag.name|urlencode),
+        "search_mays": "/search?tag_mays=" ~ (tag.name|urlencode),
-        "search_forbids": "/search?tag_forbids=" + (tag.name|urlencode),
+        "search_forbids": "/search?tag_forbids=" ~ (tag.name|urlencode),
-        "info": "/tag/" + tag.name,
+        "info": "/tag/" ~ tag.name,
         None: None,
     }.get(link, link)
 -%}
-{%- set class = ("tag_card" + " " + extra_classes).strip() -%}
+{%- set class = ("tag_card" ~ " " ~ extra_classes).strip() -%}
 {%- set title = (with_alt_description and tag.description) or None -%}
 {%- set innertext = innertext_safe or (innertext or tag.name)|e -%}
 {%- set element = "a" if (link or onclick) else "span" -%}

utilities/database_upgrader.py

@@ -1,3 +1,4 @@
+import time
 import argparse
 import os
 import sys
@@ -50,6 +51,10 @@ class Migrator:
         # which is about to get renamed to B_old and then A's reference will be
         # broken.
         self.photodb.pragma_write('foreign_keys', 'OFF')
+
+        for (name, query) in self.indices:
+            self.photodb.execute(f'DROP INDEX {name}')
+
         for (name, table) in self.tables.items():
             if name not in self.existing_tables:
                 continue
@@ -660,6 +665,164 @@ def upgrade_19_to_20(photodb):
     photodb.execute('UPDATE photos SET thumbnail = REPLACE(thumbnail, "\\site_thumbnails\\", "\\thumbnails\\")')
     photodb.on_commit_queue.append({'action': os.rename, 'args': (old, new)})
 
+def upgrade_20_to_21(photodb):
+    '''
+    In this version, the object IDs were migrated from string to int.
+    '''
+    m = Migrator(photodb)
+
+    m.tables['albums']['create'] = '''
+    CREATE TABLE IF NOT EXISTS albums(
+        id INT PRIMARY KEY NOT NULL,
+        title TEXT,
+        description TEXT,
+        created INT,
+        thumbnail_photo INT,
+        author_id INT,
+        FOREIGN KEY(author_id) REFERENCES users(id),
+        FOREIGN KEY(thumbnail_photo) REFERENCES photos(id)
+    );
+    '''
+    m.tables['albums']['transfer'] = 'INSERT INTO albums SELECT * FROM albums_old'
+
+    m.tables['bookmarks']['create'] = '''
+    CREATE TABLE IF NOT EXISTS bookmarks(
+        id INT PRIMARY KEY NOT NULL,
+        title TEXT,
+        url TEXT,
+        created INT,
+        author_id INT,
+        FOREIGN KEY(author_id) REFERENCES users(id)
+    );
+    '''
+    m.tables['bookmarks']['transfer'] = 'INSERT INTO bookmarks SELECT * FROM bookmarks_old'
+
+    m.tables['photos']['create'] = '''
+    CREATE TABLE IF NOT EXISTS photos(
+        id INT PRIMARY KEY NOT NULL,
+        filepath TEXT COLLATE NOCASE,
+        basename TEXT COLLATE NOCASE,
+        override_filename TEXT COLLATE NOCASE,
+        extension TEXT COLLATE NOCASE,
+        mtime INT,
+        sha256 TEXT,
+        width INT,
+        height INT,
+        ratio REAL,
+        area INT,
+        duration INT,
+        bytes INT,
+        created INT,
+        thumbnail TEXT,
+        tagged_at INT,
+        author_id INT,
+        searchhidden INT,
+        FOREIGN KEY(author_id) REFERENCES users(id)
+    );
+    '''
+    m.tables['photos']['transfer'] = 'INSERT INTO photos SELECT * FROM photos_old'
+
+    m.tables['tags']['create'] = '''
+    CREATE TABLE IF NOT EXISTS tags(
+        id INT PRIMARY KEY NOT NULL,
+        name TEXT NOT NULL,
+        description TEXT,
+        created INT,
+        author_id INT,
+        FOREIGN KEY(author_id) REFERENCES users(id)
+    );
+    '''
+    m.tables['tags']['transfer'] = 'INSERT INTO tags SELECT * FROM tags_old'
+
+    m.tables['users']['create'] = '''
+    CREATE TABLE IF NOT EXISTS users(
+        id INT PRIMARY KEY NOT NULL,
+        username TEXT UNIQUE NOT NULL COLLATE NOCASE,
+        password BLOB NOT NULL,
+        display_name TEXT,
+        created INT
+    );
+    '''
+    m.tables['users']['transfer'] = 'INSERT INTO users SELECT * FROM users_old'
+
+    m.tables['album_associated_directories']['create'] = '''
+    CREATE TABLE IF NOT EXISTS album_associated_directories(
+        albumid INT NOT NULL,
+        directory TEXT NOT NULL COLLATE NOCASE,
+        FOREIGN KEY(albumid) REFERENCES albums(id)
+    );
+    '''
+    m.tables['album_associated_directories']['transfer'] = 'INSERT INTO album_associated_directories SELECT * FROM album_associated_directories_old'
+
+    m.tables['album_group_rel']['create'] = '''
+    CREATE TABLE IF NOT EXISTS album_group_rel(
+        parentid INT NOT NULL,
+        memberid INT NOT NULL,
+        FOREIGN KEY(parentid) REFERENCES albums(id),
+        FOREIGN KEY(memberid) REFERENCES albums(id)
+    );
+    '''
+    m.tables['album_group_rel']['transfer'] = 'INSERT INTO album_group_rel SELECT * FROM album_group_rel_old'
+
+    m.tables['album_photo_rel']['create'] = '''
+    CREATE TABLE IF NOT EXISTS album_photo_rel(
+        albumid INT NOT NULL,
+        photoid INT NOT NULL,
+        FOREIGN KEY(albumid) REFERENCES albums(id),
+        FOREIGN KEY(photoid) REFERENCES photos(id)
+    );
+    '''
+    m.tables['album_photo_rel']['transfer'] = 'INSERT INTO album_photo_rel SELECT * FROM album_photo_rel_old'
+
+    m.tables['photo_tag_rel']['create'] = '''
+    CREATE TABLE IF NOT EXISTS photo_tag_rel(
+        photoid INT NOT NULL,
+        tagid INT NOT NULL,
+        FOREIGN KEY(photoid) REFERENCES photos(id),
+        FOREIGN KEY(tagid) REFERENCES tags(id)
+    );
+    '''
+    m.tables['photo_tag_rel']['transfer'] = 'INSERT INTO photo_tag_rel SELECT * FROM photo_tag_rel_old'
+
+    m.tables['tag_group_rel']['create'] = '''
+    CREATE TABLE IF NOT EXISTS tag_group_rel(
+        parentid INT NOT NULL,
+        memberid INT NOT NULL,
+        FOREIGN KEY(parentid) REFERENCES tags(id),
+        FOREIGN KEY(memberid) REFERENCES tags(id)
+    );
+    '''
+    m.tables['tag_group_rel']['transfer'] = 'INSERT INTO tag_group_rel SELECT * FROM tag_group_rel_old'
+
+    m.go()
+
+    users = list(photodb.get_users())
+    for user in users:
+        old_id = user.id
+        new_id = photodb.generate_id(etiquette.objects.User)
+        photodb.execute('UPDATE users SET id = ? WHERE id = ?', [new_id, old_id])
+        photodb.execute('UPDATE albums SET author_id = ? WHERE author_id = ?', [new_id, old_id])
+        photodb.execute('UPDATE bookmarks SET author_id = ? WHERE author_id = ?', [new_id, old_id])
+        photodb.execute('UPDATE photos SET author_id = ? WHERE author_id = ?', [new_id, old_id])
+        photodb.execute('UPDATE tags SET author_id = ? WHERE author_id = ?', [new_id, old_id])
+
+    def movethumbnail(old_thumbnail, new_thumbnail):
+        new_thumbnail.parent.makedirs(exist_ok=True)
+        shutil.move(old_thumbnail.absolute_path, new_thumbnail.absolute_path)
+
+    photos = photodb.get_photos()
+    import shutil
+    for photo in photos:
+        if photo.thumbnail is None:
+            continue
+        old_thumbnail = photo.thumbnail
+        new_thumbnail = photo.make_thumbnail_filepath()
+        print(old_thumbnail, new_thumbnail)
+        photodb.on_commit_queue.append({'action': movethumbnail, 'args': (old_thumbnail.absolute_path, new_thumbnail.absolute_path)})
+        store_as = new_thumbnail.relative_to(photodb.thumbnail_directory)
+        photodb.update(table=etiquette.objects.Photo, pairs={'id': photo.id, 'thumbnail': store_as}, where_key='id')
+        photo.thumbnail = new_thumbnail
+
 def upgrade_all(data_directory):
     '''
     Given the directory containing a phototagger database, apply all of the