prepare to remove TagGroups
This commit is contained in:
Ethan Dalool 2016-09-05 16:37:07 -07:00
parent 82f63a75ab
commit bfaed2e416
20 changed files with 1522 additions and 1017 deletions

18
AHK/turboclick.ahk Normal file
View file

@ -0,0 +1,18 @@
rapid_switch := false
~*CapsLock:: rapid_switch := !rapid_switch
~*LButton::
If rapid_switch = 1
{
Loop
{
GetKeyState, var, LButton, P
If var = U
{
Break
}
Send {LButton}
sleep 20
}
}

View file

@ -12,8 +12,8 @@ def multi_line_input():
except EOFError:
# If you enter nothing but ctrl-z
additional = EOF
userinput.append(additional)
else:
userinput.append(additional)
if EOF in additional:
break

View file

@ -7,9 +7,9 @@ import time
import urllib
import warnings
sys.path.append('C:\\git\\else\\bytestring'); import bytestring
sys.path.append('C:\\git\\else\\clipext'); import clipext
sys.path.append('C:\\git\\else\\ratelimiter'); import ratelimiter
sys.path.append('C:\\git\\else\\bytestring'); import bytestring
warnings.simplefilter('ignore')
@ -20,18 +20,12 @@ HEADERS = {
FILENAME_BADCHARS = '*?"<>|'
last_request = 0
CHUNKSIZE = 16 * bytestring.KIBIBYTE
CHUNKSIZE = 4 * bytestring.KIBIBYTE
TIMEOUT = 600
TEMP_EXTENSION = '.downloadytemp'
def basename_from_url(url):
'''
Determine the local filename appropriate for a URL.
'''
localname = urllib.parse.unquote(url)
localname = localname.split('?')[0]
localname = localname.split('/')[-1]
return localname
PRINT_LIMITER = ratelimiter.Ratelimiter(allowance=5, mode='reject')
def download_file(
url,
@ -41,6 +35,7 @@ def download_file(
callback_progress=None,
headers=None,
overwrite=False,
raise_for_undersized=True,
verbose=False,
):
headers = headers or {}
@ -85,25 +80,31 @@ def download_file(
bytes_downloaded = 0
download_stream = request('get', url, stream=True, headers=headers, auth=auth)
if callback_progress is not None:
callback_progress = callback_progress(plan['remote_total_bytes'])
for chunk in download_stream.iter_content(chunk_size=CHUNKSIZE):
bytes_downloaded += len(chunk)
file_handle.write(chunk)
if callback_progress is not None:
callback_progress(bytes_downloaded, plan['remote_total_bytes'])
callback_progress.step(bytes_downloaded)
if plan['limiter'] is not None and bytes_downloaded < plan['remote_total_bytes']:
plan['limiter'].limit(len(chunk))
file_handle.close()
if localname != plan['real_localname']:
os.rename(localname, plan['real_localname'])
# Don't try to rename /dev/null
if os.devnull not in [localname, plan['real_localname']]:
localsize = os.path.getsize(localname)
undersized = plan['plan_type'] != 'partial' and localsize < plan['remote_total_bytes']
if raise_for_undersized and undersized:
message = 'File does not contain expected number of bytes. Received {size} / {total}'
message = message.format(size=localsize, total=plan['remote_total_bytes'])
raise Exception(message)
localsize = os.path.getsize(plan['real_localname'])
if plan['plan_type'] != 'partial' and localsize < plan['remote_total_bytes']:
message = 'File does not contain expected number of bytes. Received {size} / {total}'
message = message.format(size=os.path.getsize(localname), total=plan['remote_total_bytes'])
raise Exception(message)
if localname != plan['real_localname']:
os.rename(localname, plan['real_localname'])
return plan['real_localname']
@ -134,7 +135,7 @@ def prepare_plan(
elif isinstance(bytespersecond, ratelimiter.Ratelimiter):
limiter = bytespersecond
else:
limiter = ratelimiter.Ratelimiter(bytespersecond)
limiter = ratelimiter.Ratelimiter(allowance=bytespersecond)
# Chapter 3: Extracting range
if user_provided_range:
@ -222,63 +223,95 @@ def prepare_plan(
print('No plan was chosen?')
return None
class Progress1:
def __init__(self, total_bytes):
self.limiter = ratelimiter.Ratelimiter(allowance=5, mode='reject')
self.limiter.balance = 1
self.total_bytes = max(1, total_bytes)
self.divisor = bytestring.get_appropriate_divisor(total_bytes)
self.total_format = bytestring.bytestring(total_bytes, force_unit=self.divisor)
self.downloaded_format = '{:>%d}' % len(self.total_format)
self.blank_char = ' '
self.solid_char = ''
def step(self, bytes_downloaded):
#print(self.limiter.balance)
percent = bytes_downloaded / self.total_bytes
percent = min(1, percent)
if self.limiter.limit(1) is False and percent < 1:
return
downloaded_string = bytestring.bytestring(bytes_downloaded, force_unit=self.divisor)
downloaded_string = self.downloaded_format.format(downloaded_string)
block_count = 50
solid_blocks = self.solid_char * int(block_count * percent)
statusbar = solid_blocks.ljust(block_count, self.blank_char)
statusbar = self.solid_char + statusbar + self.solid_char
end = '\n' if percent == 1 else ''
message = '\r{bytes_downloaded} {statusbar} {total_bytes}'
message = message.format(
bytes_downloaded=downloaded_string,
total_bytes=self.total_format,
statusbar=statusbar,
)
print(message, end=end, flush=True)
class Progress2:
def __init__(self, total_bytes):
self.total_bytes = max(1, total_bytes)
self.limiter = ratelimiter.Ratelimiter(allowance=5, mode='reject')
self.limiter.balance = 1
self.total_bytes_string = '{:,}'.format(self.total_bytes)
self.bytes_downloaded_string = '{:%d,}' % len(self.total_bytes_string)
def step(self, bytes_downloaded):
percent = (bytes_downloaded * 100) / self.total_bytes
percent = min(100, percent)
if self.limiter.limit(1) is False and percent < 100:
return
percent_string = '%08.4f' % percent
bytes_downloaded_string = self.bytes_downloaded_string.format(bytes_downloaded)
end = '\n' if percent == 100 else ''
message = '\r{bytes_downloaded} / {total_bytes} / {percent}%'
message = message.format(
bytes_downloaded=bytes_downloaded_string,
total_bytes=self.total_bytes_string,
percent=percent_string,
)
print(message, end=end, flush=True)
progress1 = Progress1
progress2 = Progress2
def basename_from_url(url):
'''
Determine the local filename appropriate for a URL.
'''
localname = urllib.parse.unquote(url)
localname = localname.split('?')[0]
localname = localname.split('/')[-1]
return localname
def get_permission(prompt='y/n\n>', affirmative=['y', 'yes']):
permission = input(prompt)
return permission.lower() in affirmative
def progress1(bytes_downloaded, bytes_total, prefix=''):
divisor = bytestring.get_appropriate_divisor(bytes_total)
bytes_total_string = bytestring.bytestring(bytes_total, force_unit=divisor)
bytes_downloaded_string = bytestring.bytestring(bytes_downloaded, force_unit=divisor)
bytes_downloaded_string = bytes_downloaded_string.rjust(len(bytes_total_string), ' ')
blocks = 50
char = ''
percent = bytes_downloaded * 100 / bytes_total
percent = int(min(100, percent))
completed_blocks = char * int(blocks * percent / 100)
incompleted_blocks = ' ' * (blocks - len(completed_blocks))
statusbar = '{char}{complete}{incomplete}{char}'.format(
char=char,
complete=completed_blocks,
incomplete=incompleted_blocks,
)
end = '\n' if percent == 100 else ''
message = '\r{prefix}{bytes_downloaded} {statusbar} {bytes_total}'
message = message.format(
prefix=prefix,
bytes_downloaded=bytes_downloaded_string,
bytes_total=bytes_total_string,
statusbar=statusbar,
)
print(message, end=end, flush=True)
def progress2(bytes_downloaded, bytes_total, prefix=''):
percent = (bytes_downloaded*100)/bytes_total
percent = min(100, percent)
percent_string = '%08.4f' % percent
bytes_downloaded_string = '{0:,}'.format(bytes_downloaded)
bytes_total_string = '{0:,}'.format(bytes_total)
bytes_downloaded_string = bytes_downloaded_string.rjust(len(bytes_total_string), ' ')
end = '\n' if percent == 100 else ''
message = '\r{prefix}{bytes_downloaded} / {bytes_total} / {percent}%'
message = message.format(
prefix=prefix,
bytes_downloaded=bytes_downloaded_string,
bytes_total=bytes_total_string,
percent=percent_string,
)
print(message, end=end, flush=True)
def request(method, url, stream=False, headers=None, timeout=TIMEOUT, **kwargs):
if headers is None:
headers = {}
for (key, value) in HEADERS.items():
headers.setdefault(key, value)
session = requests.Session()
a = requests.adapters.HTTPAdapter(max_retries=30)
b = requests.adapters.HTTPAdapter(max_retries=30)
session.mount('http://', a)
session.mount('https://', b)
session.max_redirects = 40
method = {
@ -286,8 +319,7 @@ def request(method, url, stream=False, headers=None, timeout=TIMEOUT, **kwargs):
'head': session.head,
'post': session.post,
}[method]
req = method(url, stream=stream, headers=headers, timeout=timeout, **kwargs)
req = method(url, stream=stream, headers=headers, timeout=None, **kwargs)
req.raise_for_status()
return req
@ -311,11 +343,10 @@ def download_argparse(args):
url = args.url
url = clipext.resolve(url)
callback = {
None: progress1,
'1': progress1,
'2': progress2,
None: Progress1,
'1': Progress1,
'2': Progress2,
}.get(args.callback, args.callback)
bytespersecond = args.bytespersecond

114
Etiquette/bad_search.txt Normal file
View file

@ -0,0 +1,114 @@
start_time = time.time()
# Raise for cases where the minimum > maximum
for (maxkey, maxval) in maximums.items():
if maxkey not in minimums:
continue
minval = minimums[maxkey]
if minval > maxval:
raise ValueError('Impossible min-max for %s' % maxkey)
conditions = []
minmaxers = {'<=': maximums, '>=': minimums}
# Convert the min-max parameters into query strings
print('Writing minmaxers')
for (comparator, minmaxer) in minmaxers.items():
for (field, value) in minmaxer.items():
if field not in Photo.int_properties:
raise ValueError('Unknown Photo property: %s' % field)
value = str(value)
query = min_max_query_builder(field, comparator, value)
conditions.append(query)
print(conditions)
print('Writing extension rule')
if extension is not None:
if isinstance(extension, str):
extension = [extension]
# Normalize to prevent injections
extension = [normalize_tagname(e) for e in extension]
extension = ['extension == "%s"' % e for e in extension]
extension = ' OR '.join(extension)
extension = '(%s)' % extension
conditions.append(extension)
def setify(l):
if l is None:
return set()
else:
return set(self.get_tag_by_name(t) for t in l)
tag_musts = setify(tag_musts)
tag_mays = setify(tag_mays)
tag_forbids = setify(tag_forbids)
base = '''
{negator} EXISTS(
SELECT 1 FROM photo_tag_rel
WHERE photo_tag_rel.photoid == photos.id
AND photo_tag_rel.tagid {operator} {value}
)'''
print('Writing musts')
for tag in tag_musts:
# tagid == must
query = base.format(
negator='',
operator='==',
value='"%s"' % tag.id,
)
conditions.append(query)
print('Writing mays')
if len(tag_mays) > 0:
# not any(tagid not in mays)
acceptable = tag_mays.union(tag_musts)
acceptable = ['"%s"' % t.id for t in acceptable]
acceptable = ', '.join(acceptable)
query = base.format(
negator='',
operator='IN',
value='(%s)' % acceptable,
)
conditions.append(query)
print('Writing forbids')
if len(tag_forbids) > 0:
# not any(tagid in forbids)
forbids = ['"%s"' % t.id for t in tag_forbids]
forbids = ', '.join(forbids)
query = base.format(
negator='NOT',
operator='IN',
value='(%s)' % forbids
)
conditions.append(query)
if len(conditions) == 0:
raise ValueError('No search query provided')
conditions = [query for query in conditions if query is not None]
conditions = ['(%s)' % c for c in conditions]
conditions = ' AND '.join(conditions)
conditions = 'WHERE %s' % conditions
query = 'SELECT * FROM photos %s' % conditions
query = query.replace('\n', ' ')
while ' ' in query:
query = query.replace(' ', ' ')
print(query)
temp_cur = self.sql.cursor()
temp_cur.execute(query)
for fetch in fetch_generator(temp_cur):
photo = Photo(self, fetch)
yield photo
end_time = time.time()
print(end_time - start_time)

1
Etiquette/etiquette.py Normal file
View file

@ -0,0 +1 @@
import phototagger

1163
Etiquette/phototagger.py Normal file

File diff suppressed because it is too large Load diff

View file

Before

Width:  |  Height:  |  Size: 467 KiB

After

Width:  |  Height:  |  Size: 467 KiB

View file

Before

Width:  |  Height:  |  Size: 900 KiB

After

Width:  |  Height:  |  Size: 900 KiB

View file

Before

Width:  |  Height:  |  Size: 321 KiB

After

Width:  |  Height:  |  Size: 321 KiB

View file

@ -12,7 +12,6 @@ class PhotoDBTest(unittest.TestCase):
pass
def test_add_and_remove_photo(self):
self.setUp()
photo1 = self.p.new_photo('samples\\train.jpg')
self.assertEqual(len(photo1.id), self.p.id_length)
@ -42,8 +41,6 @@ class PhotoDBTest(unittest.TestCase):
self.assertEqual(tag.name, 'one_two')
def test_add_and_remove_synonym(self):
self.setUp()
# Add synonym
giraffe = self.p.new_tag('giraffe')
horse = self.p.new_tag_synonym('long horse', 'giraffe')
@ -65,7 +62,6 @@ class PhotoDBTest(unittest.TestCase):
self.assertRaises(phototagger.NoSuchSynonym, self.p.remove_tag_synonym, 'blanc')
def test_apply_photo_tag(self):
self.setUp()
photo = self.p.new_photo('samples\\train.jpg')
self.p.new_tag('vehicles')
@ -77,8 +73,6 @@ class PhotoDBTest(unittest.TestCase):
self.assertFalse(status)
def test_convert_tag_synonym(self):
self.setUp()
# Install tags and a synonym
photo = self.p.new_photo('samples\\train.jpg')
trains = self.p.new_tag('trains')
@ -115,7 +109,6 @@ class PhotoDBTest(unittest.TestCase):
self.assertEqual(tags[0].id, trains_id)
def test_generate_id(self):
self.setUp()
i_photo = self.p.generate_id('photos')
i_tag = self.p.generate_id('tags')
self.assertRaises(ValueError, self.p.generate_id, 'other')
@ -127,21 +120,18 @@ class PhotoDBTest(unittest.TestCase):
self.assertLess(int(i_photo), int(self.p.generate_id('photos')))
def test_get_photo_by_id(self):
self.setUp()
photo = self.p.new_photo('samples\\train.jpg')
photo2 = self.p.get_photo_by_id(photo.id)
self.assertEqual(photo, photo2)
def test_get_photo_by_path(self):
self.setUp()
photo = self.p.new_photo('samples\\train.jpg')
photo2 = self.p.get_photo_by_path(photo.filepath)
self.assertEqual(photo, photo2)
def test_get_photos_by_recent(self):
self.setUp()
paths = ['train.jpg', 'bolts.jpg', 'reddit.png']
paths = ['samples\\' + path for path in paths]
paths = [os.path.abspath(path) for path in paths]
@ -176,7 +166,6 @@ class PhotoDBTest(unittest.TestCase):
self.assertEqual(tag1, tag2)
def test_get_tags_by_photo(self):
self.setUp()
photo = self.p.new_photo('samples\\train.jpg')
tag = self.p.new_tag('vehicles')
stat = self.p.apply_photo_tag(photo.id, tagname='vehicles')
@ -192,7 +181,6 @@ class PhotoDBTest(unittest.TestCase):
self.assertRaises(phototagger.TagTooShort, self.p.new_tag, '!!??&&*')
def test_photo_has_tag(self):
self.setUp()
photo = self.p.new_photo('samples\\train.jpg')
tag = self.p.new_tag('vehicles')
self.p.apply_photo_tag(photo.id, tag.id)

View file

@ -363,6 +363,7 @@ function dump_urls()
{
textbox = document.createElement("textarea");
textbox.className = "urldumpbox";
textbox.id = "url_dump_box";
workspace = document.getElementById("WORKSPACE");
workspace.appendChild(textbox);
}

View file

@ -18,6 +18,9 @@ class Path:
def __hash__(self):
return hash(self.absolute_path)
def __repr__(self):
return '{c}({path})'.format(c=self.__class__, path=self.absolute_path)
@property
def basename(self):
return os.path.basename(self.absolute_path)
@ -79,11 +82,12 @@ def get_path_casing(path):
path = path.absolute_path
(drive, subpath) = os.path.splitdrive(path)
drive = drive.upper()
subpath = subpath.lstrip(os.sep)
pattern = [glob_patternize(piece) for piece in subpath.split(os.sep)]
pattern = os.sep.join(pattern)
pattern = drive.upper() + os.sep + pattern
pattern = drive + os.sep + pattern
#print(pattern)
try:
return glob.glob(pattern)[0]

Binary file not shown.

View file

@ -1,894 +0,0 @@
import datetime
import os
import PIL.Image
import random
import sqlite3
import string
import warnings
ID_LENGTH = 22
VALID_TAG_CHARS = string.ascii_lowercase + string.digits + '_-'
MAX_TAG_NAME_LENGTH = 32
SQL_LASTID_COLUMNS = [
'table',
'last_id',
]
SQL_PHOTO_COLUMNS = [
'id',
'filepath',
'extension',
'width',
'height',
'ratio',
'area',
'bytes',
'created',
]
SQL_PHOTOTAG_COLUMNS = [
'photoid',
'tagid',
]
SQL_SYN_COLUMNS = [
'name',
'master',
]
SQL_TAG_COLUMNS = [
'id',
'name',
]
SQL_LASTID = {key:index for (index, key) in enumerate(SQL_LASTID_COLUMNS)}
SQL_PHOTO = {key:index for (index, key) in enumerate(SQL_PHOTO_COLUMNS)}
SQL_PHOTOTAG = {key:index for (index, key) in enumerate(SQL_PHOTOTAG_COLUMNS)}
SQL_SYN = {key:index for (index, key) in enumerate(SQL_SYN_COLUMNS)}
SQL_TAG = {key:index for (index, key) in enumerate(SQL_TAG_COLUMNS)}
DB_INIT = '''
CREATE TABLE IF NOT EXISTS photos(
id TEXT,
filepath TEXT,
extension TEXT,
width INT,
height INT,
ratio REAL,
area INT,
bytes INT,
created INT
);
CREATE TABLE IF NOT EXISTS tags(
id TEXT,
name TEXT
);
CREATE TABLE IF NOT EXISTS albums(
albumid TEXT,
photoid TEXT
);
CREATE TABLE IF NOT EXISTS photo_tag_rel(
photoid TEXT,
tagid TEXT
);
CREATE TABLE IF NOT EXISTS tag_synonyms(
name TEXT,
mastername TEXT
);
CREATE TABLE IF NOT EXISTS id_numbers(
tab TEXT,
last_id TEXT
);
CREATE INDEX IF NOT EXISTS index_photo_id on photos(id);
CREATE INDEX IF NOT EXISTS index_photo_path on photos(filepath);
CREATE INDEX IF NOT EXISTS index_photo_created on photos(created);
CREATE INDEX IF NOT EXISTS index_tag_id on tags(id);
CREATE INDEX IF NOT EXISTS index_tag_name on tags(name);
CREATE INDEX IF NOT EXISTS index_tagrel_photoid on photo_tag_rel(photoid);
CREATE INDEX IF NOT EXISTS index_tagrel_tagid on photo_tag_rel(tagid);
CREATE INDEX IF NOT EXISTS index_tagsyn_name on tag_synonyms(name);
'''
def basex(number, base, alphabet='0123456789abcdefghijklmnopqrstuvwxyz'):
'''
Converts an integer to a different base string.
Based on http://stackoverflow.com/a/1181922/5430534
'''
if base > len(alphabet):
raise Exception('alphabet %s does not support base %d' % (
repr(alphabet), base))
alphabet = alphabet[:base]
if not isinstance(number, (int, str)):
raise TypeError('number must be an integer')
number = int(number)
based = ''
sign = ''
if number < 0:
sign = '-'
number = -number
if 0 <= number < len(alphabet):
return sign + alphabet[number]
while number != 0:
number, i = divmod(number, len(alphabet))
based = alphabet[i] + based
return sign + based
def fetch_generator(cursor):
while True:
fetch = cursor.fetchone()
if fetch is None:
break
yield fetch
def getnow(timestamp=True):
'''
Return the current UTC timestamp or datetime object.
'''
now = datetime.datetime.now(datetime.timezone.utc)
if timestamp:
return now.timestamp()
return now
def is_xor(*args):
'''
Return True if and only if one arg is truthy.
'''
return [bool(a) for a in args].count(True) == 1
def min_max_query_builder(name, comparator, value):
return ' '.join([name, comparator, value])
def normalize_tagname(tagname):
'''
Tag names can only consist of lowercase letters, underscores,
and hyphens. The given tagname is lowercased, gets its spaces
replaced by underscores, and is stripped of any not-whitelisted
characters.
'''
tagname = tagname.lower()
tagname = tagname.replace(' ', '_')
tagname = (c for c in tagname if c in VALID_TAG_CHARS)
tagname = ''.join(tagname)
if len(tagname) == 0:
raise TagTooShort(tagname)
if len(tagname) > MAX_TAG_NAME_LENGTH:
raise TagTooLong(tagname)
return tagname
def not_implemented(function):
'''
Decorator for keeping track of which functions still need to be filled out.
'''
warnings.warn('%s is not implemented' % function.__name__)
return function
def raise_nosuchtag(tagid=None, tagname=None, comment=''):
if tagid is not None:
message = 'ID: %s. %s' % (tagid, comment)
elif tagname is not None:
message = 'Name: %s. %s' % (tagname, comment)
raise NoSuchTag(message)
class NoSuchPhoto(Exception):
pass
class NoSuchSynonym(Exception):
pass
class NoSuchTag(Exception):
pass
class PhotoExists(Exception):
pass
class TagExists(Exception):
pass
class TagTooLong(Exception):
pass
class TagTooShort(Exception):
pass
class XORException(Exception):
pass
class PhotoDB:
'''
This class represents an SQLite3 database containing the following tables:
photos:
Rows represent image files on the local disk.
Entries contain a unique ID, the image's filepath, and metadata
like dimensions and filesize.
tags:
Rows represent labels, which can be applied to an arbitrary number of
photos. Photos may be selected by which tags they contain.
Entries contain a unique ID and a name.
albums:
Rows represent the inclusion of a photo in an album
photo_tag_rel:
Rows represent a Photo's ownership of a particular Tag.
tag_synonyms:
Rows represent relationships between two tag names, so that they both
resolve to the same Tag object when selected. Entries contain the
subordinate name and master name.
The master name MUST also exist in the `tags` table.
If a new synonym is created referring to another synoym, the master name
will be resolved and used instead, so a synonym is never in the master
column.
Tag objects will ALWAYS represent the master tag.
Note that the entries in this table do not contain ID numbers.
The rationale here is that "coco" is a synonym for "chocolate" regardless
of the "chocolate" tag's ID, and that if a tag is renamed, its synonyms
do not necessarily follow.
The `rename_tag` method includes a parameter `apply_to_synonyms` if you do
want them to follow.
'''
def __init__(self, databasename='phototagger.db', id_length=None):
if id_length is None:
self.id_length = ID_LENGTH
self.databasename = databasename
self.sql = sqlite3.connect(databasename)
self.cur = self.sql.cursor()
statements = DB_INIT.split(';')
for statement in statements:
self.cur.execute(statement)
self._last_ids = {}
def __repr__(self):
return 'PhotoDB(databasename={dbname})'.format(dbname=repr(self.databasename))
def apply_photo_tag(self, photoid, tagid=None, tagname=None, commit=True):
'''
Apply a tag to a photo. `tag` may be the name of the tag or a Tag
object from the same PhotoDB.
`tag` may NOT be the tag's ID, since we can't tell if a given string is
an ID or a name.
Returns True if the tag was applied, False if the photo already had this tag.
Raises NoSuchTag and NoSuchPhoto as appropriate.
'''
tag = self.get_tag(tagid=tagid, tagname=tagname, resolve_synonyms=True)
self.cur.execute('SELECT * FROM photo_tag_rel WHERE photoid == ? AND tagid == ?', [photoid, tag.id])
if self.cur.fetchone() is not None:
return False
self.cur.execute('INSERT INTO photo_tag_rel VALUES(?, ?)', [photoid, tag.id])
if commit:
self.sql.commit()
return True
def convert_tag_to_synonym(self, oldtagname, mastertagname):
'''
Convert an independent tag into a synonym for a different independent tag.
All photos which possess the current tag will have it replaced
with the master tag. All synonyms of the old tag will point to the new tag.
Good for when two tags need to be merged under a single name.
'''
oldtagname = normalize_tagname(oldtagname)
mastertagname = normalize_tagname(mastertagname)
oldtag = self.get_tag_by_name(oldtagname, resolve_synonyms=False)
if oldtag is None:
raise NoSuchTag(oldtagname)
mastertag = self.get_tag_by_name(mastertagname, resolve_synonyms=False)
if mastertag is None:
raise NoSuchTag(mastertagname)
# Migrate the old tag's synonyms to the new one
# UPDATE is safe for this operation because there is no chance of duplicates.
self.cur.execute('UPDATE tag_synonyms SET mastername = ? WHERE mastername == ?', [mastertagname, oldtagname])
# Iterate over all photos with the old tag, and relate them to the new tag
# if they aren't already.
temp_cur = self.sql.cursor()
temp_cur.execute('SELECT * FROM photo_tag_rel WHERE tagid == ?', [oldtag.id])
for relationship in fetch_generator(temp_cur):
photoid = relationship[SQL_PHOTOTAG['photoid']]
self.cur.execute('SELECT * FROM photo_tag_rel WHERE tagid == ?', [mastertag.id])
if self.cur.fetchone() is not None:
continue
self.cur.execute('INSERT INTO photo_tag_rel VALUES(?, ?)', [photoid, mastertag.id])
# Then delete the relationships with the old tag
self.cur.execute('DELETE FROM photo_tag_rel WHERE tagid == ?', [oldtag.id])
self.cur.execute('DELETE FROM tags WHERE id == ?', [oldtag.id])
# Enjoy your new life as a monk.
self.new_tag_synonym(oldtag.name, mastertag.name, commit=False)
self.sql.commit()
def delete_photo(self, photoid):
'''
Delete a photo and its relation to any tags and albums.
'''
photo = self.get_photo_by_id(photoid)
if photo is None:
raise NoSuchPhoto(photoid)
self.cur.execute('DELETE FROM photos WHERE id == ?', [photoid])
self.cur.execute('DELETE FROM photo_tag_rel WHERE photoid == ?', [photoid])
self.sql.commit()
def delete_tag(self, tagid=None, tagname=None):
'''
Delete a tag, its synonyms, and its relation to any photos.
'''
tag = self.get_tag(tagid=tagid, tagname=tagname, resolve_synonyms=False)
if tag is None:
message = 'Is it a synonym?'
raise_nosuchtag(tagid=tagid, tagname=tagname, comment=message)
self.cur.execute('DELETE FROM tags WHERE id == ?', [tag.id])
self.cur.execute('DELETE FROM photo_tag_rel WHERE tagid == ?', [tag.id])
self.cur.execute('DELETE FROM tag_synonyms WHERE mastername == ?', [tag.name])
self.sql.commit()
def delete_tag_synonym(self, tagname):
'''
Delete a tag synonym.
This will have no effect on photos or other synonyms because
they always resolve to the master tag before application.
'''
tagname = normalize_tagname(tagname)
self.cur.execute('SELECT * FROM tag_synonyms WHERE name == ?', [tagname])
fetch = self.cur.fetchone()
if fetch is None:
raise NoSuchSynonym(tagname)
self.cur.execute('DELETE FROM tag_synonyms WHERE name == ?', [tagname])
self.sql.commit()
def generate_id(self, table):
'''
Create a new ID number that is unique to the given table.
Note that this method does not commit the database. We'll wait for that
to happen in whoever is calling us, so we know the ID is actually used.
'''
table = table.lower()
if table not in ['photos', 'tags']:
raise ValueError('Invalid table requested: %s.', table)
do_update = False
if table in self._last_ids:
# Use cache value
new_id = self._last_ids[table] + 1
do_update = True
else:
self.cur.execute('SELECT * FROM id_numbers WHERE tab == ?', [table])
fetch = self.cur.fetchone()
if fetch is None:
# Register new value
new_id = 1
else:
# Use database value
new_id = int(fetch[SQL_LASTID['last_id']]) + 1
do_update = True
new_id_s = str(new_id).rjust(self.id_length, '0')
if do_update:
self.cur.execute('UPDATE id_numbers SET last_id = ? WHERE tab == ?', [new_id_s, table])
else:
self.cur.execute('INSERT INTO id_numbers VALUES(?, ?)', [table, new_id_s])
self._last_ids[table] = new_id
return new_id_s
@not_implemented
def get_album_by_id(self, albumid):
return
def get_photo_by_id(self, photoid):
'''
Return this Photo object, or None if it does not exist.
'''
self.cur.execute('SELECT * FROM photos WHERE id == ?', [photoid])
photo = self.cur.fetchone()
if photo is None:
return None
photo = self.tuple_to_photo(photo)
return photo
def get_photo_by_path(self, filepath):
'''
Return this Photo object, or None if it does not exist.
'''
filepath = os.path.abspath(filepath)
self.cur.execute('SELECT * FROM photos WHERE filepath == ?', [filepath])
photo = self.cur.fetchone()
if photo is None:
return None
photo = self.tuple_to_photo(photo)
return photo
def get_photos_by_recent(self, count=None):
'''
Yield photo objects in order of creation time.
'''
if count is not None and count <= 0:
return
# We're going to use a second cursor because the first one may
# get used for something else, deactivating this query.
temp_cur = self.sql.cursor()
temp_cur.execute('SELECT * FROM photos ORDER BY created DESC')
while True:
f = temp_cur.fetchone()
if f is None:
return
photo = self.tuple_to_photo(f)
yield photo
if count is None:
continue
count -= 1
if count <= 0:
return
@not_implemented
def get_photos_by_search(
self,
extension=None,
maximums={},
minimums={},
tag_musts=None,
tag_mays=None,
tag_forbids=None,
tag_forbid_unspecified=False,
):
'''
Given one or multiple tags, yield photos possessing those tags.
Parameters:
extension :
A string or list of strings of acceptable file extensions.
maximums :
A dictionary, where the key is an attribute of the photo,
(area, bytes, created, height, id, or width)
and the value is the maximum desired value for that field.
minimums :
A dictionary like `maximums` where the value is the minimum
desired value for that field.
tag_musts :
A list of tag names or Tag objects.
Photos MUST have ALL tags in this list.
tag_mays :
A list of tag names or Tag objects.
If `forbid_unspecified` is True, then Photos MUST have AT LEAST ONE tag in this list.
If `forbid_unspecified` is False, then Photos MAY or MAY NOT have ANY tag in this list.
tag_forbids :
A list of tag names or Tag objects.
Photos MUST NOT have ANY tag in the list.
tag_forbid_unspecified :
True or False.
If False, Photos need only comply with the `tag_musts`.
If True, Photos need to comply with both `tag_musts` and `tag_mays`.
'''
maximums = {key:int(val) for (key, val) in maximums.items()}
minimums = {key:int(val) for (key, val) in minimums.items()}
# Raise for cases where the minimum > maximum
for (maxkey, maxval) in maximums.items():
if maxkey not in minimums:
continue
minval = minimums[maxkey]
if minval > maxval:
raise ValueError('Impossible min-max for %s' % maxkey)
conditions = []
minmaxers = {'<=': maximums, '>=': minimums}
# Convert the min-max parameters into query strings
for (comparator, minmaxer) in minmaxers.items():
for (field, value) in minmaxer.items():
if field not in Photo.int_properties:
raise ValueError('Unknown Photo property: %s' % field)
value = str(value)
query = min_max_query_builder(field, comparator, value)
conditions.append(query)
if extension is not None:
if isinstance(extension, str):
extension = [extension]
# Normalize to prevent injections
extension = [normalize_tagname(e) for e in extension]
extension = ['extension == "%s"' % e for e in extension]
extension = ' OR '.join(extension)
extension = '(%s)' % extension
conditions.append(extension)
def setify(l):
return set(self.get_tag_by_name(t) for t in l) if l else set()
tag_musts = setify(tag_musts)
tag_mays = setify(tag_mays)
tag_forbids = setify(tag_forbids)
base = '%s EXISTS (SELECT 1 FROM photo_tag_rel WHERE photo_tag_rel.photoid == photos.id AND photo_tag_rel.tagid %s %s)'
for tag in tag_musts:
query = base % ('', '==', '"%s"' % tag.id)
conditions.append(query)
if tag_forbid_unspecified and len(tag_mays) > 0:
acceptable = tag_mays.union(tag_musts)
acceptable = ['"%s"' % t.id for t in acceptable]
acceptable = ', '.join(acceptable)
query = base % ('NOT', 'NOT IN', '(%s)' % acceptable)
conditions.append(query)
for tag in tag_forbids:
query = base % ('NOT', '==', '"%s"' % tag.id)
conditions.append(query)
if len(conditions) == 0:
raise ValueError('No search query provided')
conditions = [query for query in conditions if query is not None]
conditions = ['(%s)' % c for c in conditions]
conditions = ' AND '.join(conditions)
conditions = 'WHERE %s' % conditions
query = 'SELECT * FROM photos %s' % conditions
print(query)
temp_cur = self.sql.cursor()
temp_cur.execute(query)
acceptable_tags = tag_musts.union(tag_mays)
while True:
fetch = temp_cur.fetchone()
if fetch is None:
break
photo = self.tuple_to_photo(fetch)
# if any(forbid in photo.tags for forbid in tag_forbids):
# print('Forbidden')
# continue
# if tag_forbid_unspecified:
# if any(tag not in acceptable_tags for tag in photo.tags):
# print('Forbid unspecified')
# continue
# if any(must not in photo.tags for must in tag_musts):
# print('No must')
# continue
yield photo
def get_tag(self, tagid=None, tagname=None, resolve_synonyms=True):
'''
Redirect to get_tag_by_id or get_tag_by_name after xor-checking the parameters.
'''
if not is_xor(tagid, tagname):
raise XORException('One and only one of `tagid`, `tagname` can be passed.')
if tagid is not None:
return self.get_tag_by_id(tagid)
elif tagname is not None:
return self.get_tag_by_name(tagname, resolve_synonyms=resolve_synonyms)
raise_nosuchtag(tagid=tagid, tagname=tagname)
def get_tag_by_id(self, tagid):
self.cur.execute('SELECT * FROM tags WHERE id == ?', [tagid])
tag = self.cur.fetchone()
if tag is None:
return raise_nosuchtag(tagid=tagid)
tag = self.tuple_to_tag(tag)
return tag
def get_tag_by_name(self, tagname, resolve_synonyms=True):
if isinstance(tagname, Tag):
return tagname
tagname = normalize_tagname(tagname)
if resolve_synonyms is True:
self.cur.execute('SELECT * FROM tag_synonyms WHERE name == ?', [tagname])
fetch = self.cur.fetchone()
if fetch is not None:
mastertagname = fetch[SQL_SYN['master']]
tag = self.get_tag_by_name(mastertagname)
return tag
self.cur.execute('SELECT * FROM tags WHERE name == ?', [tagname])
fetch = self.cur.fetchone()
if fetch is None:
raise_nosuchtag(tagname=tagname)
tag = self.tuple_to_tag(fetch)
return tag
def get_tags_by_photo(self, photoid):
'''
Return the tags assigned to the given photo.
'''
temp_cur = self.sql.cursor()
temp_cur.execute('SELECT * FROM photo_tag_rel WHERE photoid == ?', [photoid])
tags = fetch_generator(temp_cur)
tagobjects = set()
for tag in tags:
tagid = tag[SQL_PHOTOTAG['tagid']]
tagobj = self.get_tag_by_id(tagid)
tagobjects.add(tagobj)
return tagobjects
def new_photo(self, filename, tags=None, allow_duplicates=False):
'''
Given a filepath, determine its attributes and create a new Photo object in the
database. Tags may be applied now or later.
If `allow_duplicates` is False, we will first check the database for any files
with the same path and raise PhotoExists if found.
Returns the Photo object.
'''
filename = os.path.abspath(filename)
if not allow_duplicates:
existing = self.get_photo_by_path(filename)
if existing is not None:
raise PhotoExists(filename, existing)
# I want the caller to receive any exceptions this raises.
image = PIL.Image.open(filename)
extension = os.path.splitext(filename)[1]
extension = extension.replace('.', '')
extension = normalize_tagname(extension)
(width, height) = image.size
area = width * height
ratio = width / height
bytes = os.path.getsize(filename)
created = int(getnow())
photoid = self.generate_id('photos')
data = [None] * len(SQL_PHOTO_COLUMNS)
data[SQL_PHOTO['id']] = photoid
data[SQL_PHOTO['filepath']] = filename
data[SQL_PHOTO['extension']] = extension
data[SQL_PHOTO['width']] = width
data[SQL_PHOTO['height']] = height
data[SQL_PHOTO['area']] = area
data[SQL_PHOTO['ratio']] = ratio
data[SQL_PHOTO['bytes']] = bytes
data[SQL_PHOTO['created']] = created
photo = self.tuple_to_photo(data)
self.cur.execute('INSERT INTO photos VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)', data)
tags = tags or []
for tag in tags:
try:
self.apply_photo_tag(photoid, tagname=tag, commit=False)
except NoSuchTag:
self.sql.rollback()
raise
self.sql.commit()
image.close()
return photo
def new_tag(self, tagname):
'''
Register a new tag in the database and return the Tag object.
'''
tagname = normalize_tagname(tagname)
try:
self.get_tag_by_name(tagname)
TagExists(tagname)
except NoSuchTag:
pass
tagid = self.generate_id('tags')
self.cur.execute('INSERT INTO tags VALUES(?, ?)', [tagid, tagname])
self.sql.commit()
tag = self.tuple_to_tag([tagid, tagname])
return tag
def new_tag_synonym(self, tagname, mastertagname, commit=True):
'''
Register a new synonym for an existing tag.
'''
tagname = normalize_tagname(tagname)
mastertagname = normalize_tagname(mastertagname)
if tagname == mastertagname:
raise ValueError('Cannot assign synonym to itself.')
# We leave resolve_synonyms as True, so that if this function returns
# anything, we know the given tagname is already a synonym or master.
tag = self.get_tag_by_name(tagname, resolve_synonyms=True)
if tag is not None:
raise TagExists(tagname)
mastertag = self.get_tag_by_name(mastertagname, resolve_synonyms=True)
if mastertag is None:
raise NoSuchTag(mastertagname)
self.cur.execute('INSERT INTO tag_synonyms VALUES(?, ?)', [tagname, mastertag.name])
if commit:
self.sql.commit()
return mastertag
def photo_has_tag(self, photoid, tagid=None, tagname=None):
tag = self.get_tag(tagid=tagid, tagname=tagname, resolve_synonyms=True)
if tag is None:
raise_nosuchtag(tagid=tagid, tagname=tagname)
exe = self.cur.execute
exe('SELECT * FROM photo_tag_rel WHERE photoid == ? AND tagid == ?', [photoid, tag.id])
fetch = self.cur.fetchone()
has_tag = fetch is not None
return has_tag
@not_implemented
def rename_tag(self, tagname, newname, apply_to_synonyms):
pass
def tuple_to_photo(self, tu):
'''
Given a tuple like the ones from an sqlite query,
create a Photo object.
'''
photoid = tu[SQL_PHOTO['id']]
tags = self.get_tags_by_photo(photoid)
photo = Photo(
photodb = self,
photoid = photoid,
filepath = tu[SQL_PHOTO['filepath']],
extension = tu[SQL_PHOTO['extension']],
width = tu[SQL_PHOTO['width']],
height = tu[SQL_PHOTO['height']],
created = tu[SQL_PHOTO['created']],
bytes = tu[SQL_PHOTO['bytes']],
tags = tags,
)
return photo
def tuple_to_tag(self, tu):
'''
Given a tuple like the ones from an sqlite query,
create a Tag object.
'''
tag = Tag(
photodb = self,
tagid = tu[SQL_TAG['id']],
name = tu[SQL_TAG['name']]
)
return tag
class Photo:
'''
This class represents a PhotoDB entry containing information about an image file.
Photo objects cannot exist without a corresponding PhotoDB object, because
Photos are not the actual files, just the database entry.
'''
int_properties = set(['area', 'bytes', 'created', 'height', 'id', 'width'])
def __init__(
self,
photodb,
photoid,
filepath,
extension,
width,
height,
bytes,
created,
tags=None,
):
if tags is None:
tags = []
self.photodb = photodb
self.id = photoid
self.filepath = filepath
self.extension = extension
self.width = int(width)
self.height = int(height)
self.ratio = self.width / self.height
self.area = self.width * self.height
self.bytes = int(bytes)
self.created = int(created)
self.tags = tags
def __eq__(self, other):
if not isinstance(other, Photo):
return False
return self.id == other.id
def __hash__(self):