else
prepare to remove TagGroups
This commit is contained in:
Ethan Dalool
parent
82f63a75ab
commit
bfaed2e416
20 changed files with 1522 additions and 1017 deletions
18
AHK/turboclick.ahk
Normal file
18
AHK/turboclick.ahk
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
rapid_switch := false
|
||||
|
||||
~*CapsLock:: rapid_switch := !rapid_switch
|
||||
|
||||
~*LButton::
|
||||
If rapid_switch = 1
|
||||
{
|
||||
Loop
|
||||
{
|
||||
GetKeyState, var, LButton, P
|
||||
If var = U
|
||||
{
|
||||
Break
|
||||
}
|
||||
Send {LButton}
|
||||
sleep 20
|
||||
}
|
||||
}
|
||||
|
|
@ -12,7 +12,7 @@ def multi_line_input():
|
|||
except EOFError:
|
||||
# If you enter nothing but ctrl-z
|
||||
additional = EOF
|
||||
|
||||
else:
|
||||
userinput.append(additional)
|
||||
|
||||
if EOF in additional:
|
||||
|
|
|
|||
|
|
@ -7,9 +7,9 @@ import time
|
|||
import urllib
|
||||
import warnings
|
||||
|
||||
sys.path.append('C:\\git\\else\\bytestring'); import bytestring
|
||||
sys.path.append('C:\\git\\else\\clipext'); import clipext
|
||||
sys.path.append('C:\\git\\else\\ratelimiter'); import ratelimiter
|
||||
sys.path.append('C:\\git\\else\\bytestring'); import bytestring
|
||||
|
||||
warnings.simplefilter('ignore')
|
||||
|
||||
|
|
@ -20,18 +20,12 @@ HEADERS = {
|
|||
FILENAME_BADCHARS = '*?"<>|'
|
||||
|
||||
last_request = 0
|
||||
CHUNKSIZE = 16 * bytestring.KIBIBYTE
|
||||
CHUNKSIZE = 4 * bytestring.KIBIBYTE
|
||||
TIMEOUT = 600
|
||||
TEMP_EXTENSION = '.downloadytemp'
|
||||
|
||||
def basename_from_url(url):
|
||||
'''
|
||||
Determine the local filename appropriate for a URL.
|
||||
'''
|
||||
localname = urllib.parse.unquote(url)
|
||||
localname = localname.split('?')[0]
|
||||
localname = localname.split('/')[-1]
|
||||
return localname
|
||||
PRINT_LIMITER = ratelimiter.Ratelimiter(allowance=5, mode='reject')
|
||||
|
||||
|
||||
def download_file(
|
||||
url,
|
||||
|
|
@ -41,6 +35,7 @@ def download_file(
|
|||
callback_progress=None,
|
||||
headers=None,
|
||||
overwrite=False,
|
||||
raise_for_undersized=True,
|
||||
verbose=False,
|
||||
):
|
||||
headers = headers or {}
|
||||
|
|
@ -85,26 +80,32 @@ def download_file(
|
|||
bytes_downloaded = 0
|
||||
|
||||
download_stream = request('get', url, stream=True, headers=headers, auth=auth)
|
||||
if callback_progress is not None:
|
||||
callback_progress = callback_progress(plan['remote_total_bytes'])
|
||||
|
||||
for chunk in download_stream.iter_content(chunk_size=CHUNKSIZE):
|
||||
bytes_downloaded += len(chunk)
|
||||
file_handle.write(chunk)
|
||||
if callback_progress is not None:
|
||||
callback_progress(bytes_downloaded, plan['remote_total_bytes'])
|
||||
callback_progress.step(bytes_downloaded)
|
||||
|
||||
if plan['limiter'] is not None and bytes_downloaded < plan['remote_total_bytes']:
|
||||
plan['limiter'].limit(len(chunk))
|
||||
|
||||
file_handle.close()
|
||||
|
||||
# Don't try to rename /dev/null
|
||||
if os.devnull not in [localname, plan['real_localname']]:
|
||||
localsize = os.path.getsize(localname)
|
||||
undersized = plan['plan_type'] != 'partial' and localsize < plan['remote_total_bytes']
|
||||
if raise_for_undersized and undersized:
|
||||
message = 'File does not contain expected number of bytes. Received {size} / {total}'
|
||||
message = message.format(size=localsize, total=plan['remote_total_bytes'])
|
||||
raise Exception(message)
|
||||
|
||||
if localname != plan['real_localname']:
|
||||
os.rename(localname, plan['real_localname'])
|
||||
|
||||
localsize = os.path.getsize(plan['real_localname'])
|
||||
if plan['plan_type'] != 'partial' and localsize < plan['remote_total_bytes']:
|
||||
message = 'File does not contain expected number of bytes. Received {size} / {total}'
|
||||
message = message.format(size=os.path.getsize(localname), total=plan['remote_total_bytes'])
|
||||
raise Exception(message)
|
||||
|
||||
return plan['real_localname']
|
||||
|
||||
def prepare_plan(
|
||||
|
|
@ -134,7 +135,7 @@ def prepare_plan(
|
|||
elif isinstance(bytespersecond, ratelimiter.Ratelimiter):
|
||||
limiter = bytespersecond
|
||||
else:
|
||||
limiter = ratelimiter.Ratelimiter(bytespersecond)
|
||||
limiter = ratelimiter.Ratelimiter(allowance=bytespersecond)
|
||||
|
||||
# Chapter 3: Extracting range
|
||||
if user_provided_range:
|
||||
|
|
@ -223,62 +224,94 @@ def prepare_plan(
|
|||
print('No plan was chosen?')
|
||||
return None
|
||||
|
||||
def get_permission(prompt='y/n\n>', affirmative=['y', 'yes']):
|
||||
permission = input(prompt)
|
||||
return permission.lower() in affirmative
|
||||
|
||||
def progress1(bytes_downloaded, bytes_total, prefix=''):
|
||||
divisor = bytestring.get_appropriate_divisor(bytes_total)
|
||||
bytes_total_string = bytestring.bytestring(bytes_total, force_unit=divisor)
|
||||
bytes_downloaded_string = bytestring.bytestring(bytes_downloaded, force_unit=divisor)
|
||||
bytes_downloaded_string = bytes_downloaded_string.rjust(len(bytes_total_string), ' ')
|
||||
class Progress1:
|
||||
def __init__(self, total_bytes):
|
||||
self.limiter = ratelimiter.Ratelimiter(allowance=5, mode='reject')
|
||||
self.limiter.balance = 1
|
||||
self.total_bytes = max(1, total_bytes)
|
||||
self.divisor = bytestring.get_appropriate_divisor(total_bytes)
|
||||
self.total_format = bytestring.bytestring(total_bytes, force_unit=self.divisor)
|
||||
self.downloaded_format = '{:>%d}' % len(self.total_format)
|
||||
self.blank_char = ' '
|
||||
self.solid_char = '█'
|
||||
|
||||
blocks = 50
|
||||
char = '█'
|
||||
percent = bytes_downloaded * 100 / bytes_total
|
||||
percent = int(min(100, percent))
|
||||
completed_blocks = char * int(blocks * percent / 100)
|
||||
incompleted_blocks = ' ' * (blocks - len(completed_blocks))
|
||||
statusbar = '{char}{complete}{incomplete}{char}'.format(
|
||||
char=char,
|
||||
complete=completed_blocks,
|
||||
incomplete=incompleted_blocks,
|
||||
)
|
||||
def step(self, bytes_downloaded):
|
||||
#print(self.limiter.balance)
|
||||
percent = bytes_downloaded / self.total_bytes
|
||||
percent = min(1, percent)
|
||||
if self.limiter.limit(1) is False and percent < 1:
|
||||
return
|
||||
|
||||
end = '\n' if percent == 100 else ''
|
||||
message = '\r{prefix}{bytes_downloaded} {statusbar} {bytes_total}'
|
||||
downloaded_string = bytestring.bytestring(bytes_downloaded, force_unit=self.divisor)
|
||||
downloaded_string = self.downloaded_format.format(downloaded_string)
|
||||
block_count = 50
|
||||
solid_blocks = self.solid_char * int(block_count * percent)
|
||||
statusbar = solid_blocks.ljust(block_count, self.blank_char)
|
||||
statusbar = self.solid_char + statusbar + self.solid_char
|
||||
|
||||
end = '\n' if percent == 1 else ''
|
||||
message = '\r{bytes_downloaded} {statusbar} {total_bytes}'
|
||||
message = message.format(
|
||||
prefix=prefix,
|
||||
bytes_downloaded=bytes_downloaded_string,
|
||||
bytes_total=bytes_total_string,
|
||||
bytes_downloaded=downloaded_string,
|
||||
total_bytes=self.total_format,
|
||||
statusbar=statusbar,
|
||||
)
|
||||
print(message, end=end, flush=True)
|
||||
|
||||
def progress2(bytes_downloaded, bytes_total, prefix=''):
|
||||
percent = (bytes_downloaded*100)/bytes_total
|
||||
|
||||
class Progress2:
|
||||
def __init__(self, total_bytes):
|
||||
self.total_bytes = max(1, total_bytes)
|
||||
self.limiter = ratelimiter.Ratelimiter(allowance=5, mode='reject')
|
||||
self.limiter.balance = 1
|
||||
self.total_bytes_string = '{:,}'.format(self.total_bytes)
|
||||
self.bytes_downloaded_string = '{:%d,}' % len(self.total_bytes_string)
|
||||
|
||||
def step(self, bytes_downloaded):
|
||||
percent = (bytes_downloaded * 100) / self.total_bytes
|
||||
percent = min(100, percent)
|
||||
if self.limiter.limit(1) is False and percent < 100:
|
||||
return
|
||||
|
||||
percent_string = '%08.4f' % percent
|
||||
bytes_downloaded_string = '{0:,}'.format(bytes_downloaded)
|
||||
bytes_total_string = '{0:,}'.format(bytes_total)
|
||||
bytes_downloaded_string = bytes_downloaded_string.rjust(len(bytes_total_string), ' ')
|
||||
bytes_downloaded_string = self.bytes_downloaded_string.format(bytes_downloaded)
|
||||
|
||||
end = '\n' if percent == 100 else ''
|
||||
message = '\r{prefix}{bytes_downloaded} / {bytes_total} / {percent}%'
|
||||
message = '\r{bytes_downloaded} / {total_bytes} / {percent}%'
|
||||
message = message.format(
|
||||
prefix=prefix,
|
||||
bytes_downloaded=bytes_downloaded_string,
|
||||
bytes_total=bytes_total_string,
|
||||
total_bytes=self.total_bytes_string,
|
||||
percent=percent_string,
|
||||
)
|
||||
print(message, end=end, flush=True)
|
||||
|
||||
progress1 = Progress1
|
||||
progress2 = Progress2
|
||||
|
||||
def basename_from_url(url):
|
||||
'''
|
||||
Determine the local filename appropriate for a URL.
|
||||
'''
|
||||
localname = urllib.parse.unquote(url)
|
||||
localname = localname.split('?')[0]
|
||||
localname = localname.split('/')[-1]
|
||||
return localname
|
||||
|
||||
def get_permission(prompt='y/n\n>', affirmative=['y', 'yes']):
|
||||
permission = input(prompt)
|
||||
return permission.lower() in affirmative
|
||||
|
||||
def request(method, url, stream=False, headers=None, timeout=TIMEOUT, **kwargs):
|
||||
if headers is None:
|
||||
headers = {}
|
||||
for (key, value) in HEADERS.items():
|
||||
headers.setdefault(key, value)
|
||||
session = requests.Session()
|
||||
a = requests.adapters.HTTPAdapter(max_retries=30)
|
||||
b = requests.adapters.HTTPAdapter(max_retries=30)
|
||||
session.mount('http://', a)
|
||||
session.mount('https://', b)
|
||||
session.max_redirects = 40
|
||||
|
||||
method = {
|
||||
|
|
@ -286,8 +319,7 @@ def request(method, url, stream=False, headers=None, timeout=TIMEOUT, **kwargs):
|
|||
'head': session.head,
|
||||
'post': session.post,
|
||||
}[method]
|
||||
|
||||
req = method(url, stream=stream, headers=headers, timeout=timeout, **kwargs)
|
||||
req = method(url, stream=stream, headers=headers, timeout=None, **kwargs)
|
||||
req.raise_for_status()
|
||||
return req
|
||||
|
||||
|
|
@ -311,11 +343,10 @@ def download_argparse(args):
|
|||
url = args.url
|
||||
|
||||
url = clipext.resolve(url)
|
||||
|
||||
callback = {
|
||||
None: progress1,
|
||||
'1': progress1,
|
||||
'2': progress2,
|
||||
None: Progress1,
|
||||
'1': Progress1,
|
||||
'2': Progress2,
|
||||
}.get(args.callback, args.callback)
|
||||
|
||||
bytespersecond = args.bytespersecond
|
||||
|
|
|
|||
114
Etiquette/bad_search.txt
Normal file
114
Etiquette/bad_search.txt
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
start_time = time.time()
|
||||
|
||||
|
||||
# Raise for cases where the minimum > maximum
|
||||
for (maxkey, maxval) in maximums.items():
|
||||
if maxkey not in minimums:
|
||||
continue
|
||||
minval = minimums[maxkey]
|
||||
if minval > maxval:
|
||||
raise ValueError('Impossible min-max for %s' % maxkey)
|
||||
|
||||
conditions = []
|
||||
minmaxers = {'<=': maximums, '>=': minimums}
|
||||
|
||||
# Convert the min-max parameters into query strings
|
||||
print('Writing minmaxers')
|
||||
for (comparator, minmaxer) in minmaxers.items():
|
||||
for (field, value) in minmaxer.items():
|
||||
if field not in Photo.int_properties:
|
||||
raise ValueError('Unknown Photo property: %s' % field)
|
||||
|
||||
value = str(value)
|
||||
query = min_max_query_builder(field, comparator, value)
|
||||
conditions.append(query)
|
||||
|
||||
print(conditions)
|
||||
|
||||
print('Writing extension rule')
|
||||
if extension is not None:
|
||||
if isinstance(extension, str):
|
||||
extension = [extension]
|
||||
|
||||
# Normalize to prevent injections
|
||||
extension = [normalize_tagname(e) for e in extension]
|
||||
extension = ['extension == "%s"' % e for e in extension]
|
||||
extension = ' OR '.join(extension)
|
||||
extension = '(%s)' % extension
|
||||
conditions.append(extension)
|
||||
|
||||
def setify(l):
|
||||
if l is None:
|
||||
return set()
|
||||
else:
|
||||
return set(self.get_tag_by_name(t) for t in l)
|
||||
|
||||
tag_musts = setify(tag_musts)
|
||||
tag_mays = setify(tag_mays)
|
||||
tag_forbids = setify(tag_forbids)
|
||||
|
||||
base = '''
|
||||
{negator} EXISTS(
|
||||
SELECT 1 FROM photo_tag_rel
|
||||
WHERE photo_tag_rel.photoid == photos.id
|
||||
AND photo_tag_rel.tagid {operator} {value}
|
||||
)'''
|
||||
|
||||
print('Writing musts')
|
||||
for tag in tag_musts:
|
||||
# tagid == must
|
||||
query = base.format(
|
||||
negator='',
|
||||
operator='==',
|
||||
value='"%s"' % tag.id,
|
||||
)
|
||||
conditions.append(query)
|
||||
|
||||
print('Writing mays')
|
||||
if len(tag_mays) > 0:
|
||||
# not any(tagid not in mays)
|
||||
acceptable = tag_mays.union(tag_musts)
|
||||
acceptable = ['"%s"' % t.id for t in acceptable]
|
||||
acceptable = ', '.join(acceptable)
|
||||
query = base.format(
|
||||
negator='',
|
||||
operator='IN',
|
||||
value='(%s)' % acceptable,
|
||||
)
|
||||
conditions.append(query)
|
||||
|
||||
print('Writing forbids')
|
||||
if len(tag_forbids) > 0:
|
||||
# not any(tagid in forbids)
|
||||
forbids = ['"%s"' % t.id for t in tag_forbids]
|
||||
forbids = ', '.join(forbids)
|
||||
query = base.format(
|
||||
negator='NOT',
|
||||
operator='IN',
|
||||
value='(%s)' % forbids
|
||||
)
|
||||
conditions.append(query)
|
||||
|
||||
if len(conditions) == 0:
|
||||
raise ValueError('No search query provided')
|
||||
|
||||
conditions = [query for query in conditions if query is not None]
|
||||
conditions = ['(%s)' % c for c in conditions]
|
||||
conditions = ' AND '.join(conditions)
|
||||
conditions = 'WHERE %s' % conditions
|
||||
|
||||
query = 'SELECT * FROM photos %s' % conditions
|
||||
query = query.replace('\n', ' ')
|
||||
while ' ' in query:
|
||||
query = query.replace(' ', ' ')
|
||||
print(query)
|
||||
|
||||
temp_cur = self.sql.cursor()
|
||||
temp_cur.execute(query)
|
||||
|
||||
for fetch in fetch_generator(temp_cur):
|
||||
photo = Photo(self, fetch)
|
||||
yield photo
|
||||
|
||||
end_time = time.time()
|
||||
print(end_time - start_time)
|
||||
1
Etiquette/etiquette.py
Normal file
1
Etiquette/etiquette.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
import phototagger
|
||||
1163
Etiquette/phototagger.py
Normal file
1163
Etiquette/phototagger.py
Normal file
File diff suppressed because it is too large
Load diff
|
Before Width: | Height: | Size: 467 KiB After Width: | Height: | Size: 467 KiB |
|
Before Width: | Height: | Size: 900 KiB After Width: | Height: | Size: 900 KiB |
|
Before Width: | Height: | Size: 321 KiB After Width: | Height: | Size: 321 KiB |
|
|
@ -12,7 +12,6 @@ class PhotoDBTest(unittest.TestCase):
|
|||
pass
|
||||
|
||||
def test_add_and_remove_photo(self):
|
||||
self.setUp()
|
||||
photo1 = self.p.new_photo('samples\\train.jpg')
|
||||
self.assertEqual(len(photo1.id), self.p.id_length)
|
||||
|
||||
|
|
@ -42,8 +41,6 @@ class PhotoDBTest(unittest.TestCase):
|
|||
self.assertEqual(tag.name, 'one_two')
|
||||
|
||||
def test_add_and_remove_synonym(self):
|
||||
self.setUp()
|
||||
|
||||
# Add synonym
|
||||
giraffe = self.p.new_tag('giraffe')
|
||||
horse = self.p.new_tag_synonym('long horse', 'giraffe')
|
||||
|
|
@ -65,7 +62,6 @@ class PhotoDBTest(unittest.TestCase):
|
|||
self.assertRaises(phototagger.NoSuchSynonym, self.p.remove_tag_synonym, 'blanc')
|
||||
|
||||
def test_apply_photo_tag(self):
|
||||
self.setUp()
|
||||
photo = self.p.new_photo('samples\\train.jpg')
|
||||
self.p.new_tag('vehicles')
|
||||
|
||||
|
|
@ -77,8 +73,6 @@ class PhotoDBTest(unittest.TestCase):
|
|||
self.assertFalse(status)
|
||||
|
||||
def test_convert_tag_synonym(self):
|
||||
self.setUp()
|
||||
|
||||
# Install tags and a synonym
|
||||
photo = self.p.new_photo('samples\\train.jpg')
|
||||
trains = self.p.new_tag('trains')
|
||||
|
|
@ -115,7 +109,6 @@ class PhotoDBTest(unittest.TestCase):
|
|||
self.assertEqual(tags[0].id, trains_id)
|
||||
|
||||
def test_generate_id(self):
|
||||
self.setUp()
|
||||
i_photo = self.p.generate_id('photos')
|
||||
i_tag = self.p.generate_id('tags')
|
||||
self.assertRaises(ValueError, self.p.generate_id, 'other')
|
||||
|
|
@ -127,21 +120,18 @@ class PhotoDBTest(unittest.TestCase):
|
|||
self.assertLess(int(i_photo), int(self.p.generate_id('photos')))
|
||||
|
||||
def test_get_photo_by_id(self):
|
||||
self.setUp()
|
||||
photo = self.p.new_photo('samples\\train.jpg')
|
||||
|
||||
photo2 = self.p.get_photo_by_id(photo.id)
|
||||
self.assertEqual(photo, photo2)
|
||||
|
||||
def test_get_photo_by_path(self):
|
||||
self.setUp()
|
||||
photo = self.p.new_photo('samples\\train.jpg')
|
||||
|
||||
photo2 = self.p.get_photo_by_path(photo.filepath)
|
||||
self.assertEqual(photo, photo2)
|
||||
|
||||
def test_get_photos_by_recent(self):
|
||||
self.setUp()
|
||||
paths = ['train.jpg', 'bolts.jpg', 'reddit.png']
|
||||
paths = ['samples\\' + path for path in paths]
|
||||
paths = [os.path.abspath(path) for path in paths]
|
||||
|
|
@ -176,7 +166,6 @@ class PhotoDBTest(unittest.TestCase):
|
|||
self.assertEqual(tag1, tag2)
|
||||
|
||||
def test_get_tags_by_photo(self):
|
||||
self.setUp()
|
||||
photo = self.p.new_photo('samples\\train.jpg')
|
||||
tag = self.p.new_tag('vehicles')
|
||||
stat = self.p.apply_photo_tag(photo.id, tagname='vehicles')
|
||||
|
|
@ -192,7 +181,6 @@ class PhotoDBTest(unittest.TestCase):
|
|||
self.assertRaises(phototagger.TagTooShort, self.p.new_tag, '!!??&&*')
|
||||
|
||||
def test_photo_has_tag(self):
|
||||
self.setUp()
|
||||
photo = self.p.new_photo('samples\\train.jpg')
|
||||
tag = self.p.new_tag('vehicles')
|
||||
self.p.apply_photo_tag(photo.id, tag.id)
|
||||
|
|
@ -363,6 +363,7 @@ function dump_urls()
|
|||
{
|
||||
textbox = document.createElement("textarea");
|
||||
textbox.className = "urldumpbox";
|
||||
textbox.id = "url_dump_box";
|
||||
workspace = document.getElementById("WORKSPACE");
|
||||
workspace.appendChild(textbox);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,6 +18,9 @@ class Path:
|
|||
def __hash__(self):
|
||||
return hash(self.absolute_path)
|
||||
|
||||
def __repr__(self):
|
||||
return '{c}({path})'.format(c=self.__class__, path=self.absolute_path)
|
||||
|
||||
@property
|
||||
def basename(self):
|
||||
return os.path.basename(self.absolute_path)
|
||||
|
|
@ -79,11 +82,12 @@ def get_path_casing(path):
|
|||
path = path.absolute_path
|
||||
|
||||
(drive, subpath) = os.path.splitdrive(path)
|
||||
drive = drive.upper()
|
||||
subpath = subpath.lstrip(os.sep)
|
||||
|
||||
pattern = [glob_patternize(piece) for piece in subpath.split(os.sep)]
|
||||
pattern = os.sep.join(pattern)
|
||||
pattern = drive.upper() + os.sep + pattern
|
||||
pattern = drive + os.sep + pattern
|
||||
#print(pattern)
|
||||
try:
|
||||
return glob.glob(pattern)[0]
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -1,894 +0,0 @@
|
|||
|
||||
|
||||
|
||||
|
||||
|
||||
import datetime
|
||||
import os
|
||||
import PIL.Image
|
||||
import random
|
||||
import sqlite3
|
||||
import string
|
||||
import warnings
|
||||
|
||||
ID_LENGTH = 22
|
||||
VALID_TAG_CHARS = string.ascii_lowercase + string.digits + '_-'
|
||||
MAX_TAG_NAME_LENGTH = 32
|
||||
|
||||
SQL_LASTID_COLUMNS = [
|
||||
'table',
|
||||
'last_id',
|
||||
]
|
||||
|
||||
SQL_PHOTO_COLUMNS = [
|
||||
'id',
|
||||
'filepath',
|
||||
'extension',
|
||||
'width',
|
||||
'height',
|
||||
'ratio',
|
||||
'area',
|
||||
'bytes',
|
||||
'created',
|
||||
]
|
||||
|
||||
SQL_PHOTOTAG_COLUMNS = [
|
||||
'photoid',
|
||||
'tagid',
|
||||
]
|
||||
|
||||
SQL_SYN_COLUMNS = [
|
||||
'name',
|
||||
'master',
|
||||
]
|
||||
|
||||
SQL_TAG_COLUMNS = [
|
||||
'id',
|
||||
'name',
|
||||
]
|
||||
|
||||
SQL_LASTID = {key:index for (index, key) in enumerate(SQL_LASTID_COLUMNS)}
|
||||
SQL_PHOTO = {key:index for (index, key) in enumerate(SQL_PHOTO_COLUMNS)}
|
||||
SQL_PHOTOTAG = {key:index for (index, key) in enumerate(SQL_PHOTOTAG_COLUMNS)}
|
||||
SQL_SYN = {key:index for (index, key) in enumerate(SQL_SYN_COLUMNS)}
|
||||
SQL_TAG = {key:index for (index, key) in enumerate(SQL_TAG_COLUMNS)}
|
||||
|
||||
|
||||
DB_INIT = '''
|
||||
CREATE TABLE IF NOT EXISTS photos(
|
||||
id TEXT,
|
||||
filepath TEXT,
|
||||
extension TEXT,
|
||||
width INT,
|
||||
height INT,
|
||||
ratio REAL,
|
||||
area INT,
|
||||
bytes INT,
|
||||
created INT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS tags(
|
||||
id TEXT,
|
||||
name TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS albums(
|
||||
albumid TEXT,
|
||||
photoid TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS photo_tag_rel(
|
||||
photoid TEXT,
|
||||
tagid TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS tag_synonyms(
|
||||
name TEXT,
|
||||
mastername TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS id_numbers(
|
||||
tab TEXT,
|
||||
last_id TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS index_photo_id on photos(id);
|
||||
CREATE INDEX IF NOT EXISTS index_photo_path on photos(filepath);
|
||||
CREATE INDEX IF NOT EXISTS index_photo_created on photos(created);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS index_tag_id on tags(id);
|
||||
CREATE INDEX IF NOT EXISTS index_tag_name on tags(name);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS index_tagrel_photoid on photo_tag_rel(photoid);
|
||||
CREATE INDEX IF NOT EXISTS index_tagrel_tagid on photo_tag_rel(tagid);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS index_tagsyn_name on tag_synonyms(name);
|
||||
'''
|
||||
|
||||
def basex(number, base, alphabet='0123456789abcdefghijklmnopqrstuvwxyz'):
|
||||
'''
|
||||
Converts an integer to a different base string.
|
||||
Based on http://stackoverflow.com/a/1181922/5430534
|
||||
'''
|
||||
if base > len(alphabet):
|
||||
raise Exception('alphabet %s does not support base %d' % (
|
||||
repr(alphabet), base))
|
||||
alphabet = alphabet[:base]
|
||||
if not isinstance(number, (int, str)):
|
||||
raise TypeError('number must be an integer')
|
||||
number = int(number)
|
||||
based = ''
|
||||
sign = ''
|
||||
if number < 0:
|
||||
sign = '-'
|
||||
number = -number
|
||||
if 0 <= number < len(alphabet):
|
||||
return sign + alphabet[number]
|
||||
while number != 0:
|
||||
number, i = divmod(number, len(alphabet))
|
||||
based = alphabet[i] + based
|
||||
return sign + based
|
||||
|
||||
def fetch_generator(cursor):
|
||||
while True:
|
||||
fetch = cursor.fetchone()
|
||||
if fetch is None:
|
||||
break
|
||||
yield fetch
|
||||
|
||||
def getnow(timestamp=True):
|
||||
'''
|
||||
Return the current UTC timestamp or datetime object.
|
||||
'''
|
||||
now = datetime.datetime.now(datetime.timezone.utc)
|
||||
if timestamp:
|
||||
return now.timestamp()
|
||||
return now
|
||||
|
||||
def is_xor(*args):
|
||||
'''
|
||||
Return True if and only if one arg is truthy.
|
||||
'''
|
||||
return [bool(a) for a in args].count(True) == 1
|
||||
|
||||
def min_max_query_builder(name, comparator, value):
|
||||
return ' '.join([name, comparator, value])
|
||||
|
||||
def normalize_tagname(tagname):
|
||||
'''
|
||||
Tag names can only consist of lowercase letters, underscores,
|
||||
and hyphens. The given tagname is lowercased, gets its spaces
|
||||
replaced by underscores, and is stripped of any not-whitelisted
|
||||
characters.
|
||||
'''
|
||||
tagname = tagname.lower()
|
||||
tagname = tagname.replace(' ', '_')
|
||||
tagname = (c for c in tagname if c in VALID_TAG_CHARS)
|
||||
tagname = ''.join(tagname)
|
||||
if len(tagname) == 0:
|
||||
raise TagTooShort(tagname)
|
||||
if len(tagname) > MAX_TAG_NAME_LENGTH:
|
||||
raise TagTooLong(tagname)
|
||||
return tagname
|
||||
|
||||
def not_implemented(function):
|
||||
'''
|
||||
Decorator for keeping track of which functions still need to be filled out.
|
||||
'''
|
||||
warnings.warn('%s is not implemented' % function.__name__)
|
||||
return function
|
||||
|
||||
def raise_nosuchtag(tagid=None, tagname=None, comment=''):
|
||||
if tagid is not None:
|
||||
message = 'ID: %s. %s' % (tagid, comment)
|
||||
elif tagname is not None:
|
||||
message = 'Name: %s. %s' % (tagname, comment)
|
||||
raise NoSuchTag(message)
|
||||
|
||||
class NoSuchPhoto(Exception):
|
||||
pass
|
||||
|
||||
class NoSuchSynonym(Exception):
|
||||
pass
|
||||
|
||||
class NoSuchTag(Exception):
|
||||
pass
|
||||
|
||||
class PhotoExists(Exception):
|
||||
pass
|
||||
|
||||
class TagExists(Exception):
|
||||
pass
|
||||
|
||||
class TagTooLong(Exception):
|
||||
pass
|
||||
|
||||
class TagTooShort(Exception):
|
||||
pass
|
||||
|
||||
class XORException(Exception):
|
||||
pass
|
||||
|
||||
class PhotoDB:
|
||||
'''
|
||||
This class represents an SQLite3 database containing the following tables:
|
||||
|
||||
photos:
|
||||
Rows represent image files on the local disk.
|
||||
Entries contain a unique ID, the image's filepath, and metadata
|
||||
like dimensions and filesize.
|
||||
|
||||
tags:
|
||||
Rows represent labels, which can be applied to an arbitrary number of
|
||||
photos. Photos may be selected by which tags they contain.
|
||||
Entries contain a unique ID and a name.
|
||||
|
||||
albums:
|
||||
Rows represent the inclusion of a photo in an album
|
||||
|
||||
photo_tag_rel:
|
||||
Rows represent a Photo's ownership of a particular Tag.
|
||||
|
||||
tag_synonyms:
|
||||
Rows represent relationships between two tag names, so that they both
|
||||
resolve to the same Tag object when selected. Entries contain the
|
||||
subordinate name and master name.
|
||||
The master name MUST also exist in the `tags` table.
|
||||
If a new synonym is created referring to another synoym, the master name
|
||||
will be resolved and used instead, so a synonym is never in the master
|
||||
column.
|
||||
Tag objects will ALWAYS represent the master tag.
|
||||
|
||||
Note that the entries in this table do not contain ID numbers.
|
||||
The rationale here is that "coco" is a synonym for "chocolate" regardless
|
||||
of the "chocolate" tag's ID, and that if a tag is renamed, its synonyms
|
||||
do not necessarily follow.
|
||||
The `rename_tag` method includes a parameter `apply_to_synonyms` if you do
|
||||
want them to follow.
|
||||
'''
|
||||
def __init__(self, databasename='phototagger.db', id_length=None):
|
||||
if id_length is None:
|
||||
self.id_length = ID_LENGTH
|
||||
self.databasename = databasename
|
||||
self.sql = sqlite3.connect(databasename)
|
||||
self.cur = self.sql.cursor()
|
||||
statements = DB_INIT.split(';')
|
||||
for statement in statements:
|
||||
self.cur.execute(statement)
|
||||
self._last_ids = {}
|
||||
|
||||
def __repr__(self):
|
||||
return 'PhotoDB(databasename={dbname})'.format(dbname=repr(self.databasename))
|
||||
|
||||
def apply_photo_tag(self, photoid, tagid=None, tagname=None, commit=True):
|
||||
'''
|
||||
Apply a tag to a photo. `tag` may be the name of the tag or a Tag
|
||||
object from the same PhotoDB.
|
||||
|
||||
`tag` may NOT be the tag's ID, since we can't tell if a given string is
|
||||
an ID or a name.
|
||||
|
||||
Returns True if the tag was applied, False if the photo already had this tag.
|
||||
Raises NoSuchTag and NoSuchPhoto as appropriate.
|
||||
'''
|
||||
tag = self.get_tag(tagid=tagid, tagname=tagname, resolve_synonyms=True)
|
||||
|
||||
self.cur.execute('SELECT * FROM photo_tag_rel WHERE photoid == ? AND tagid == ?', [photoid, tag.id])
|
||||
if self.cur.fetchone() is not None:
|
||||
return False
|
||||
|
||||
self.cur.execute('INSERT INTO photo_tag_rel VALUES(?, ?)', [photoid, tag.id])
|
||||
if commit:
|
||||
self.sql.commit()
|
||||
return True
|
||||
|
||||
def convert_tag_to_synonym(self, oldtagname, mastertagname):
|
||||
'''
|
||||
Convert an independent tag into a synonym for a different independent tag.
|
||||
All photos which possess the current tag will have it replaced
|
||||
with the master tag. All synonyms of the old tag will point to the new tag.
|
||||
|
||||
Good for when two tags need to be merged under a single name.
|
||||
'''
|
||||
oldtagname = normalize_tagname(oldtagname)
|
||||
mastertagname = normalize_tagname(mastertagname)
|
||||
|
||||
oldtag = self.get_tag_by_name(oldtagname, resolve_synonyms=False)
|
||||
if oldtag is None:
|
||||
raise NoSuchTag(oldtagname)
|
||||
|
||||
mastertag = self.get_tag_by_name(mastertagname, resolve_synonyms=False)
|
||||
if mastertag is None:
|
||||
raise NoSuchTag(mastertagname)
|
||||
|
||||
# Migrate the old tag's synonyms to the new one
|
||||
# UPDATE is safe for this operation because there is no chance of duplicates.
|
||||
self.cur.execute('UPDATE tag_synonyms SET mastername = ? WHERE mastername == ?', [mastertagname, oldtagname])
|
||||
|
||||
# Iterate over all photos with the old tag, and relate them to the new tag
|
||||
# if they aren't already.
|
||||
temp_cur = self.sql.cursor()
|
||||
temp_cur.execute('SELECT * FROM photo_tag_rel WHERE tagid == ?', [oldtag.id])
|
||||
for relationship in fetch_generator(temp_cur):
|
||||
photoid = relationship[SQL_PHOTOTAG['photoid']]
|
||||
self.cur.execute('SELECT * FROM photo_tag_rel WHERE tagid == ?', [mastertag.id])
|
||||
if self.cur.fetchone() is not None:
|
||||
continue
|
||||
self.cur.execute('INSERT INTO photo_tag_rel VALUES(?, ?)', [photoid, mastertag.id])
|
||||
|
||||
# Then delete the relationships with the old tag
|
||||
self.cur.execute('DELETE FROM photo_tag_rel WHERE tagid == ?', [oldtag.id])
|
||||
self.cur.execute('DELETE FROM tags WHERE id == ?', [oldtag.id])
|
||||
|
||||
# Enjoy your new life as a monk.
|
||||
self.new_tag_synonym(oldtag.name, mastertag.name, commit=False)
|
||||
self.sql.commit()
|
||||
|
||||
def delete_photo(self, photoid):
|
||||
'''
|
||||
Delete a photo and its relation to any tags and albums.
|
||||
'''
|
||||
photo = self.get_photo_by_id(photoid)
|
||||
if photo is None:
|
||||
raise NoSuchPhoto(photoid)
|
||||
self.cur.execute('DELETE FROM photos WHERE id == ?', [photoid])
|
||||
self.cur.execute('DELETE FROM photo_tag_rel WHERE photoid == ?', [photoid])
|
||||
self.sql.commit()
|
||||
|
||||
def delete_tag(self, tagid=None, tagname=None):
|
||||
'''
|
||||
Delete a tag, its synonyms, and its relation to any photos.
|
||||
'''
|
||||
|
||||
tag = self.get_tag(tagid=tagid, tagname=tagname, resolve_synonyms=False)
|
||||
|
||||
if tag is None:
|
||||
message = 'Is it a synonym?'
|
||||
raise_nosuchtag(tagid=tagid, tagname=tagname, comment=message)
|
||||
|
||||
self.cur.execute('DELETE FROM tags WHERE id == ?', [tag.id])
|
||||
self.cur.execute('DELETE FROM photo_tag_rel WHERE tagid == ?', [tag.id])
|
||||
self.cur.execute('DELETE FROM tag_synonyms WHERE mastername == ?', [tag.name])
|
||||
self.sql.commit()
|
||||
|
||||
def delete_tag_synonym(self, tagname):
|
||||
'''
|
||||
Delete a tag synonym.
|
||||
This will have no effect on photos or other synonyms because
|
||||
they always resolve to the master tag before application.
|
||||
'''
|
||||
tagname = normalize_tagname(tagname)
|
||||
self.cur.execute('SELECT * FROM tag_synonyms WHERE name == ?', [tagname])
|
||||
fetch = self.cur.fetchone()
|
||||
if fetch is None:
|
||||
raise NoSuchSynonym(tagname)
|
||||
|
||||
self.cur.execute('DELETE FROM tag_synonyms WHERE name == ?', [tagname])
|
||||
self.sql.commit()
|
||||
|
||||
def generate_id(self, table):
|
||||
'''
|
||||
Create a new ID number that is unique to the given table.
|
||||
Note that this method does not commit the database. We'll wait for that
|
||||
to happen in whoever is calling us, so we know the ID is actually used.
|
||||
'''
|
||||
table = table.lower()
|
||||
if table not in ['photos', 'tags']:
|
||||
raise ValueError('Invalid table requested: %s.', table)
|
||||
|
||||
do_update = False
|
||||
if table in self._last_ids:
|
||||
# Use cache value
|
||||
new_id = self._last_ids[table] + 1
|
||||
do_update = True
|
||||
else:
|
||||
self.cur.execute('SELECT * FROM id_numbers WHERE tab == ?', [table])
|
||||
fetch = self.cur.fetchone()
|
||||
if fetch is None:
|
||||
# Register new value
|
||||
new_id = 1
|
||||
else:
|
||||
# Use database value
|
||||
new_id = int(fetch[SQL_LASTID['last_id']]) + 1
|
||||
do_update = True
|
||||
|
||||
new_id_s = str(new_id).rjust(self.id_length, '0')
|
||||
if do_update:
|
||||
self.cur.execute('UPDATE id_numbers SET last_id = ? WHERE tab == ?', [new_id_s, table])
|
||||
else:
|
||||
self.cur.execute('INSERT INTO id_numbers VALUES(?, ?)', [table, new_id_s])
|
||||
self._last_ids[table] = new_id
|
||||
return new_id_s
|
||||
|
||||
@not_implemented
|
||||
def get_album_by_id(self, albumid):
|
||||
return
|
||||
|
||||
def get_photo_by_id(self, photoid):
|
||||
'''
|
||||
Return this Photo object, or None if it does not exist.
|
||||
'''
|
||||
self.cur.execute('SELECT * FROM photos WHERE id == ?', [photoid])
|
||||
photo = self.cur.fetchone()
|
||||
if photo is None:
|
||||
return None
|
||||
photo = self.tuple_to_photo(photo)
|
||||
return photo
|
||||
|
||||
def get_photo_by_path(self, filepath):
|
||||
'''
|
||||
Return this Photo object, or None if it does not exist.
|
||||
'''
|
||||
filepath = os.path.abspath(filepath)
|
||||
self.cur.execute('SELECT * FROM photos WHERE filepath == ?', [filepath])
|
||||
photo = self.cur.fetchone()
|
||||
if photo is None:
|
||||
return None
|
||||
photo = self.tuple_to_photo(photo)
|
||||
return photo
|
||||
|
||||
def get_photos_by_recent(self, count=None):
|
||||
'''
|
||||
Yield photo objects in order of creation time.
|
||||
'''
|
||||
if count is not None and count <= 0:
|
||||
return
|
||||
# We're going to use a second cursor because the first one may
|
||||
# get used for something else, deactivating this query.
|
||||
temp_cur = self.sql.cursor()
|
||||
temp_cur.execute('SELECT * FROM photos ORDER BY created DESC')
|
||||
while True:
|
||||
f = temp_cur.fetchone()
|
||||
if f is None:
|
||||
return
|
||||
photo = self.tuple_to_photo(f)
|
||||
|
||||
yield photo
|
||||
|
||||
if count is None:
|
||||
continue
|
||||
count -= 1
|
||||
if count <= 0:
|
||||
return
|
||||
|
||||
@not_implemented
|
||||
def get_photos_by_search(
|
||||
self,
|
||||
extension=None,
|
||||
maximums={},
|
||||
minimums={},
|
||||
tag_musts=None,
|
||||
tag_mays=None,
|
||||
tag_forbids=None,
|
||||
tag_forbid_unspecified=False,
|
||||
):
|
||||
'''
|
||||
Given one or multiple tags, yield photos possessing those tags.
|
||||
|
||||
Parameters:
|
||||
extension :
|
||||
A string or list of strings of acceptable file extensions.
|
||||
|
||||
maximums :
|
||||
A dictionary, where the key is an attribute of the photo,
|
||||
(area, bytes, created, height, id, or width)
|
||||
and the value is the maximum desired value for that field.
|
||||
|
||||
minimums :
|
||||
A dictionary like `maximums` where the value is the minimum
|
||||
desired value for that field.
|
||||
|
||||
tag_musts :
|
||||
A list of tag names or Tag objects.
|
||||
Photos MUST have ALL tags in this list.
|
||||
|
||||
tag_mays :
|
||||
A list of tag names or Tag objects.
|
||||
If `forbid_unspecified` is True, then Photos MUST have AT LEAST ONE tag in this list.
|
||||
If `forbid_unspecified` is False, then Photos MAY or MAY NOT have ANY tag in this list.
|
||||
|
||||
tag_forbids :
|
||||
A list of tag names or Tag objects.
|
||||
Photos MUST NOT have ANY tag in the list.
|
||||
|
||||
tag_forbid_unspecified :
|
||||
True or False.
|
||||
If False, Photos need only comply with the `tag_musts`.
|
||||
If True, Photos need to comply with both `tag_musts` and `tag_mays`.
|
||||
'''
|
||||
maximums = {key:int(val) for (key, val) in maximums.items()}
|
||||
minimums = {key:int(val) for (key, val) in minimums.items()}
|
||||
|
||||
# Raise for cases where the minimum > maximum
|
||||
for (maxkey, maxval) in maximums.items():
|
||||
if maxkey not in minimums:
|
||||
continue
|
||||
minval = minimums[maxkey]
|
||||
if minval > maxval:
|
||||
raise ValueError('Impossible min-max for %s' % maxkey)
|
||||
|
||||
conditions = []
|
||||
minmaxers = {'<=': maximums, '>=': minimums}
|
||||
|
||||
# Convert the min-max parameters into query strings
|
||||
for (comparator, minmaxer) in minmaxers.items():
|
||||
for (field, value) in minmaxer.items():
|
||||
if field not in Photo.int_properties:
|
||||
raise ValueError('Unknown Photo property: %s' % field)
|
||||
|
||||
value = str(value)
|
||||
query = min_max_query_builder(field, comparator, value)
|
||||
conditions.append(query)
|
||||
|
||||
if extension is not None:
|
||||
if isinstance(extension, str):
|
||||
extension = [extension]
|
||||
|
||||
# Normalize to prevent injections
|
||||
extension = [normalize_tagname(e) for e in extension]
|
||||
extension = ['extension == "%s"' % e for e in extension]
|
||||
extension = ' OR '.join(extension)
|
||||
extension = '(%s)' % extension
|
||||
conditions.append(extension)
|
||||
|
||||
def setify(l):
|
||||
return set(self.get_tag_by_name(t) for t in l) if l else set()
|
||||
tag_musts = setify(tag_musts)
|
||||
tag_mays = setify(tag_mays)
|
||||
tag_forbids = setify(tag_forbids)
|
||||
|
||||
base = '%s EXISTS (SELECT 1 FROM photo_tag_rel WHERE photo_tag_rel.photoid == photos.id AND photo_tag_rel.tagid %s %s)'
|
||||
for tag in tag_musts:
|
||||
query = base % ('', '==', '"%s"' % tag.id)
|
||||
conditions.append(query)
|
||||
|
||||
if tag_forbid_unspecified and len(tag_mays) > 0:
|
||||
acceptable = tag_mays.union(tag_musts)
|
||||
acceptable = ['"%s"' % t.id for t in acceptable]
|
||||
acceptable = ', '.join(acceptable)
|
||||
query = base % ('NOT', 'NOT IN', '(%s)' % acceptable)
|
||||
conditions.append(query)
|
||||
|
||||
for tag in tag_forbids:
|
||||
query = base % ('NOT', '==', '"%s"' % tag.id)
|
||||
conditions.append(query)
|
||||
|
||||
if len(conditions) == 0:
|
||||
raise ValueError('No search query provided')
|
||||
|
||||
conditions = [query for query in conditions if query is not None]
|
||||
conditions = ['(%s)' % c for c in conditions]
|
||||
conditions = ' AND '.join(conditions)
|
||||
conditions = 'WHERE %s' % conditions
|
||||
|
||||
|
||||
query = 'SELECT * FROM photos %s' % conditions
|
||||
print(query)
|
||||
temp_cur = self.sql.cursor()
|
||||
temp_cur.execute(query)
|
||||
acceptable_tags = tag_musts.union(tag_mays)
|
||||
while True:
|
||||
fetch = temp_cur.fetchone()
|
||||
if fetch is None:
|
||||
break
|
||||
|
||||
photo = self.tuple_to_photo(fetch)
|
||||
|
||||
# if any(forbid in photo.tags for forbid in tag_forbids):
|
||||
# print('Forbidden')
|
||||
# continue
|
||||
|
||||
# if tag_forbid_unspecified:
|
||||
# if any(tag not in acceptable_tags for tag in photo.tags):
|
||||
# print('Forbid unspecified')
|
||||
# continue
|
||||
|
||||
# if any(must not in photo.tags for must in tag_musts):
|
||||
# print('No must')
|
||||
# continue
|
||||
|
||||
yield photo
|
||||
|
||||
|
||||
def get_tag(self, tagid=None, tagname=None, resolve_synonyms=True):
|
||||
'''
|
||||
Redirect to get_tag_by_id or get_tag_by_name after xor-checking the parameters.
|
||||
'''
|
||||
if not is_xor(tagid, tagname):
|
||||
raise XORException('One and only one of `tagid`, `tagname` can be passed.')
|
||||
|
||||
if tagid is not None:
|
||||
return self.get_tag_by_id(tagid)
|
||||
elif tagname is not None:
|
||||
return self.get_tag_by_name(tagname, resolve_synonyms=resolve_synonyms)
|
||||
raise_nosuchtag(tagid=tagid, tagname=tagname)
|
||||
|
||||
def get_tag_by_id(self, tagid):
|
||||
self.cur.execute('SELECT * FROM tags WHERE id == ?', [tagid])
|
||||
tag = self.cur.fetchone()
|
||||
if tag is None:
|
||||
return raise_nosuchtag(tagid=tagid)
|
||||
tag = self.tuple_to_tag(tag)
|
||||
return tag
|
||||
|
||||
def get_tag_by_name(self, tagname, resolve_synonyms=True):
|
||||
if isinstance(tagname, Tag):
|
||||
return tagname
|
||||
|
||||
tagname = normalize_tagname(tagname)
|
||||
|
||||
if resolve_synonyms is True:
|
||||
self.cur.execute('SELECT * FROM tag_synonyms WHERE name == ?', [tagname])
|
||||
fetch = self.cur.fetchone()
|
||||
if fetch is not None:
|
||||
mastertagname = fetch[SQL_SYN['master']]
|
||||
tag = self.get_tag_by_name(mastertagname)
|
||||
return tag
|
||||
|
||||
self.cur.execute('SELECT * FROM tags WHERE name == ?', [tagname])
|
||||
fetch = self.cur.fetchone()
|
||||
if fetch is None:
|
||||
raise_nosuchtag(tagname=tagname)
|
||||
|
||||
tag = self.tuple_to_tag(fetch)
|
||||
return tag
|
||||
|
||||
def get_tags_by_photo(self, photoid):
|
||||
'''
|
||||
Return the tags assigned to the given photo.
|
||||
'''
|
||||
temp_cur = self.sql.cursor()
|
||||
temp_cur.execute('SELECT * FROM photo_tag_rel WHERE photoid == ?', [photoid])
|
||||
tags = fetch_generator(temp_cur)
|
||||
tagobjects = set()
|
||||
for tag in tags:
|
||||
tagid = tag[SQL_PHOTOTAG['tagid']]
|
||||
tagobj = self.get_tag_by_id(tagid)
|
||||
tagobjects.add(tagobj)
|
||||
return tagobjects
|
||||
|
||||
def new_photo(self, filename, tags=None, allow_duplicates=False):
|
||||
'''
|
||||
Given a filepath, determine its attributes and create a new Photo object in the
|
||||
database. Tags may be applied now or later.
|
||||
|
||||
If `allow_duplicates` is False, we will first check the database for any files
|
||||
with the same path and raise PhotoExists if found.
|
||||
|
||||
Returns the Photo object.
|
||||
'''
|
||||
filename = os.path.abspath(filename)
|
||||
if not allow_duplicates:
|
||||
existing = self.get_photo_by_path(filename)
|
||||
if existing is not None:
|
||||
raise PhotoExists(filename, existing)
|
||||
|
||||
# I want the caller to receive any exceptions this raises.
|
||||
image = PIL.Image.open(filename)
|
||||
|
||||
extension = os.path.splitext(filename)[1]
|
||||
extension = extension.replace('.', '')
|
||||
extension = normalize_tagname(extension)
|
||||
(width, height) = image.size
|
||||
area = width * height
|
||||
ratio = width / height
|
||||
bytes = os.path.getsize(filename)
|
||||
created = int(getnow())
|
||||
photoid = self.generate_id('photos')
|
||||
|
||||
data = [None] * len(SQL_PHOTO_COLUMNS)
|
||||
data[SQL_PHOTO['id']] = photoid
|
||||
data[SQL_PHOTO['filepath']] = filename
|
||||
data[SQL_PHOTO['extension']] = extension
|
||||
data[SQL_PHOTO['width']] = width
|
||||
data[SQL_PHOTO['height']] = height
|
||||
data[SQL_PHOTO['area']] = area
|
||||
data[SQL_PHOTO['ratio']] = ratio
|
||||
data[SQL_PHOTO['bytes']] = bytes
|
||||
data[SQL_PHOTO['created']] = created
|
||||
photo = self.tuple_to_photo(data)
|
||||
|
||||
self.cur.execute('INSERT INTO photos VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)', data)
|
||||
|
||||
tags = tags or []
|
||||
for tag in tags:
|
||||
try:
|
||||
self.apply_photo_tag(photoid, tagname=tag, commit=False)
|
||||
except NoSuchTag:
|
||||
self.sql.rollback()
|
||||
raise
|
||||
self.sql.commit()
|
||||
image.close()
|
||||
return photo
|
||||
|
||||
def new_tag(self, tagname):
|
||||
'''
|
||||
Register a new tag in the database and return the Tag object.
|
||||
'''
|
||||
tagname = normalize_tagname(tagname)
|
||||
try:
|
||||
self.get_tag_by_name(tagname)
|
||||
TagExists(tagname)
|
||||
except NoSuchTag:
|
||||
pass
|
||||
tagid = self.generate_id('tags')
|
||||
self.cur.execute('INSERT INTO tags VALUES(?, ?)', [tagid, tagname])
|
||||
self.sql.commit()
|
||||
tag = self.tuple_to_tag([tagid, tagname])
|
||||
return tag
|
||||
|
||||
def new_tag_synonym(self, tagname, mastertagname, commit=True):
|
||||
'''
|
||||
Register a new synonym for an existing tag.
|
||||
'''
|
||||
tagname = normalize_tagname(tagname)
|
||||
mastertagname = normalize_tagname(mastertagname)
|
||||
|
||||
if tagname == mastertagname:
|
||||
raise ValueError('Cannot assign synonym to itself.')
|
||||
|
||||
# We leave resolve_synonyms as True, so that if this function returns
|
||||
# anything, we know the given tagname is already a synonym or master.
|
||||
tag = self.get_tag_by_name(tagname, resolve_synonyms=True)
|
||||
if tag is not None:
|
||||
raise TagExists(tagname)
|
||||
|
||||
mastertag = self.get_tag_by_name(mastertagname, resolve_synonyms=True)
|
||||
if mastertag is None:
|
||||
raise NoSuchTag(mastertagname)
|
||||
|
||||
self.cur.execute('INSERT INTO tag_synonyms VALUES(?, ?)', [tagname, mastertag.name])
|
||||
|
||||
if commit:
|
||||
self.sql.commit()
|
||||
|
||||
return mastertag
|
||||
|
||||
def photo_has_tag(self, photoid, tagid=None, tagname=None):
|
||||
tag = self.get_tag(tagid=tagid, tagname=tagname, resolve_synonyms=True)
|
||||
if tag is None:
|
||||
raise_nosuchtag(tagid=tagid, tagname=tagname)
|
||||
|
||||
exe = self.cur.execute
|
||||
exe('SELECT * FROM photo_tag_rel WHERE photoid == ? AND tagid == ?', [photoid, tag.id])
|
||||
fetch = self.cur.fetchone()
|
||||
has_tag = fetch is not None
|
||||
return has_tag
|
||||
|
||||
@not_implemented
|
||||
def rename_tag(self, tagname, newname, apply_to_synonyms):
|
||||
pass
|
||||
|
||||
def tuple_to_photo(self, tu):
|
||||
'''
|
||||
Given a tuple like the ones from an sqlite query,
|
||||
create a Photo object.
|
||||
'''
|
||||
photoid = tu[SQL_PHOTO['id']]
|
||||
tags = self.get_tags_by_photo(photoid)
|
||||
|
||||
photo = Photo(
|
||||
photodb = self,
|
||||
photoid = photoid,
|
||||
filepath = tu[SQL_PHOTO['filepath']],
|
||||
extension = tu[SQL_PHOTO['extension']],
|
||||
width = tu[SQL_PHOTO['width']],
|
||||
height = tu[SQL_PHOTO['height']],
|
||||
created = tu[SQL_PHOTO['created']],
|
||||
bytes = tu[SQL_PHOTO['bytes']],
|
||||
tags = tags,
|
||||
)
|
||||
return photo
|
||||
|
||||
def tuple_to_tag(self, tu):
|
||||
'''
|
||||
Given a tuple like the ones from an sqlite query,
|
||||
create a Tag object.
|
||||
'''
|
||||
tag = Tag(
|
||||
photodb = self,
|
||||
tagid = tu[SQL_TAG['id']],
|
||||
name = tu[SQL_TAG['name']]
|
||||
)
|
||||
return tag
|
||||
|
||||
class Photo:
|
||||
'''
|
||||
This class represents a PhotoDB entry containing information about an image file.
|
||||
Photo objects cannot exist without a corresponding PhotoDB object, because
|
||||
Photos are not the actual files, just the database entry.
|
||||
'''
|
||||
int_properties = set(['area', 'bytes', 'created', 'height', 'id', 'width'])
|
||||
def __init__(
|
||||
self,
|
||||
photodb,
|
||||
photoid,
|
||||
filepath,
|
||||
extension,
|
||||
width,
|
||||
height,
|
||||
bytes,
|
||||
created,
|
||||
tags=None,
|
||||
):
|
||||
if tags is None:
|
||||
tags = []
|
||||
|
||||
self.photodb = photodb
|
||||
self.id = photoid
|
||||
self.filepath = filepath
|
||||
self.extension = extension
|
||||
self.width = int(width)
|
||||
self.height = int(height)
|
||||
self.ratio = self.width / self.height
|
||||
self.area = self.width * self.height
|
||||
self.bytes = int(bytes)
|
||||
self.created = int(created)
|
||||
self.tags = tags
|
||||
|
||||
def __eq__(self, other):
|
||||
if not isinstance(other, Photo):
|
||||
return False
|
||||
return self.id == other.id
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.id)
|
||||
|
||||
def __repr__(self):
|
||||
r = ('Photo(photodb={photodb}, ',
|
||||
'photoid={photoid}, ',
|
||||
'filepath={filepath}, ',
|
||||
'extension={extension}, ',
|
||||
'width={width}, ',
|
||||
'height={height}, ',
|
||||
'created={created})'
|
||||
)
|
||||
r = ''.join(r)
|
||||
r = r.format(
|
||||
photodb = repr(self.photodb),
|
||||
photoid = repr(self.id),
|
||||
filepath = repr(self.filepath),
|
||||
extension = repr(self.extension),
|
||||
width = repr(self.width),
|
||||
height = repr(self.height),
|
||||
bytes = repr(self.bytes),
|
||||
area = repr(self.area),
|
||||
created = repr(self.created),
|
||||
)
|
||||
return r
|
||||
|
||||
def __str__(self):
|
||||
return 'Photo: %s' % self.id
|
||||
|
||||
def apply_photo_tag(self, tagname):
|
||||
return self.photodb.apply_photo_tag(self.id, tagname=tagname, commit=True)
|
||||
|
||||
def photo_has_tag(self, tagname):
|
||||
return self.photodb.photo_has_tag(self.id, tagname=tagname)
|
||||
|
||||
class Tag:
|
||||
'''
|
||||
This class represents a Tag, which can be applied to Photos for
|
||||
organization.
|
||||
'''
|
||||
def __init__(self, photodb, tagid, name):
|
||||
self.photodb = photodb
|
||||
self.id = tagid
|
||||
self.name = name
|
||||
|
||||
def __eq__(self, other):
|
||||
if not isinstance(other, Tag):
|
||||
return False
|
||||
return self.id == other.id
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.id)
|
||||
|
||||
def __repr__(self):
|
||||
r = 'Tag(photodb={photodb}, name={name}, tagid={tagid})'
|
||||
r = r.format(
|
||||
photodb = repr(self.photodb),
|
||||
name = repr(self.name),
|
||||
tagid = repr(self.id),
|
||||
)
|
||||
return r
|
||||
|
||||
def __str__(self):
|
||||
return 'Tag: %s : %s' % (self.id, self.name)
|
||||
|
||||
if __name__ == '__main__':
|
||||
p = PhotoDB()
|
||||
|
|
@ -19,17 +19,20 @@ g.total = 0
|
|||
g.start = None
|
||||
g.last = time.time()
|
||||
|
||||
def callback_progress(bytes_downloaded, bytes_total):
|
||||
class P:
|
||||
def __init__(self, bytes_total):
|
||||
self.bytes_total = bytes_total
|
||||
def step(self, bytes_downloaded):
|
||||
if g.start is None:
|
||||
g.start = time.time()
|
||||
percent = 100 * bytes_downloaded / bytes_total
|
||||
percent = 100 * bytes_downloaded / self.bytes_total
|
||||
percent = '%07.3f%%:' % percent
|
||||
chunk = bytes_downloaded - g.total
|
||||
g.total = bytes_downloaded
|
||||
METER.digest(chunk)
|
||||
METER_2.digest(chunk)
|
||||
now = round(time.time(), 1)
|
||||
if now > g.last or (bytes_downloaded >= bytes_total):
|
||||
if now > g.last or (bytes_downloaded >= self.bytes_total):
|
||||
g.last = now
|
||||
percent = percent.rjust(9, ' ')
|
||||
rate = bytestring.bytestring(METER.report()[2]).rjust(15, ' ')
|
||||
|
|
@ -40,4 +43,4 @@ def callback_progress(bytes_downloaded, bytes_total):
|
|||
|
||||
print(URL)
|
||||
print('Progress'.rjust(9, ' '), 'bps over 5s'.rjust(15, ' '), 'bps overall'.rjust(15, ' '), 'elapsed'.rjust(10, ' '))
|
||||
downloady.download_file(URL, 'nul', callback_progress=callback_progress)
|
||||
downloady.download_file(URL, 'nul', callback_progress=P)
|
||||
|
|
@ -2,13 +2,13 @@ import time
|
|||
|
||||
|
||||
class Ratelimiter:
|
||||
def __init__(self, allowance_per_period, period=1, operation_cost=1, mode='sleep'):
|
||||
def __init__(self, allowance, period=1, operation_cost=1, mode='sleep'):
|
||||
'''
|
||||
allowance_per_period:
|
||||
allowance:
|
||||
Our spending balance per `period` seconds.
|
||||
|
||||
period:
|
||||
The number of seconds over which we can perform `allowance_per_period` operations.
|
||||
The number of seconds over which we can perform `allowance` operations.
|
||||
|
||||
operation_cost:
|
||||
The default amount to remove from our balance after each operation.
|
||||
|
|
@ -26,7 +26,7 @@ class Ratelimiter:
|
|||
if mode not in ('sleep', 'reject'):
|
||||
raise ValueError('Invalid mode %s' % repr(mode))
|
||||
|
||||
self.allowance_per_period = allowance_per_period
|
||||
self.allowance = allowance
|
||||
self.period = period
|
||||
self.operation_cost = operation_cost
|
||||
self.mode = mode
|
||||
|
|
@ -36,7 +36,7 @@ class Ratelimiter:
|
|||
|
||||
@property
|
||||
def gain_rate(self):
|
||||
return self.allowance_per_period / self.period
|
||||
return self.allowance / self.period
|
||||
|
||||
def limit(self, cost=None):
|
||||
'''
|
||||
|
|
@ -47,7 +47,7 @@ class Ratelimiter:
|
|||
|
||||
time_diff = time.time() - self.last_operation
|
||||
self.balance += time_diff * self.gain_rate
|
||||
self.balance = min(self.balance, self.allowance_per_period)
|
||||
self.balance = min(self.balance, self.allowance)
|
||||
|
||||
if self.balance >= cost:
|
||||
self.balance -= cost
|
||||
|
|
|
|||
65
RenameMap/rename_map.py
Normal file
65
RenameMap/rename_map.py
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
import os
|
||||
import tkinter
|
||||
|
||||
|
||||
class Application:
|
||||
def __init__(self):
|
||||
self.windowtitle = 'Rename Map'
|
||||
|
||||
self.t = tkinter.Tk()
|
||||
self.t.title(self.windowtitle)
|
||||
w = 800
|
||||
h = 525
|
||||
screenwidth = self.t.winfo_screenwidth()
|
||||
screenheight = self.t.winfo_screenheight()
|
||||
windowwidth = w
|
||||
windowheight = h
|
||||
windowx = (screenwidth-windowwidth) / 2
|
||||
windowy = ((screenheight-windowheight) / 2) - 27
|
||||
self.geometrystring = '%dx%d+%d+%d' % (windowwidth, windowheight, windowx, windowy)
|
||||
self.t.geometry(self.geometrystring)
|
||||
|
||||
self.create_mainframe()
|
||||
self.build_gui_mainmenu()
|
||||
|
||||
def annihilate(self):
|
||||
self.mainframe.destroy()
|
||||
self.create_mainframe()
|
||||
|
||||
def build_gui_mainmenu(self):
|
||||
self.annihilate()
|
||||
font = ('Consolas', 15)
|
||||
|
||||
self.entry_left = tkinter.Entry(self.mainframe, font=font)
|
||||
self.entry_right = tkinter.Entry(self.mainframe, font=font)
|
||||
self.entry_left.grid(row=0, column=0, sticky='ew')
|
||||
self.entry_right.grid(row=1, column=0, sticky='ew')
|
||||
self.entry_left.insert(0, 'Left path')
|
||||
self.entry_right.insert(0, 'Right path')
|
||||
|
||||
button_start = tkinter.Button(self.mainframe, text='Go', command=self.build_gui_mapper)
|
||||
button_start.grid(row=2, column=0, sticky='ew')
|
||||
self.mainframe.rowconfigure(0, weight=1)
|
||||
self.mainframe.rowconfigure(1, weight=1)
|
||||
self.mainframe.columnconfigure(0, weight=1)
|
||||
|
||||
def build_gui_mapper(self):
|
||||
left_path = self.entry_left.get()
|
||||
right_path = self.entry_right.get()
|
||||
self.annihilate()
|
||||
left_files = os.listdir(left_path)
|
||||
right_files = os.listdir(right_path)
|
||||
print(left_files)
|
||||
print(right_files)
|
||||
|
||||
|
||||
def create_mainframe(self):
|
||||
self.mainframe = tkinter.Frame(self.t)
|
||||
self.mainframe.pack(fill='both', expand=True)
|
||||
|
||||
def mainloop(self):
|
||||
self.t.mainloop()
|
||||
|
||||
if __name__ == '__main__':
|
||||
a = Application()
|
||||
a.mainloop()
|
||||
|
|
@ -48,6 +48,7 @@ class Path(pathclass.Path):
|
|||
return any(self in okay for okay in OKAY_PATHS)
|
||||
|
||||
def anchor(self, display_name=None):
|
||||
self.correct_case()
|
||||
if display_name is None:
|
||||
display_name = self.basename
|
||||
|
||||
|
|
@ -59,7 +60,11 @@ class Path(pathclass.Path):
|
|||
icon = '\U0001F48E'
|
||||
|
||||
#print('anchor', path)
|
||||
a = '<a href="{full}">{icon} {display}</a>'.format(
|
||||
if display_name.endswith('.placeholder'):
|
||||
a = '<a>{icon} {display}</a>'
|
||||
else:
|
||||
a = '<a href="{full}">{icon} {display}</a>'
|
||||
a = a.format(
|
||||
full=self.url_path,
|
||||
icon=icon,
|
||||
display=display_name,
|
||||
|
|
|
|||
|
|
@ -189,7 +189,6 @@ def copy_dir(
|
|||
Returns: [destination path, number of bytes written to destination]
|
||||
(Written bytes is 0 if all files already existed.)
|
||||
'''
|
||||
|
||||
# Prepare parameters
|
||||
if not is_xor(destination, destination_new_root):
|
||||
m = 'One and only one of `destination` and '
|
||||
|
|
@ -199,6 +198,7 @@ def copy_dir(
|
|||
source = str_to_fp(source)
|
||||
|
||||
if destination_new_root is not None:
|
||||
source.correct_case()
|
||||
destination = new_root(source, destination_new_root)
|
||||
destination = str_to_fp(destination)
|
||||
|
||||
|
|
@ -279,10 +279,10 @@ def copy_file(
|
|||
destination_new_root=None,
|
||||
bytes_per_second=None,
|
||||
callback=None,
|
||||
callback_permission_denied=None,
|
||||
callback_verbose=None,
|
||||
dry_run=False,
|
||||
overwrite_old=True,
|
||||
callback_permission_denied=None,
|
||||
):
|
||||
'''
|
||||
Copy a file from one place to another.
|
||||
|
|
@ -412,8 +412,9 @@ def copy_file(
|
|||
callback(destination, written_bytes, source_bytes)
|
||||
|
||||
# Fin
|
||||
callback_verbose('Closing handles.')
|
||||
callback_verbose('Closing source handle.')
|
||||
source_file.close()
|
||||
callback_verbose('Closing dest handle.')
|
||||
destination_file.close()
|
||||
callback_verbose('Copying metadata')
|
||||
shutil.copystat(source.absolute_path, destination.absolute_path)
|
||||
|
|
@ -461,7 +462,7 @@ def limiter_or_none(value):
|
|||
if isinstance(value, ratelimiter.Ratelimiter):
|
||||
limiter = value
|
||||
elif value is not None:
|
||||
limiter = ratelimiter.Ratelimiter(allowance_per_period=value, period=1)
|
||||
limiter = ratelimiter.Ratelimiter(allowance=value, period=1)
|
||||
else:
|
||||
limiter = None
|
||||
return limiter
|
||||
|
|
|
|||
|
|
@ -30,16 +30,18 @@ def listget(li, index, fallback):
|
|||
except IndexError:
|
||||
return fallback
|
||||
|
||||
def threaded_dl(urls, thread_count=4):
|
||||
def threaded_dl(urls, thread_count, prefix=None):
|
||||
threads = []
|
||||
prefix_digits = len(str(len(urls)))
|
||||
prefix_text = '%0{digits}d_'.format(digits=prefix_digits)
|
||||
if prefix is None:
|
||||
prefix = now = int(time.time())
|
||||
prefix_text = '{prefix}_{{index:0{digits}d}}_'.format(prefix=prefix, digits=prefix_digits)
|
||||
for (index, url) in enumerate(urls):
|
||||
while len(threads) == thread_count:
|
||||
threads = remove_finished(threads)
|
||||
time.sleep(0.1)
|
||||
|
||||
prefix = prefix_text % index
|
||||
prefix = prefix_text.format(index=index)
|
||||
t = threading.Thread(target=download_thread, args=[url, prefix])
|
||||
t.daemon = True
|
||||
threads.append(t)
|
||||
|
|
@ -47,6 +49,7 @@ def threaded_dl(urls, thread_count=4):
|
|||
|
||||
while len(threads) > 0:
|
||||
threads = remove_finished(threads)
|
||||
print('%d threads remaining\r' % len(threads), end='', flush=True)
|
||||
time.sleep(0.1)
|
||||
|
||||
def main():
|
||||
|
|
@ -55,11 +58,13 @@ def main():
|
|||
f = open(filename, 'r')
|
||||
with f:
|
||||
urls = f.read()
|
||||
urls = urls.split()
|
||||
urls = urls.split('\n')
|
||||
else:
|
||||
urls = clipext.resolve(filename)
|
||||
urls = urls.split()
|
||||
threaded_dl(urls)
|
||||
urls = urls.split('\n')
|
||||
thread_count = int(listget(sys.argv, 2, 4))
|
||||
prefix = listget(sys.argv, 3, None)
|
||||
threaded_dl(urls, thread_count=thread_count, prefix=prefix)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Reference in a new issue