diff --git a/AHK/turboclick.ahk b/AHK/turboclick.ahk new file mode 100644 index 0000000..33ea06d --- /dev/null +++ b/AHK/turboclick.ahk @@ -0,0 +1,18 @@ +rapid_switch := false + +~*CapsLock:: rapid_switch := !rapid_switch + +~*LButton:: +If rapid_switch = 1 +{ + Loop + { + GetKeyState, var, LButton, P + If var = U + { + Break + } + Send {LButton} + sleep 20 + } +} \ No newline at end of file diff --git a/Clipext/clipext.py b/Clipext/clipext.py index 0f5dfb1..759255e 100644 --- a/Clipext/clipext.py +++ b/Clipext/clipext.py @@ -12,8 +12,8 @@ def multi_line_input(): except EOFError: # If you enter nothing but ctrl-z additional = EOF - - userinput.append(additional) + else: + userinput.append(additional) if EOF in additional: break diff --git a/Downloady/downloady.py b/Downloady/downloady.py index 9ab9797..1ac59de 100644 --- a/Downloady/downloady.py +++ b/Downloady/downloady.py @@ -7,9 +7,9 @@ import time import urllib import warnings +sys.path.append('C:\\git\\else\\bytestring'); import bytestring sys.path.append('C:\\git\\else\\clipext'); import clipext sys.path.append('C:\\git\\else\\ratelimiter'); import ratelimiter -sys.path.append('C:\\git\\else\\bytestring'); import bytestring warnings.simplefilter('ignore') @@ -20,18 +20,12 @@ HEADERS = { FILENAME_BADCHARS = '*?"<>|' last_request = 0 -CHUNKSIZE = 16 * bytestring.KIBIBYTE +CHUNKSIZE = 4 * bytestring.KIBIBYTE TIMEOUT = 600 TEMP_EXTENSION = '.downloadytemp' -def basename_from_url(url): - ''' - Determine the local filename appropriate for a URL. - ''' - localname = urllib.parse.unquote(url) - localname = localname.split('?')[0] - localname = localname.split('/')[-1] - return localname +PRINT_LIMITER = ratelimiter.Ratelimiter(allowance=5, mode='reject') + def download_file( url, @@ -41,6 +35,7 @@ def download_file( callback_progress=None, headers=None, overwrite=False, + raise_for_undersized=True, verbose=False, ): headers = headers or {} @@ -85,25 +80,31 @@ def download_file( bytes_downloaded = 0 download_stream = request('get', url, stream=True, headers=headers, auth=auth) + if callback_progress is not None: + callback_progress = callback_progress(plan['remote_total_bytes']) + for chunk in download_stream.iter_content(chunk_size=CHUNKSIZE): bytes_downloaded += len(chunk) file_handle.write(chunk) if callback_progress is not None: - callback_progress(bytes_downloaded, plan['remote_total_bytes']) + callback_progress.step(bytes_downloaded) if plan['limiter'] is not None and bytes_downloaded < plan['remote_total_bytes']: plan['limiter'].limit(len(chunk)) file_handle.close() - if localname != plan['real_localname']: - os.rename(localname, plan['real_localname']) + # Don't try to rename /dev/null + if os.devnull not in [localname, plan['real_localname']]: + localsize = os.path.getsize(localname) + undersized = plan['plan_type'] != 'partial' and localsize < plan['remote_total_bytes'] + if raise_for_undersized and undersized: + message = 'File does not contain expected number of bytes. Received {size} / {total}' + message = message.format(size=localsize, total=plan['remote_total_bytes']) + raise Exception(message) - localsize = os.path.getsize(plan['real_localname']) - if plan['plan_type'] != 'partial' and localsize < plan['remote_total_bytes']: - message = 'File does not contain expected number of bytes. Received {size} / {total}' - message = message.format(size=os.path.getsize(localname), total=plan['remote_total_bytes']) - raise Exception(message) + if localname != plan['real_localname']: + os.rename(localname, plan['real_localname']) return plan['real_localname'] @@ -134,7 +135,7 @@ def prepare_plan( elif isinstance(bytespersecond, ratelimiter.Ratelimiter): limiter = bytespersecond else: - limiter = ratelimiter.Ratelimiter(bytespersecond) + limiter = ratelimiter.Ratelimiter(allowance=bytespersecond) # Chapter 3: Extracting range if user_provided_range: @@ -222,63 +223,95 @@ def prepare_plan( print('No plan was chosen?') return None + + +class Progress1: + def __init__(self, total_bytes): + self.limiter = ratelimiter.Ratelimiter(allowance=5, mode='reject') + self.limiter.balance = 1 + self.total_bytes = max(1, total_bytes) + self.divisor = bytestring.get_appropriate_divisor(total_bytes) + self.total_format = bytestring.bytestring(total_bytes, force_unit=self.divisor) + self.downloaded_format = '{:>%d}' % len(self.total_format) + self.blank_char = ' ' + self.solid_char = '█' + + def step(self, bytes_downloaded): + #print(self.limiter.balance) + percent = bytes_downloaded / self.total_bytes + percent = min(1, percent) + if self.limiter.limit(1) is False and percent < 1: + return + + downloaded_string = bytestring.bytestring(bytes_downloaded, force_unit=self.divisor) + downloaded_string = self.downloaded_format.format(downloaded_string) + block_count = 50 + solid_blocks = self.solid_char * int(block_count * percent) + statusbar = solid_blocks.ljust(block_count, self.blank_char) + statusbar = self.solid_char + statusbar + self.solid_char + + end = '\n' if percent == 1 else '' + message = '\r{bytes_downloaded} {statusbar} {total_bytes}' + message = message.format( + bytes_downloaded=downloaded_string, + total_bytes=self.total_format, + statusbar=statusbar, + ) + print(message, end=end, flush=True) + + +class Progress2: + def __init__(self, total_bytes): + self.total_bytes = max(1, total_bytes) + self.limiter = ratelimiter.Ratelimiter(allowance=5, mode='reject') + self.limiter.balance = 1 + self.total_bytes_string = '{:,}'.format(self.total_bytes) + self.bytes_downloaded_string = '{:%d,}' % len(self.total_bytes_string) + + def step(self, bytes_downloaded): + percent = (bytes_downloaded * 100) / self.total_bytes + percent = min(100, percent) + if self.limiter.limit(1) is False and percent < 100: + return + + percent_string = '%08.4f' % percent + bytes_downloaded_string = self.bytes_downloaded_string.format(bytes_downloaded) + + end = '\n' if percent == 100 else '' + message = '\r{bytes_downloaded} / {total_bytes} / {percent}%' + message = message.format( + bytes_downloaded=bytes_downloaded_string, + total_bytes=self.total_bytes_string, + percent=percent_string, + ) + print(message, end=end, flush=True) + +progress1 = Progress1 +progress2 = Progress2 + +def basename_from_url(url): + ''' + Determine the local filename appropriate for a URL. + ''' + localname = urllib.parse.unquote(url) + localname = localname.split('?')[0] + localname = localname.split('/')[-1] + return localname def get_permission(prompt='y/n\n>', affirmative=['y', 'yes']): permission = input(prompt) return permission.lower() in affirmative -def progress1(bytes_downloaded, bytes_total, prefix=''): - divisor = bytestring.get_appropriate_divisor(bytes_total) - bytes_total_string = bytestring.bytestring(bytes_total, force_unit=divisor) - bytes_downloaded_string = bytestring.bytestring(bytes_downloaded, force_unit=divisor) - bytes_downloaded_string = bytes_downloaded_string.rjust(len(bytes_total_string), ' ') - - blocks = 50 - char = '█' - percent = bytes_downloaded * 100 / bytes_total - percent = int(min(100, percent)) - completed_blocks = char * int(blocks * percent / 100) - incompleted_blocks = ' ' * (blocks - len(completed_blocks)) - statusbar = '{char}{complete}{incomplete}{char}'.format( - char=char, - complete=completed_blocks, - incomplete=incompleted_blocks, - ) - - end = '\n' if percent == 100 else '' - message = '\r{prefix}{bytes_downloaded} {statusbar} {bytes_total}' - message = message.format( - prefix=prefix, - bytes_downloaded=bytes_downloaded_string, - bytes_total=bytes_total_string, - statusbar=statusbar, - ) - print(message, end=end, flush=True) - -def progress2(bytes_downloaded, bytes_total, prefix=''): - percent = (bytes_downloaded*100)/bytes_total - percent = min(100, percent) - percent_string = '%08.4f' % percent - bytes_downloaded_string = '{0:,}'.format(bytes_downloaded) - bytes_total_string = '{0:,}'.format(bytes_total) - bytes_downloaded_string = bytes_downloaded_string.rjust(len(bytes_total_string), ' ') - - end = '\n' if percent == 100 else '' - message = '\r{prefix}{bytes_downloaded} / {bytes_total} / {percent}%' - message = message.format( - prefix=prefix, - bytes_downloaded=bytes_downloaded_string, - bytes_total=bytes_total_string, - percent=percent_string, - ) - print(message, end=end, flush=True) - def request(method, url, stream=False, headers=None, timeout=TIMEOUT, **kwargs): if headers is None: headers = {} for (key, value) in HEADERS.items(): headers.setdefault(key, value) session = requests.Session() + a = requests.adapters.HTTPAdapter(max_retries=30) + b = requests.adapters.HTTPAdapter(max_retries=30) + session.mount('http://', a) + session.mount('https://', b) session.max_redirects = 40 method = { @@ -286,8 +319,7 @@ def request(method, url, stream=False, headers=None, timeout=TIMEOUT, **kwargs): 'head': session.head, 'post': session.post, }[method] - - req = method(url, stream=stream, headers=headers, timeout=timeout, **kwargs) + req = method(url, stream=stream, headers=headers, timeout=None, **kwargs) req.raise_for_status() return req @@ -311,11 +343,10 @@ def download_argparse(args): url = args.url url = clipext.resolve(url) - callback = { - None: progress1, - '1': progress1, - '2': progress2, + None: Progress1, + '1': Progress1, + '2': Progress2, }.get(args.callback, args.callback) bytespersecond = args.bytespersecond diff --git a/Etiquette/bad_search.txt b/Etiquette/bad_search.txt new file mode 100644 index 0000000..d6f5e86 --- /dev/null +++ b/Etiquette/bad_search.txt @@ -0,0 +1,114 @@ + start_time = time.time() + + + # Raise for cases where the minimum > maximum + for (maxkey, maxval) in maximums.items(): + if maxkey not in minimums: + continue + minval = minimums[maxkey] + if minval > maxval: + raise ValueError('Impossible min-max for %s' % maxkey) + + conditions = [] + minmaxers = {'<=': maximums, '>=': minimums} + + # Convert the min-max parameters into query strings + print('Writing minmaxers') + for (comparator, minmaxer) in minmaxers.items(): + for (field, value) in minmaxer.items(): + if field not in Photo.int_properties: + raise ValueError('Unknown Photo property: %s' % field) + + value = str(value) + query = min_max_query_builder(field, comparator, value) + conditions.append(query) + + print(conditions) + + print('Writing extension rule') + if extension is not None: + if isinstance(extension, str): + extension = [extension] + + # Normalize to prevent injections + extension = [normalize_tagname(e) for e in extension] + extension = ['extension == "%s"' % e for e in extension] + extension = ' OR '.join(extension) + extension = '(%s)' % extension + conditions.append(extension) + + def setify(l): + if l is None: + return set() + else: + return set(self.get_tag_by_name(t) for t in l) + + tag_musts = setify(tag_musts) + tag_mays = setify(tag_mays) + tag_forbids = setify(tag_forbids) + + base = ''' + {negator} EXISTS( + SELECT 1 FROM photo_tag_rel + WHERE photo_tag_rel.photoid == photos.id + AND photo_tag_rel.tagid {operator} {value} + )''' + + print('Writing musts') + for tag in tag_musts: + # tagid == must + query = base.format( + negator='', + operator='==', + value='"%s"' % tag.id, + ) + conditions.append(query) + + print('Writing mays') + if len(tag_mays) > 0: + # not any(tagid not in mays) + acceptable = tag_mays.union(tag_musts) + acceptable = ['"%s"' % t.id for t in acceptable] + acceptable = ', '.join(acceptable) + query = base.format( + negator='', + operator='IN', + value='(%s)' % acceptable, + ) + conditions.append(query) + + print('Writing forbids') + if len(tag_forbids) > 0: + # not any(tagid in forbids) + forbids = ['"%s"' % t.id for t in tag_forbids] + forbids = ', '.join(forbids) + query = base.format( + negator='NOT', + operator='IN', + value='(%s)' % forbids + ) + conditions.append(query) + + if len(conditions) == 0: + raise ValueError('No search query provided') + + conditions = [query for query in conditions if query is not None] + conditions = ['(%s)' % c for c in conditions] + conditions = ' AND '.join(conditions) + conditions = 'WHERE %s' % conditions + + query = 'SELECT * FROM photos %s' % conditions + query = query.replace('\n', ' ') + while ' ' in query: + query = query.replace(' ', ' ') + print(query) + + temp_cur = self.sql.cursor() + temp_cur.execute(query) + + for fetch in fetch_generator(temp_cur): + photo = Photo(self, fetch) + yield photo + + end_time = time.time() + print(end_time - start_time) \ No newline at end of file diff --git a/Etiquette/etiquette.py b/Etiquette/etiquette.py new file mode 100644 index 0000000..8f92c4a --- /dev/null +++ b/Etiquette/etiquette.py @@ -0,0 +1 @@ +import phototagger diff --git a/Etiquette/phototagger.py b/Etiquette/phototagger.py new file mode 100644 index 0000000..1be42c5 --- /dev/null +++ b/Etiquette/phototagger.py @@ -0,0 +1,1163 @@ +import datetime +import os +import PIL.Image +import random +import sqlite3 +import string +import time +import warnings + +ID_LENGTH = 22 +VALID_TAG_CHARS = string.ascii_lowercase + string.digits + '_-' +MIN_TAG_NAME_LENGTH = 1 +MAX_TAG_NAME_LENGTH = 32 + +SQL_LASTID_COLUMNS = [ + 'table', + 'last_id', +] + +SQL_PHOTO_COLUMNS = [ + 'id', + 'filepath', + 'extension', + 'width', + 'height', + 'ratio', + 'area', + 'bytes', + 'created', +] + +SQL_PHOTOTAG_COLUMNS = [ + 'photoid', + 'tagid', +] + +SQL_SYN_COLUMNS = [ + 'name', + 'master', +] + +SQL_TAG_COLUMNS = [ + 'id', + 'name', +] + +SQL_GROUP_COLUMNS = [ + 'id', + 'name', +] + +SQL_TAGGROUP_COLUMNS = [ + 'groupid', + 'memberid', + 'membertype', +] + +SQL_GROUP = {key:index for (index, key) in enumerate(SQL_GROUP_COLUMNS)} +SQL_LASTID = {key:index for (index, key) in enumerate(SQL_LASTID_COLUMNS)} +SQL_PHOTO = {key:index for (index, key) in enumerate(SQL_PHOTO_COLUMNS)} +SQL_PHOTOTAG = {key:index for (index, key) in enumerate(SQL_PHOTOTAG_COLUMNS)} +SQL_SYN = {key:index for (index, key) in enumerate(SQL_SYN_COLUMNS)} +SQL_TAG = {key:index for (index, key) in enumerate(SQL_TAG_COLUMNS)} +SQL_TAGGROUP = {key:index for (index, key) in enumerate(SQL_TAGGROUP_COLUMNS)} + + +DB_INIT = ''' +CREATE TABLE IF NOT EXISTS albums( + albumid TEXT, + photoid TEXT +); +CREATE TABLE IF NOT EXISTS tags( + id TEXT, + name TEXT +); +CREATE TABLE IF NOT EXISTS groups( + id TEXT, + name TEXT +); +CREATE TABLE IF NOT EXISTS photos( + id TEXT, + filepath TEXT, + extension TEXT, + width INT, + height INT, + ratio REAL, + area INT, + bytes INT, + created INT +); +CREATE TABLE IF NOT EXISTS photo_tag_rel( + photoid TEXT, + tagid TEXT +); +CREATE TABLE IF NOT EXISTS tag_group_rel( + groupid TEXT, + memberid TEXT, + membertype TEXT +); +CREATE TABLE IF NOT EXISTS tag_synonyms( + name TEXT, + mastername TEXT +); +CREATE TABLE IF NOT EXISTS id_numbers( + tab TEXT, + last_id TEXT +); +CREATE INDEX IF NOT EXISTS index_photo_id on photos(id); +CREATE INDEX IF NOT EXISTS index_photo_path on photos(filepath); +CREATE INDEX IF NOT EXISTS index_photo_created on photos(created); + +CREATE INDEX IF NOT EXISTS index_tag_id on tags(id); +CREATE INDEX IF NOT EXISTS index_tag_name on tags(name); + +CREATE INDEX IF NOT EXISTS index_group_id on groups(id); + +CREATE INDEX IF NOT EXISTS index_tagrel_photoid on photo_tag_rel(photoid); +CREATE INDEX IF NOT EXISTS index_tagrel_tagid on photo_tag_rel(tagid); + +CREATE INDEX IF NOT EXISTS index_tagsyn_name on tag_synonyms(name); + +CREATE INDEX IF NOT EXISTS index_grouprel_groupid on tag_group_rel(groupid); +CREATE INDEX IF NOT EXISTS index_grouprel_memberid on tag_group_rel(memberid); +''' + +def basex(number, base, alphabet='0123456789abcdefghijklmnopqrstuvwxyz'): + ''' + Converts an integer to a different base string. + Based on http://stackoverflow.com/a/1181922/5430534 + ''' + if base > len(alphabet): + raise Exception('alphabet %s does not support base %d' % (repr(alphabet), base)) + + if not isinstance(number, (int, str)): + raise TypeError('number must be an integer') + + alphabet = alphabet[:base] + number = int(number) + based = '' + sign = '' + if number < 0: + sign = '-' + number = -number + if 0 <= number < len(alphabet): + return sign + alphabet[number] + while number != 0: + number, i = divmod(number, len(alphabet)) + based = alphabet[i] + based + return sign + based + +def getnow(timestamp=True): + ''' + Return the current UTC timestamp or datetime object. + ''' + now = datetime.datetime.now(datetime.timezone.utc) + if timestamp: + return now.timestamp() + return now + +def int_or_none(i): + if i is None: + return i + return int(i) + +def is_xor(*args): + ''' + Return True if and only if one arg is truthy. + ''' + return [bool(a) for a in args].count(True) == 1 + +def min_max_query_builder(name, comparator, value): + return ' '.join([name, comparator, value]) + +def normalize_tagname(tagname): + ''' + Tag names can only consist of VALID_TAG_CHARS. + The given tagname is lowercased, gets its spaces + replaced by underscores, and is stripped of any not-whitelisted + characters. + ''' + tagname = tagname.lower() + tagname = tagname.replace(' ', '_') + tagname = (c for c in tagname if c in VALID_TAG_CHARS) + tagname = ''.join(tagname) + + if len(tagname) < MIN_TAG_NAME_LENGTH: + raise TagTooShort(tagname) + if len(tagname) > MAX_TAG_NAME_LENGTH: + raise TagTooLong(tagname) + + return tagname + +def not_implemented(function): + ''' + Decorator for keeping track of which functions still need to be filled out. + ''' + warnings.warn('%s is not implemented' % function.__name__) + return function + +def raise_no_such_thing(exception_class, thing_id=None, thing_name=None, comment=''): + if thing_id is not None: + message = 'ID: %s. %s' % (thing_id, comment) + elif thing_name is not None: + message = 'Name: %s. %s' % (thing_name, comment) + else: + message = '' + raise exception_class(message) + +def select(sql, query, bindings=[]): + cursor = sql.cursor() + cursor.execute(query, bindings) + while True: + fetch = cursor.fetchone() + if fetch is None: + break + yield fetch + + +#################################################################################################### +#################################################################################################### + + +class NoSuchGroup(Exception): + pass + +class NoSuchPhoto(Exception): + pass + +class NoSuchSynonym(Exception): + pass + +class NoSuchTag(Exception): + pass + + +class PhotoExists(Exception): + pass + +class TagExists(Exception): + pass + +class GroupExists(Exception): + pass + + +class TagTooLong(Exception): + pass + +class TagTooShort(Exception): + pass + +class XORException(Exception): + pass + + +#################################################################################################### +#################################################################################################### + + +class PDBGroupMixin: + def get_group(self, groupname=None, groupid=None): + ''' + Redirect to get_group_by_id or get_group_by_name after xor-checking the parameters. + ''' + if not is_xor(groupid, groupname): + raise XORException('One and only one of `groupid`, `groupname` can be passed.') + + if groupid is not None: + return self.get_thing_by_id('group', thing_id=groupid) + elif groupname is not None: + return self.get_group_by_name(groupname) + else: + raise_no_such_thing(NoSuchTag, thing_id=groupid, thing_name=groupname) + + def get_group_by_id(self, groupid): + return self.get_thing_by_id('group', groupid) + + def get_group_by_name(self, groupname): + if isinstance(groupname, Group): + if groupname.photodb == self: + return groupname + groupname = groupname.name + + groupname = normalize_tagname(groupname) + + self.cur.execute('SELECT * FROM groups WHERE name == ?', [groupname]) + fetch = self.cur.fetchone() + if fetch is None: + raise_no_such_thing(NoSuchGroup, thing_name=groupname) + + group = Group(self, fetch) + return group + + def get_groups(self): + yield from self.get_things(thing_type='group') + + def new_group(self, groupname, commit=True): + ''' + Register a new tag group and return the Group object. + ''' + groupname = normalize_tagname(groupname) + try: + self.get_group_by_name(groupname) + except NoSuchGroup: + pass + else: + raise GroupExists(groupname) + + groupid = self.generate_id('groups') + self.cur.execute('INSERT INTO groups VALUES(?, ?)', [groupid, groupname]) + if commit: + self.sql.commit() + + group = Group(self, [groupid, groupname]) + return group + + +class PDBPhotoMixin: + def get_photo_by_id(self, photoid): + return self.get_thing_by_id('photo', photoid) + + def get_photo_by_path(self, filepath): + filepath = os.path.abspath(filepath) + self.cur.execute('SELECT * FROM photos WHERE filepath == ?', [filepath]) + fetch = self.cur.fetchone() + if fetch is None: + raise_no_such_thing(NoSuchPhoto, thing_name=filepath) + photo = Photo(self, fetch) + return photo + + def get_photos_by_recent(self, count=None): + ''' + Yield photo objects in order of creation time. + ''' + if count is not None and count <= 0: + return + # We're going to use a second cursor because the first one may + # get used for something else, deactivating this query. + temp_cur = self.sql.cursor() + temp_cur.execute('SELECT * FROM photos ORDER BY created DESC') + while True: + f = temp_cur.fetchone() + if f is None: + break + photo = Photo(self, f) + + yield photo + + if count is None: + continue + count -= 1 + if count <= 0: + break + + def get_photos_by_search( + self, + extension=None, + maximums={}, + minimums={}, + no_tags=None, + orderby=[], + tag_musts=None, + tag_mays=None, + tag_forbids=None, + ): + ''' + extension: + A string or list of strings of acceptable file extensions. + + maximums + A dictionary, where the key is an attribute of the photo, + (area, bytes, created, height, id, or width) + and the value is the maximum desired value for that field. + + minimums: + A dictionary like `maximums` where the value is the minimum + desired value for that field. + + no_tags: + If True, require that the Photo has no tags. + If False, require that the Photo has >=1 tag. + If None, not considered. + + orderby: + A list of strings like ['ratio DESC', 'created ASC'] to sort + and subsort the results. + Descending is assumed if not provided. + + tag_musts: + A list of tag names or Tag objects. + Photos MUST have ALL tags in this list. + + tag_mays: + A list of tag names or Tag objects. + Photos MUST have AT LEAST ONE tag in this list. + + tag_forbids: + A list of tag names or Tag objects. + Photos MUST NOT have ANY tag in the list. + ''' + maximums = {key:int(val) for (key, val) in maximums.items()} + minimums = {key:int(val) for (key, val) in minimums.items()} + # Raise for cases where the minimum > maximum + for (maxkey, maxval) in maximums.items(): + if maxkey not in minimums: + continue + minval = minimums[maxkey] + if minval > maxval: + raise ValueError('Impossible min-max for %s' % maxkey) + + if isinstance(extension, str): + extension = [extension] + if extension is not None: + extension = [e.lower() for e in extension] + + def validate_orderby(o): + o = o.lower() + o = o.split(' ') + assert len(o) in (1, 2) + if len(o) == 1: + o.append('desc') + assert o[0] in ['extension', 'width', 'height', 'ratio', 'area', 'bytes', 'created'] + o = ' '.join(o) + return o + + orderby = [validate_orderby(o) for o in orderby] + orderby = ', '.join(orderby) + print(orderby) + + def setify(l): + if l is None: + return set() + else: + return set(self.get_tag_by_name(t) for t in l) + + tag_musts = setify(tag_musts) + tag_mays = setify(tag_mays) + tag_forbids = setify(tag_forbids) + + query = 'SELECT * FROM photos' + if orderby: + query += ' ORDER BY %s' % orderby + print(query) + generator = select(self.sql, query) + for fetch in generator: + if extension and not any(fetch[SQL_PHOTO['extension']].lower() == e for e in extension): + #print('Failed extension') + continue + + if any(fetch[SQL_PHOTO[key]] and fetch[SQL_PHOTO[key]] > value for (key, value) in maximums.items()): + #print('Failed maximums') + continue + + if any(fetch[SQL_PHOTO[key]] and fetch[SQL_PHOTO[key]] < value for (key, value) in minimums.items()): + #print('Failed minimums') + continue + + photo = Photo(self, fetch) + if (no_tags is not None) or tag_musts or tag_mays or tag_forbids: + photo_tags = photo.get_tags() + + if no_tags is True and len(photo_tags) > 0: + continue + + if no_tags is False and len(photo_tags) == 0: + continue + + if tag_musts and any(tag not in photo_tags for tag in tag_musts): + #print('Failed musts') + continue + + if tag_mays and not any(may in photo_tags for may in tag_mays): + #print('Failed mays') + continue + + if tag_forbids and any(forbid in photo_tags for forbid in tag_forbids): + #print('Failed forbids') + continue + + yield photo + + def new_photo(self, filename, tags=None, allow_duplicates=False, commit=True): + ''' + Given a filepath, determine its attributes and create a new Photo object in the + database. Tags may be applied now or later. + + If `allow_duplicates` is False, we will first check the database for any files + with the same path and raise PhotoExists if found. + + Returns the Photo object. + ''' + filename = os.path.abspath(filename) + if not allow_duplicates: + try: + existing = self.get_photo_by_path(filename) + except NoSuchPhoto: + pass + else: + raise PhotoExists(filename, existing) + + try: + image = PIL.Image.open(filename) + (width, height) = image.size + area = width * height + ratio = width / height + image.close() + except OSError: + # PIL did not recognize it as an image + width = None + height = None + area = None + ratio = None + + extension = os.path.splitext(filename)[1] + extension = extension.replace('.', '') + extension = normalize_tagname(extension) + bytes = os.path.getsize(filename) + created = int(getnow()) + photoid = self.generate_id('photos') + + data = [None] * len(SQL_PHOTO_COLUMNS) + data[SQL_PHOTO['id']] = photoid + data[SQL_PHOTO['filepath']] = filename + data[SQL_PHOTO['extension']] = extension + data[SQL_PHOTO['width']] = width + data[SQL_PHOTO['height']] = height + data[SQL_PHOTO['area']] = area + data[SQL_PHOTO['ratio']] = ratio + data[SQL_PHOTO['bytes']] = bytes + data[SQL_PHOTO['created']] = created + photo = Photo(self, data) + + self.cur.execute('INSERT INTO photos VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)', data) + + tags = tags or [] + for tag in tags: + try: + photo.apply_tag(tag, commit=False) + except NoSuchTag: + self.sql.rollback() + raise + if commit: + self.sql.commit() + return photo + + +class PDBTagMixin: + def convert_tag_to_synonym(self, oldtagname, mastertagname): + ''' + Convert an independent tag into a synonym for a different independent tag. + All photos which possess the current tag will have it replaced + with the new master tag. + All synonyms of the old tag will point to the new tag. + + Good for when two tags need to be merged under a single name. + ''' + oldtagname = normalize_tagname(oldtagname) + mastertagname = normalize_tagname(mastertagname) + + oldtag = self.get_tag_by_name(oldtagname) + if oldtag.name != oldtagname: + # The inputted name was a synonym and we got redirected. + raise NoSuchTag('%s is not an independent tag!' % oldtagname) + + mastertag = self.get_tag_by_name(mastertagname) + if mastertag.name != mastertagname: + raise NoSuchTag('%s is not an independent tag!' % mastertagname) + + # Migrate the old tag's synonyms to the new one + # UPDATE is safe for this operation because there is no chance of duplicates. + self.cur.execute( + 'UPDATE tag_synonyms SET mastername = ? WHERE mastername == ?', + [mastertagname, oldtagname] + ) + + # Iterate over all photos with the old tag, and relate them to the new tag + # if they aren't already. + generator = select(self.sql, 'SELECT * FROM photo_tag_rel WHERE tagid == ?', [oldtag.id]) + for relationship in generator: + photoid = relationship[SQL_PHOTOTAG['photoid']] + self.cur.execute('SELECT * FROM photo_tag_rel WHERE tagid == ?', [mastertag.id]) + if self.cur.fetchone() is not None: + continue + self.cur.execute('INSERT INTO photo_tag_rel VALUES(?, ?)', [photoid, mastertag.id]) + + # Then delete the relationships with the old tag + self.cur.execute('DELETE FROM photo_tag_rel WHERE tagid == ?', [oldtag.id]) + self.cur.execute('DELETE FROM tags WHERE id == ?', [oldtag.id]) + + # Enjoy your new life as a monk. + mastertag.new_synonym(oldtag.name, commit=True) + + def export_tags(self): + def print_children(obj, depth=1): + msg = ' ' * (4 * depth) + if isinstance(obj, Group): + children = obj.children() + children.sort(key=lambda x: (x._membertype, x.name)) + for child in children: + print(msg, child, sep='') + print_children(child, depth=depth+1) + else: + synonyms = obj.get_synonyms() + synonyms.sort() + for synonym in synonyms: + print(msg, synonym, sep='') + + + items = list(self.get_groups()) + list(self.get_tags()) + items.sort(key=lambda x: (x._membertype, x.name)) + for item in items: + if item.group() is not None: + continue + print(item) + print_children(item) + + def get_tag(self, tagname=None, tagid=None): + ''' + Redirect to get_tag_by_id or get_tag_by_name after xor-checking the parameters. + ''' + if not is_xor(tagid, tagname): + raise XORException('One and only one of `tagid`, `tagname` can be passed.') + + if tagid is not None: + return self.get_tag_by_id(tagid) + elif tagname is not None: + return self.get_tag_by_name(tagname) + else: + raise_no_such_thing(NoSuchTag, thing_id=tagid, thing_name=tagname) + + def get_tag_by_id(self, tagid): + return self.get_thing_by_id('tag', thing_id=tagid) + + def get_tag_by_name(self, tagname): + if isinstance(tagname, Tag): + if tagname.photodb == self: + return tagname + tagname = tagname.name + + tagname = normalize_tagname(tagname) + + while True: + # Return if it's a toplevel, or resolve the synonym and try that. + self.cur.execute('SELECT * FROM tags WHERE name == ?', [tagname]) + fetch = self.cur.fetchone() + if fetch is not None: + return Tag(self, fetch) + + self.cur.execute('SELECT * FROM tag_synonyms WHERE name == ?', [tagname]) + fetch = self.cur.fetchone() + if fetch is None: + # was not a top tag or synonym + raise_no_such_thing(NoSuchTag, thing_name=tagname) + tagname = fetch[SQL_SYN['master']] + + def get_tags(self): + yield from self.get_things(thing_type='tag') + + def new_tag(self, tagname, commit=True): + ''' + Register a new tag in and return the Tag object. + ''' + tagname = normalize_tagname(tagname) + try: + self.get_tag_by_name(tagname) + except NoSuchTag: + pass + else: + raise TagExists(tagname) + tagid = self.generate_id('tags') + self.cur.execute('INSERT INTO tags VALUES(?, ?)', [tagid, tagname]) + if commit: + self.sql.commit() + tag = Tag(self, [tagid, tagname]) + return tag + + +class PhotoDB(PDBGroupMixin, PDBPhotoMixin, PDBTagMixin): + ''' + This class represents an SQLite3 database containing the following tables: + + albums: + Rows represent the inclusion of a photo in an album + + photos: + Rows represent image files on the local disk. + Entries contain a unique ID, the image's filepath, and metadata + like dimensions and filesize. + + tags: + Rows represent labels, which can be applied to an arbitrary number of + photos. Photos may be selected by which tags they contain. + Entries contain a unique ID and a name. + + photo_tag_rel: + Rows represent a Photo's ownership of a particular Tag. + + tag_synonyms: + Rows represent relationships between two tag names, so that they both + resolve to the same Tag object when selected. Entries contain the + subordinate name and master name. + The master name MUST also exist in the `tags` table. + If a new synonym is created referring to another synoym, the master name + will be resolved and used instead, so a synonym never points to another synonym. + Tag objects will ALWAYS represent the master tag. + + Note that the entries in this table do not contain ID numbers. + The rationale here is that "coco" is a synonym for "chocolate" regardless + of the "chocolate" tag's ID, and that if a tag is renamed, its synonyms + do not necessarily follow. + The `rename` method of Tag objects includes a parameter + `apply_to_synonyms` if you do want them to follow. + ''' + def __init__(self, databasename='phototagger.db', id_length=None): + if id_length is None: + self.id_length = ID_LENGTH + self.databasename = databasename + self.sql = sqlite3.connect(databasename) + self.cur = self.sql.cursor() + statements = DB_INIT.split(';') + for statement in statements: + self.cur.execute(statement) + self._last_ids = {} + + def __repr__(self): + return 'PhotoDB(databasename={dbname})'.format(dbname=repr(self.databasename)) + + def generate_id(self, table): + ''' + Create a new ID number that is unique to the given table. + Note that this method does not commit the database. We'll wait for that + to happen in whoever is calling us, so we know the ID is actually used. + ''' + table = table.lower() + if table not in ['photos', 'tags', 'groups']: + raise ValueError('Invalid table requested: %s.', table) + + do_insert = False + if table in self._last_ids: + # Use cache value + new_id_int = self._last_ids[table] + 1 + else: + self.cur.execute('SELECT * FROM id_numbers WHERE tab == ?', [table]) + fetch = self.cur.fetchone() + if fetch is None: + # Register new value + new_id_int = 1 + do_insert = True + else: + # Use database value + new_id_int = int(fetch[SQL_LASTID['last_id']]) + 1 + + new_id = str(new_id_int).rjust(self.id_length, '0') + if do_insert: + self.cur.execute('INSERT INTO id_numbers VALUES(?, ?)', [table, new_id]) + else: + self.cur.execute('UPDATE id_numbers SET last_id = ? WHERE tab == ?', [new_id, table]) + self._last_ids[table] = new_id_int + return new_id + + def get_thing_by_id(self, thing_type, thing_id): + thing_map = self.thing_map(thing_type) + + if isinstance(thing_id, thing_map['class']): + if thing_id.photodb == self: + return thing_id + thing_id = thing_id.id + + query = 'SELECT * FROM %s WHERE id == ?' % thing_map['table'] + self.cur.execute(query, [thing_id]) + thing = self.cur.fetchone() + if thing is None: + return raise_no_such_thing(thing_map['exception'], thing_id=thing_id) + thing = thing_map['class'](self, thing) + return thing + + def get_things(self, thing_type, orderby=None): + thing_map = self.thing_map(thing_type) + + if orderby: + self.cur.execute('SELECT * FROM %s ORDER BY %s' % (thing_map['table'], orderby)) + else: + self.cur.execute('SELECT * FROM %s' % thing_map['table']) + + things = self.cur.fetchall() + for thing in things: + thing = thing_map['class'](self, row_tuple=thing) + yield thing + + def thing_map(self, thing_type): + if thing_type == 'tag': + return { + 'class': Tag, + 'exception': NoSuchTag, + 'table': 'tags', + } + + elif thing_type == 'group': + return { + 'class': Group, + 'exception': NoSuchGroup, + 'table': 'groups', + } + + elif thing_type == 'photo': + return { + 'class': Photo, + 'exception': NoSuchPhoto, + 'table': 'photos', + } + + else: + raise Exception('Unknown type %s' % thing_type) + + +#################################################################################################### +#################################################################################################### + + +class ObjectBase: + def __eq__(self, other): + if not isinstance(other, type(self)): + return False + return self.id == other.id + + def __hash__(self): + return hash(self.id) + + +class Groupable: + def group(self): + ''' + Return the Group object of which this is a member, or None. + ''' + self.photodb.cur.execute( + 'SELECT * FROM tag_group_rel WHERE memberid == ? AND membertype == ?', + [self.id, self._membertype] + ) + fetch = self.photodb.cur.fetchone() + if fetch is None: + return None + + groupid = fetch[SQL_TAGGROUP['groupid']] + return self.photodb.get_group(groupid=groupid) + + def join(self, group, commit=True): + ''' + Leave the current group, then call `group.add(self)`. + ''' + group = self.photodb.get_group(group) + self.leave_group(commit=commit) + group.add(self, commit=commit) + + def leave_group(self, commit=True): + ''' + Leave the current group and become independent. + ''' + self.photodb.cur.execute( + 'DELETE FROM tag_group_rel WHERE memberid == ? AND membertype == ?', + [self.id, self._membertype] + ) + if commit: + self.photodb.sql.commit() + + +class Group(ObjectBase, Groupable): + ''' + A heirarchical organization of related Tags. + ''' + _membertype = 'group' + def __init__(self, photodb, row_tuple): + self.photodb = photodb + self.id = row_tuple[SQL_GROUP['id']] + self.name = row_tuple[SQL_GROUP['name']] + + def __repr__(self): + return 'Group:{name}'.format(name=self.name) + + def add(self, member, commit=True): + ''' + Add a Tag or Group object to this group. + + If that object is already a member of another group, a GroupExists is raised. + ''' + if isinstance(member, (str, Tag)): + member = self.photodb.get_tag(member) + membertype = 'tag' + elif isinstance(member, Group): + member = self.photodb.get_group(member) + membertype = 'group' + else: + raise TypeError('Type `%s` cannot join a Group' % type(member)) + + self.photodb.cur.execute( + 'SELECT * FROM tag_group_rel WHERE memberid == ? AND membertype == ?', + [member.id, membertype] + ) + fetch = self.photodb.cur.fetchone() + if fetch is not None: + if fetch[SQL_TAGGROUP['groupid']] == self.id: + that_group = self + else: + that_group = self.photodb.get_group(groupid=fetch[SQL_TAGGROUP['groupid']]) + raise GroupExists('%s already in group %s' % (member.name, that_group.name)) + + self.photodb.cur.execute( + 'INSERT INTO tag_group_rel VALUES(?, ?, ?)', + [self.id, member.id, membertype] + ) + if commit: + self.photodb.sql.commit() + + def children(self): + self.photodb.cur.execute('SELECT * FROM tag_group_rel WHERE groupid == ?', [self.id]) + fetch = self.photodb.cur.fetchall() + results = [] + for f in fetch: + memberid = f[SQL_TAGGROUP['memberid']] + if f[SQL_TAGGROUP['membertype']] == 'tag': + results.append(self.photodb.get_tag(tagid=memberid)) + else: + results.append(self.photodb.get_group(groupid=memberid)) + return results + + def delete(self, delete_children=False, commit=True): + ''' + Delete the group. + + delete_children: + If True, all child groups and tags will be deleted. + Otherwise they'll just be raised up one level. + ''' + if delete_children: + for child in self.children(): + child.delete() + else: + # Lift children + parent = self.group() + if parent is None: + # Since this group was a root, children become roots by removing the row. + self.photodb.cur.execute('DELETE FROM tag_group_rel WHERE groupid == ?', [self.id]) + else: + # Since this group was a child, its parent adopts all its children. + self.photodb.cur.execute( + 'UPDATE tag_group_rel SET groupid == ? WHERE groupid == ?', + [parent.id, self.id] + ) + # Note that this part comes after the deletion of children to prevent issues of recursion. + self.photodb.cur.execute('DELETE FROM groups WHERE id == ?', [self.id]) + self.photodb.cur.execute( + 'DELETE FROM tag_group_rel WHERE memberid == ? AND membertype == "group"', + [self.id] + ) + if commit: + self.photodb.sql.commit() + + def rename(self, new_name, commit=True): + ''' + Rename the group. Does not affect its tags. + ''' + new_name = normalize_tagname(new_name) + + try: + existing = self.photodb.get_group(new_name) + except NoSuchGroup: + pass + else: + raise GroupExists(new_name) + + self.photodb.cur.execute('UPDATE groups SET name = ? WHERE id == ?', [new_name, self.id]) + + self.name = new_name + if commit: + self.photodb.sql.commit() + + + +class Photo(ObjectBase): + ''' + A PhotoDB entry containing information about an image file. + Photo objects cannot exist without a corresponding PhotoDB object, because + Photos are not the actual image data, just the database entry. + ''' + def __init__(self, photodb, row_tuple): + width = row_tuple[SQL_PHOTO['width']] + height = row_tuple[SQL_PHOTO['height']] + if width is not None: + area = width * height + ratio = width / height + else: + area = None + ratio = None + + self.photodb = photodb + self.id = row_tuple[SQL_PHOTO['id']] + self.filepath = row_tuple[SQL_PHOTO['filepath']] + self.extension = row_tuple[SQL_PHOTO['extension']] + self.width = int_or_none(width) + self.height = int_or_none(height) + self.ratio = ratio + self.area = area + self.bytes = int(row_tuple[SQL_PHOTO['bytes']]) + self.created = int(row_tuple[SQL_PHOTO['created']]) + + def __repr__(self): + return 'Photo:{id}'.format(id=self.id) + + def apply_tag(self, tag, commit=True): + tag = self.photodb.get_tag(tag) + + if self.has_tag(tag): + return + + self.photodb.cur.execute('INSERT INTO photo_tag_rel VALUES(?, ?)', [self.id, tag.id]) + if commit: + self.photodb.sql.commit() + + def delete(self, commit=True): + ''' + Delete the Photo and its relation to any tags and albums. + ''' + self.photodb.cur.execute('DELETE FROM photos WHERE id == ?', [self.id]) + self.photodb.cur.execute('DELETE FROM photo_tag_rel WHERE photoid == ?', [self.id]) + self.photodb.cur.execute( + 'DELETE FROM tag_group_rel WHERE memberid == ? AND membertype == "tag"', + [self.id] + ) + if commit: + self.photodb.sql.commit() + + def tags(self): + ''' + Return the tags assigned to this Photo. + ''' + tagobjects = set() + generator = select( + self.photodb.sql, + 'SELECT * FROM photo_tag_rel WHERE photoid == ?', + [self.id] + ) + for tag in generator: + tagid = tag[SQL_PHOTOTAG['tagid']] + tagobj = self.photodb.get_tag(tagid=tagid) + tagobjects.add(tagobj) + return tagobjects + + def has_tag(self, tag): + tag = self.photodb.get_tag(tag) + + self.photodb.cur.execute( + 'SELECT * FROM photo_tag_rel WHERE photoid == ? AND tagid == ?', + [self.id, tag.id] + ) + fetch = self.photodb.cur.fetchone() + has_tag = fetch is not None + return has_tag + + def remove_tag(self, tag, commit=True): + tag = self.photodb.get_tag(tag) + + if not self.has_tag(tag): + return + + self.photodb.cur.execute('DELETE FROM photo_tag_rel VALUES(?, ?)', [self.id, tag.id]) + if commit: + self.photodb.sql.commit() + + +class Tag(ObjectBase, Groupable): + ''' + A Tag, which can be applied to Photos for organization. + ''' + _membertype = 'tag' + def __init__(self, photodb, row_tuple): + self.photodb = photodb + self.id = row_tuple[SQL_TAG['id']] + self.name = row_tuple[SQL_TAG['name']] + + def __repr__(self): + r = 'Tag:{name}'.format(name= self.name) + return r + + def add_synonym(self, synname, commit=True): + synname = normalize_tagname(synname) + + if synname == self.name: + raise ValueError('Cannot assign synonym to itself.') + + try: + tag = self.photodb.get_tag_by_name(synname) + except NoSuchTag: + pass + else: + raise TagExists(synname) + + self.photodb.cur.execute('INSERT INTO tag_synonyms VALUES(?, ?)', [synname, self.name]) + + if commit: + self.photodb.sql.commit() + + def delete(self, commit=True): + ''' + Delete a tag and its relationship with synonyms, groups, and photos. + ''' + self.photodb.cur.execute('DELETE FROM tags WHERE id == ?', [self.id]) + self.photodb.cur.execute('DELETE FROM photo_tag_rel WHERE tagid == ?', [self.id]) + self.photodb.cur.execute('DELETE FROM tag_synonyms WHERE mastername == ?', [self.name]) + self.photodb.cur.execute( + 'DELETE FROM tag_group_rel WHERE memberid == ? AND membertype == "tag"', + [self.id] + ) + if commit: + self.photodb.sql.commit() + + def delete_synonym(self, synname, commit=True): + ''' + Delete a synonym. + This will have no effect on photos or other synonyms because + they always resolve to the master tag before application. + ''' + synname = normalize_tagname(synname) + self.photodb.cur.execute('SELECT * FROM tag_synonyms WHERE name == ?', [synname]) + fetch = self.photodb.cur.fetchone() + if fetch is None: + raise NoSuchSynonym(synname) + + self.photodb.cur.execute('DELETE FROM tag_synonyms WHERE name == ?', [synname]) + if commit: + self.photodb.sql.commit() + + def get_synonyms(self): + self.photodb.cur.execute('SELECT name FROM tag_synonyms WHERE mastername == ?', [self.name]) + fetch = self.photodb.cur.fetchall() + fetch = [f[0] for f in fetch] + return fetch + + def rename(self, new_name, apply_to_synonyms=True, commit=True): + ''' + Rename the tag. Does not affect its relation to Photos. + ''' + new_name = normalize_tagname(new_name) + + try: + existing = self.photodb.get_tag(new_name) + except NoSuchTag: + pass + else: + raise TagExists(new_name) + + self.photodb.cur.execute('UPDATE tags SET name = ? WHERE id == ?', [new_name, self.id]) + if apply_to_synonyms: + self.photodb.cur.execute( + 'UPDATE tag_synonyms SET mastername = ? WHERE mastername = ?', + [new_name, self.name] + ) + + self.name = new_name + if commit: + self.photodb.sql.commit() + + +if __name__ == '__main__': + p = PhotoDB() \ No newline at end of file diff --git a/Phototagger/samples/bolts.jpg b/Etiquette/samples/bolts.jpg similarity index 100% rename from Phototagger/samples/bolts.jpg rename to Etiquette/samples/bolts.jpg diff --git a/Phototagger/samples/reddit.png b/Etiquette/samples/reddit.png similarity index 100% rename from Phototagger/samples/reddit.png rename to Etiquette/samples/reddit.png diff --git a/Phototagger/samples/train.jpg b/Etiquette/samples/train.jpg similarity index 100% rename from Phototagger/samples/train.jpg rename to Etiquette/samples/train.jpg diff --git a/Phototagger/test_phototagger.py b/Etiquette/test_phototagger.py similarity index 96% rename from Phototagger/test_phototagger.py rename to Etiquette/test_phototagger.py index d79fcc1..7a025d9 100644 --- a/Phototagger/test_phototagger.py +++ b/Etiquette/test_phototagger.py @@ -12,7 +12,6 @@ class PhotoDBTest(unittest.TestCase): pass def test_add_and_remove_photo(self): - self.setUp() photo1 = self.p.new_photo('samples\\train.jpg') self.assertEqual(len(photo1.id), self.p.id_length) @@ -42,8 +41,6 @@ class PhotoDBTest(unittest.TestCase): self.assertEqual(tag.name, 'one_two') def test_add_and_remove_synonym(self): - self.setUp() - # Add synonym giraffe = self.p.new_tag('giraffe') horse = self.p.new_tag_synonym('long horse', 'giraffe') @@ -65,7 +62,6 @@ class PhotoDBTest(unittest.TestCase): self.assertRaises(phototagger.NoSuchSynonym, self.p.remove_tag_synonym, 'blanc') def test_apply_photo_tag(self): - self.setUp() photo = self.p.new_photo('samples\\train.jpg') self.p.new_tag('vehicles') @@ -77,8 +73,6 @@ class PhotoDBTest(unittest.TestCase): self.assertFalse(status) def test_convert_tag_synonym(self): - self.setUp() - # Install tags and a synonym photo = self.p.new_photo('samples\\train.jpg') trains = self.p.new_tag('trains') @@ -115,7 +109,6 @@ class PhotoDBTest(unittest.TestCase): self.assertEqual(tags[0].id, trains_id) def test_generate_id(self): - self.setUp() i_photo = self.p.generate_id('photos') i_tag = self.p.generate_id('tags') self.assertRaises(ValueError, self.p.generate_id, 'other') @@ -127,21 +120,18 @@ class PhotoDBTest(unittest.TestCase): self.assertLess(int(i_photo), int(self.p.generate_id('photos'))) def test_get_photo_by_id(self): - self.setUp() photo = self.p.new_photo('samples\\train.jpg') photo2 = self.p.get_photo_by_id(photo.id) self.assertEqual(photo, photo2) def test_get_photo_by_path(self): - self.setUp() photo = self.p.new_photo('samples\\train.jpg') photo2 = self.p.get_photo_by_path(photo.filepath) self.assertEqual(photo, photo2) def test_get_photos_by_recent(self): - self.setUp() paths = ['train.jpg', 'bolts.jpg', 'reddit.png'] paths = ['samples\\' + path for path in paths] paths = [os.path.abspath(path) for path in paths] @@ -176,7 +166,6 @@ class PhotoDBTest(unittest.TestCase): self.assertEqual(tag1, tag2) def test_get_tags_by_photo(self): - self.setUp() photo = self.p.new_photo('samples\\train.jpg') tag = self.p.new_tag('vehicles') stat = self.p.apply_photo_tag(photo.id, tagname='vehicles') @@ -192,7 +181,6 @@ class PhotoDBTest(unittest.TestCase): self.assertRaises(phototagger.TagTooShort, self.p.new_tag, '!!??&&*') def test_photo_has_tag(self): - self.setUp() photo = self.p.new_photo('samples\\train.jpg') tag = self.p.new_tag('vehicles') self.p.apply_photo_tag(photo.id, tag.id) diff --git a/Javascript/opendir_image.js b/Javascript/opendir_image.js index e82f659..5e50368 100644 --- a/Javascript/opendir_image.js +++ b/Javascript/opendir_image.js @@ -363,6 +363,7 @@ function dump_urls() { textbox = document.createElement("textarea"); textbox.className = "urldumpbox"; + textbox.id = "url_dump_box"; workspace = document.getElementById("WORKSPACE"); workspace.appendChild(textbox); } diff --git a/Pathclass/pathclass.py b/Pathclass/pathclass.py index d7a97dd..8e5e5b1 100644 --- a/Pathclass/pathclass.py +++ b/Pathclass/pathclass.py @@ -18,6 +18,9 @@ class Path: def __hash__(self): return hash(self.absolute_path) + def __repr__(self): + return '{c}({path})'.format(c=self.__class__, path=self.absolute_path) + @property def basename(self): return os.path.basename(self.absolute_path) @@ -79,11 +82,12 @@ def get_path_casing(path): path = path.absolute_path (drive, subpath) = os.path.splitdrive(path) + drive = drive.upper() subpath = subpath.lstrip(os.sep) pattern = [glob_patternize(piece) for piece in subpath.split(os.sep)] pattern = os.sep.join(pattern) - pattern = drive.upper() + os.sep + pattern + pattern = drive + os.sep + pattern #print(pattern) try: return glob.glob(pattern)[0] diff --git a/Phototagger/phototagger.db b/Phototagger/phototagger.db deleted file mode 100644 index 478cb04..0000000 Binary files a/Phototagger/phototagger.db and /dev/null differ diff --git a/Phototagger/phototagger.py b/Phototagger/phototagger.py deleted file mode 100644 index 4a3812d..0000000 --- a/Phototagger/phototagger.py +++ /dev/null @@ -1,894 +0,0 @@ - - - - - -import datetime -import os -import PIL.Image -import random -import sqlite3 -import string -import warnings - -ID_LENGTH = 22 -VALID_TAG_CHARS = string.ascii_lowercase + string.digits + '_-' -MAX_TAG_NAME_LENGTH = 32 - -SQL_LASTID_COLUMNS = [ - 'table', - 'last_id', -] - -SQL_PHOTO_COLUMNS = [ - 'id', - 'filepath', - 'extension', - 'width', - 'height', - 'ratio', - 'area', - 'bytes', - 'created', -] - -SQL_PHOTOTAG_COLUMNS = [ - 'photoid', - 'tagid', -] - -SQL_SYN_COLUMNS = [ - 'name', - 'master', -] - -SQL_TAG_COLUMNS = [ - 'id', - 'name', -] - -SQL_LASTID = {key:index for (index, key) in enumerate(SQL_LASTID_COLUMNS)} -SQL_PHOTO = {key:index for (index, key) in enumerate(SQL_PHOTO_COLUMNS)} -SQL_PHOTOTAG = {key:index for (index, key) in enumerate(SQL_PHOTOTAG_COLUMNS)} -SQL_SYN = {key:index for (index, key) in enumerate(SQL_SYN_COLUMNS)} -SQL_TAG = {key:index for (index, key) in enumerate(SQL_TAG_COLUMNS)} - - -DB_INIT = ''' -CREATE TABLE IF NOT EXISTS photos( - id TEXT, - filepath TEXT, - extension TEXT, - width INT, - height INT, - ratio REAL, - area INT, - bytes INT, - created INT - ); -CREATE TABLE IF NOT EXISTS tags( - id TEXT, - name TEXT - ); -CREATE TABLE IF NOT EXISTS albums( - albumid TEXT, - photoid TEXT - ); -CREATE TABLE IF NOT EXISTS photo_tag_rel( - photoid TEXT, - tagid TEXT - ); -CREATE TABLE IF NOT EXISTS tag_synonyms( - name TEXT, - mastername TEXT - ); -CREATE TABLE IF NOT EXISTS id_numbers( - tab TEXT, - last_id TEXT - ); -CREATE INDEX IF NOT EXISTS index_photo_id on photos(id); -CREATE INDEX IF NOT EXISTS index_photo_path on photos(filepath); -CREATE INDEX IF NOT EXISTS index_photo_created on photos(created); - -CREATE INDEX IF NOT EXISTS index_tag_id on tags(id); -CREATE INDEX IF NOT EXISTS index_tag_name on tags(name); - -CREATE INDEX IF NOT EXISTS index_tagrel_photoid on photo_tag_rel(photoid); -CREATE INDEX IF NOT EXISTS index_tagrel_tagid on photo_tag_rel(tagid); - -CREATE INDEX IF NOT EXISTS index_tagsyn_name on tag_synonyms(name); -''' - -def basex(number, base, alphabet='0123456789abcdefghijklmnopqrstuvwxyz'): - ''' - Converts an integer to a different base string. - Based on http://stackoverflow.com/a/1181922/5430534 - ''' - if base > len(alphabet): - raise Exception('alphabet %s does not support base %d' % ( - repr(alphabet), base)) - alphabet = alphabet[:base] - if not isinstance(number, (int, str)): - raise TypeError('number must be an integer') - number = int(number) - based = '' - sign = '' - if number < 0: - sign = '-' - number = -number - if 0 <= number < len(alphabet): - return sign + alphabet[number] - while number != 0: - number, i = divmod(number, len(alphabet)) - based = alphabet[i] + based - return sign + based - -def fetch_generator(cursor): - while True: - fetch = cursor.fetchone() - if fetch is None: - break - yield fetch - -def getnow(timestamp=True): - ''' - Return the current UTC timestamp or datetime object. - ''' - now = datetime.datetime.now(datetime.timezone.utc) - if timestamp: - return now.timestamp() - return now - -def is_xor(*args): - ''' - Return True if and only if one arg is truthy. - ''' - return [bool(a) for a in args].count(True) == 1 - -def min_max_query_builder(name, comparator, value): - return ' '.join([name, comparator, value]) - -def normalize_tagname(tagname): - ''' - Tag names can only consist of lowercase letters, underscores, - and hyphens. The given tagname is lowercased, gets its spaces - replaced by underscores, and is stripped of any not-whitelisted - characters. - ''' - tagname = tagname.lower() - tagname = tagname.replace(' ', '_') - tagname = (c for c in tagname if c in VALID_TAG_CHARS) - tagname = ''.join(tagname) - if len(tagname) == 0: - raise TagTooShort(tagname) - if len(tagname) > MAX_TAG_NAME_LENGTH: - raise TagTooLong(tagname) - return tagname - -def not_implemented(function): - ''' - Decorator for keeping track of which functions still need to be filled out. - ''' - warnings.warn('%s is not implemented' % function.__name__) - return function - -def raise_nosuchtag(tagid=None, tagname=None, comment=''): - if tagid is not None: - message = 'ID: %s. %s' % (tagid, comment) - elif tagname is not None: - message = 'Name: %s. %s' % (tagname, comment) - raise NoSuchTag(message) - -class NoSuchPhoto(Exception): - pass - -class NoSuchSynonym(Exception): - pass - -class NoSuchTag(Exception): - pass - -class PhotoExists(Exception): - pass - -class TagExists(Exception): - pass - -class TagTooLong(Exception): - pass - -class TagTooShort(Exception): - pass - -class XORException(Exception): - pass - -class PhotoDB: - ''' - This class represents an SQLite3 database containing the following tables: - - photos: - Rows represent image files on the local disk. - Entries contain a unique ID, the image's filepath, and metadata - like dimensions and filesize. - - tags: - Rows represent labels, which can be applied to an arbitrary number of - photos. Photos may be selected by which tags they contain. - Entries contain a unique ID and a name. - - albums: - Rows represent the inclusion of a photo in an album - - photo_tag_rel: - Rows represent a Photo's ownership of a particular Tag. - - tag_synonyms: - Rows represent relationships between two tag names, so that they both - resolve to the same Tag object when selected. Entries contain the - subordinate name and master name. - The master name MUST also exist in the `tags` table. - If a new synonym is created referring to another synoym, the master name - will be resolved and used instead, so a synonym is never in the master - column. - Tag objects will ALWAYS represent the master tag. - - Note that the entries in this table do not contain ID numbers. - The rationale here is that "coco" is a synonym for "chocolate" regardless - of the "chocolate" tag's ID, and that if a tag is renamed, its synonyms - do not necessarily follow. - The `rename_tag` method includes a parameter `apply_to_synonyms` if you do - want them to follow. - ''' - def __init__(self, databasename='phototagger.db', id_length=None): - if id_length is None: - self.id_length = ID_LENGTH - self.databasename = databasename - self.sql = sqlite3.connect(databasename) - self.cur = self.sql.cursor() - statements = DB_INIT.split(';') - for statement in statements: - self.cur.execute(statement) - self._last_ids = {} - - def __repr__(self): - return 'PhotoDB(databasename={dbname})'.format(dbname=repr(self.databasename)) - - def apply_photo_tag(self, photoid, tagid=None, tagname=None, commit=True): - ''' - Apply a tag to a photo. `tag` may be the name of the tag or a Tag - object from the same PhotoDB. - - `tag` may NOT be the tag's ID, since we can't tell if a given string is - an ID or a name. - - Returns True if the tag was applied, False if the photo already had this tag. - Raises NoSuchTag and NoSuchPhoto as appropriate. - ''' - tag = self.get_tag(tagid=tagid, tagname=tagname, resolve_synonyms=True) - - self.cur.execute('SELECT * FROM photo_tag_rel WHERE photoid == ? AND tagid == ?', [photoid, tag.id]) - if self.cur.fetchone() is not None: - return False - - self.cur.execute('INSERT INTO photo_tag_rel VALUES(?, ?)', [photoid, tag.id]) - if commit: - self.sql.commit() - return True - - def convert_tag_to_synonym(self, oldtagname, mastertagname): - ''' - Convert an independent tag into a synonym for a different independent tag. - All photos which possess the current tag will have it replaced - with the master tag. All synonyms of the old tag will point to the new tag. - - Good for when two tags need to be merged under a single name. - ''' - oldtagname = normalize_tagname(oldtagname) - mastertagname = normalize_tagname(mastertagname) - - oldtag = self.get_tag_by_name(oldtagname, resolve_synonyms=False) - if oldtag is None: - raise NoSuchTag(oldtagname) - - mastertag = self.get_tag_by_name(mastertagname, resolve_synonyms=False) - if mastertag is None: - raise NoSuchTag(mastertagname) - - # Migrate the old tag's synonyms to the new one - # UPDATE is safe for this operation because there is no chance of duplicates. - self.cur.execute('UPDATE tag_synonyms SET mastername = ? WHERE mastername == ?', [mastertagname, oldtagname]) - - # Iterate over all photos with the old tag, and relate them to the new tag - # if they aren't already. - temp_cur = self.sql.cursor() - temp_cur.execute('SELECT * FROM photo_tag_rel WHERE tagid == ?', [oldtag.id]) - for relationship in fetch_generator(temp_cur): - photoid = relationship[SQL_PHOTOTAG['photoid']] - self.cur.execute('SELECT * FROM photo_tag_rel WHERE tagid == ?', [mastertag.id]) - if self.cur.fetchone() is not None: - continue - self.cur.execute('INSERT INTO photo_tag_rel VALUES(?, ?)', [photoid, mastertag.id]) - - # Then delete the relationships with the old tag - self.cur.execute('DELETE FROM photo_tag_rel WHERE tagid == ?', [oldtag.id]) - self.cur.execute('DELETE FROM tags WHERE id == ?', [oldtag.id]) - - # Enjoy your new life as a monk. - self.new_tag_synonym(oldtag.name, mastertag.name, commit=False) - self.sql.commit() - - def delete_photo(self, photoid): - ''' - Delete a photo and its relation to any tags and albums. - ''' - photo = self.get_photo_by_id(photoid) - if photo is None: - raise NoSuchPhoto(photoid) - self.cur.execute('DELETE FROM photos WHERE id == ?', [photoid]) - self.cur.execute('DELETE FROM photo_tag_rel WHERE photoid == ?', [photoid]) - self.sql.commit() - - def delete_tag(self, tagid=None, tagname=None): - ''' - Delete a tag, its synonyms, and its relation to any photos. - ''' - - tag = self.get_tag(tagid=tagid, tagname=tagname, resolve_synonyms=False) - - if tag is None: - message = 'Is it a synonym?' - raise_nosuchtag(tagid=tagid, tagname=tagname, comment=message) - - self.cur.execute('DELETE FROM tags WHERE id == ?', [tag.id]) - self.cur.execute('DELETE FROM photo_tag_rel WHERE tagid == ?', [tag.id]) - self.cur.execute('DELETE FROM tag_synonyms WHERE mastername == ?', [tag.name]) - self.sql.commit() - - def delete_tag_synonym(self, tagname): - ''' - Delete a tag synonym. - This will have no effect on photos or other synonyms because - they always resolve to the master tag before application. - ''' - tagname = normalize_tagname(tagname) - self.cur.execute('SELECT * FROM tag_synonyms WHERE name == ?', [tagname]) - fetch = self.cur.fetchone() - if fetch is None: - raise NoSuchSynonym(tagname) - - self.cur.execute('DELETE FROM tag_synonyms WHERE name == ?', [tagname]) - self.sql.commit() - - def generate_id(self, table): - ''' - Create a new ID number that is unique to the given table. - Note that this method does not commit the database. We'll wait for that - to happen in whoever is calling us, so we know the ID is actually used. - ''' - table = table.lower() - if table not in ['photos', 'tags']: - raise ValueError('Invalid table requested: %s.', table) - - do_update = False - if table in self._last_ids: - # Use cache value - new_id = self._last_ids[table] + 1 - do_update = True - else: - self.cur.execute('SELECT * FROM id_numbers WHERE tab == ?', [table]) - fetch = self.cur.fetchone() - if fetch is None: - # Register new value - new_id = 1 - else: - # Use database value - new_id = int(fetch[SQL_LASTID['last_id']]) + 1 - do_update = True - - new_id_s = str(new_id).rjust(self.id_length, '0') - if do_update: - self.cur.execute('UPDATE id_numbers SET last_id = ? WHERE tab == ?', [new_id_s, table]) - else: - self.cur.execute('INSERT INTO id_numbers VALUES(?, ?)', [table, new_id_s]) - self._last_ids[table] = new_id - return new_id_s - - @not_implemented - def get_album_by_id(self, albumid): - return - - def get_photo_by_id(self, photoid): - ''' - Return this Photo object, or None if it does not exist. - ''' - self.cur.execute('SELECT * FROM photos WHERE id == ?', [photoid]) - photo = self.cur.fetchone() - if photo is None: - return None - photo = self.tuple_to_photo(photo) - return photo - - def get_photo_by_path(self, filepath): - ''' - Return this Photo object, or None if it does not exist. - ''' - filepath = os.path.abspath(filepath) - self.cur.execute('SELECT * FROM photos WHERE filepath == ?', [filepath]) - photo = self.cur.fetchone() - if photo is None: - return None - photo = self.tuple_to_photo(photo) - return photo - - def get_photos_by_recent(self, count=None): - ''' - Yield photo objects in order of creation time. - ''' - if count is not None and count <= 0: - return - # We're going to use a second cursor because the first one may - # get used for something else, deactivating this query. - temp_cur = self.sql.cursor() - temp_cur.execute('SELECT * FROM photos ORDER BY created DESC') - while True: - f = temp_cur.fetchone() - if f is None: - return - photo = self.tuple_to_photo(f) - - yield photo - - if count is None: - continue - count -= 1 - if count <= 0: - return - - @not_implemented - def get_photos_by_search( - self, - extension=None, - maximums={}, - minimums={}, - tag_musts=None, - tag_mays=None, - tag_forbids=None, - tag_forbid_unspecified=False, - ): - ''' - Given one or multiple tags, yield photos possessing those tags. - - Parameters: - extension : - A string or list of strings of acceptable file extensions. - - maximums : - A dictionary, where the key is an attribute of the photo, - (area, bytes, created, height, id, or width) - and the value is the maximum desired value for that field. - - minimums : - A dictionary like `maximums` where the value is the minimum - desired value for that field. - - tag_musts : - A list of tag names or Tag objects. - Photos MUST have ALL tags in this list. - - tag_mays : - A list of tag names or Tag objects. - If `forbid_unspecified` is True, then Photos MUST have AT LEAST ONE tag in this list. - If `forbid_unspecified` is False, then Photos MAY or MAY NOT have ANY tag in this list. - - tag_forbids : - A list of tag names or Tag objects. - Photos MUST NOT have ANY tag in the list. - - tag_forbid_unspecified : - True or False. - If False, Photos need only comply with the `tag_musts`. - If True, Photos need to comply with both `tag_musts` and `tag_mays`. - ''' - maximums = {key:int(val) for (key, val) in maximums.items()} - minimums = {key:int(val) for (key, val) in minimums.items()} - - # Raise for cases where the minimum > maximum - for (maxkey, maxval) in maximums.items(): - if maxkey not in minimums: - continue - minval = minimums[maxkey] - if minval > maxval: - raise ValueError('Impossible min-max for %s' % maxkey) - - conditions = [] - minmaxers = {'<=': maximums, '>=': minimums} - - # Convert the min-max parameters into query strings - for (comparator, minmaxer) in minmaxers.items(): - for (field, value) in minmaxer.items(): - if field not in Photo.int_properties: - raise ValueError('Unknown Photo property: %s' % field) - - value = str(value) - query = min_max_query_builder(field, comparator, value) - conditions.append(query) - - if extension is not None: - if isinstance(extension, str): - extension = [extension] - - # Normalize to prevent injections - extension = [normalize_tagname(e) for e in extension] - extension = ['extension == "%s"' % e for e in extension] - extension = ' OR '.join(extension) - extension = '(%s)' % extension - conditions.append(extension) - - def setify(l): - return set(self.get_tag_by_name(t) for t in l) if l else set() - tag_musts = setify(tag_musts) - tag_mays = setify(tag_mays) - tag_forbids = setify(tag_forbids) - - base = '%s EXISTS (SELECT 1 FROM photo_tag_rel WHERE photo_tag_rel.photoid == photos.id AND photo_tag_rel.tagid %s %s)' - for tag in tag_musts: - query = base % ('', '==', '"%s"' % tag.id) - conditions.append(query) - - if tag_forbid_unspecified and len(tag_mays) > 0: - acceptable = tag_mays.union(tag_musts) - acceptable = ['"%s"' % t.id for t in acceptable] - acceptable = ', '.join(acceptable) - query = base % ('NOT', 'NOT IN', '(%s)' % acceptable) - conditions.append(query) - - for tag in tag_forbids: - query = base % ('NOT', '==', '"%s"' % tag.id) - conditions.append(query) - - if len(conditions) == 0: - raise ValueError('No search query provided') - - conditions = [query for query in conditions if query is not None] - conditions = ['(%s)' % c for c in conditions] - conditions = ' AND '.join(conditions) - conditions = 'WHERE %s' % conditions - - - query = 'SELECT * FROM photos %s' % conditions - print(query) - temp_cur = self.sql.cursor() - temp_cur.execute(query) - acceptable_tags = tag_musts.union(tag_mays) - while True: - fetch = temp_cur.fetchone() - if fetch is None: - break - - photo = self.tuple_to_photo(fetch) - - # if any(forbid in photo.tags for forbid in tag_forbids): - # print('Forbidden') - # continue - - # if tag_forbid_unspecified: - # if any(tag not in acceptable_tags for tag in photo.tags): - # print('Forbid unspecified') - # continue - - # if any(must not in photo.tags for must in tag_musts): - # print('No must') - # continue - - yield photo - - - def get_tag(self, tagid=None, tagname=None, resolve_synonyms=True): - ''' - Redirect to get_tag_by_id or get_tag_by_name after xor-checking the parameters. - ''' - if not is_xor(tagid, tagname): - raise XORException('One and only one of `tagid`, `tagname` can be passed.') - - if tagid is not None: - return self.get_tag_by_id(tagid) - elif tagname is not None: - return self.get_tag_by_name(tagname, resolve_synonyms=resolve_synonyms) - raise_nosuchtag(tagid=tagid, tagname=tagname) - - def get_tag_by_id(self, tagid): - self.cur.execute('SELECT * FROM tags WHERE id == ?', [tagid]) - tag = self.cur.fetchone() - if tag is None: - return raise_nosuchtag(tagid=tagid) - tag = self.tuple_to_tag(tag) - return tag - - def get_tag_by_name(self, tagname, resolve_synonyms=True): - if isinstance(tagname, Tag): - return tagname - - tagname = normalize_tagname(tagname) - - if resolve_synonyms is True: - self.cur.execute('SELECT * FROM tag_synonyms WHERE name == ?', [tagname]) - fetch = self.cur.fetchone() - if fetch is not None: - mastertagname = fetch[SQL_SYN['master']] - tag = self.get_tag_by_name(mastertagname) - return tag - - self.cur.execute('SELECT * FROM tags WHERE name == ?', [tagname]) - fetch = self.cur.fetchone() - if fetch is None: - raise_nosuchtag(tagname=tagname) - - tag = self.tuple_to_tag(fetch) - return tag - - def get_tags_by_photo(self, photoid): - ''' - Return the tags assigned to the given photo. - ''' - temp_cur = self.sql.cursor() - temp_cur.execute('SELECT * FROM photo_tag_rel WHERE photoid == ?', [photoid]) - tags = fetch_generator(temp_cur) - tagobjects = set() - for tag in tags: - tagid = tag[SQL_PHOTOTAG['tagid']] - tagobj = self.get_tag_by_id(tagid) - tagobjects.add(tagobj) - return tagobjects - - def new_photo(self, filename, tags=None, allow_duplicates=False): - ''' - Given a filepath, determine its attributes and create a new Photo object in the - database. Tags may be applied now or later. - - If `allow_duplicates` is False, we will first check the database for any files - with the same path and raise PhotoExists if found. - - Returns the Photo object. - ''' - filename = os.path.abspath(filename) - if not allow_duplicates: - existing = self.get_photo_by_path(filename) - if existing is not None: - raise PhotoExists(filename, existing) - - # I want the caller to receive any exceptions this raises. - image = PIL.Image.open(filename) - - extension = os.path.splitext(filename)[1] - extension = extension.replace('.', '') - extension = normalize_tagname(extension) - (width, height) = image.size - area = width * height - ratio = width / height - bytes = os.path.getsize(filename) - created = int(getnow()) - photoid = self.generate_id('photos') - - data = [None] * len(SQL_PHOTO_COLUMNS) - data[SQL_PHOTO['id']] = photoid - data[SQL_PHOTO['filepath']] = filename - data[SQL_PHOTO['extension']] = extension - data[SQL_PHOTO['width']] = width - data[SQL_PHOTO['height']] = height - data[SQL_PHOTO['area']] = area - data[SQL_PHOTO['ratio']] = ratio - data[SQL_PHOTO['bytes']] = bytes - data[SQL_PHOTO['created']] = created - photo = self.tuple_to_photo(data) - - self.cur.execute('INSERT INTO photos VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)', data) - - tags = tags or [] - for tag in tags: - try: - self.apply_photo_tag(photoid, tagname=tag, commit=False) - except NoSuchTag: - self.sql.rollback() - raise - self.sql.commit() - image.close() - return photo - - def new_tag(self, tagname): - ''' - Register a new tag in the database and return the Tag object. - ''' - tagname = normalize_tagname(tagname) - try: - self.get_tag_by_name(tagname) - TagExists(tagname) - except NoSuchTag: - pass - tagid = self.generate_id('tags') - self.cur.execute('INSERT INTO tags VALUES(?, ?)', [tagid, tagname]) - self.sql.commit() - tag = self.tuple_to_tag([tagid, tagname]) - return tag - - def new_tag_synonym(self, tagname, mastertagname, commit=True): - ''' - Register a new synonym for an existing tag. - ''' - tagname = normalize_tagname(tagname) - mastertagname = normalize_tagname(mastertagname) - - if tagname == mastertagname: - raise ValueError('Cannot assign synonym to itself.') - - # We leave resolve_synonyms as True, so that if this function returns - # anything, we know the given tagname is already a synonym or master. - tag = self.get_tag_by_name(tagname, resolve_synonyms=True) - if tag is not None: - raise TagExists(tagname) - - mastertag = self.get_tag_by_name(mastertagname, resolve_synonyms=True) - if mastertag is None: - raise NoSuchTag(mastertagname) - - self.cur.execute('INSERT INTO tag_synonyms VALUES(?, ?)', [tagname, mastertag.name]) - - if commit: - self.sql.commit() - - return mastertag - - def photo_has_tag(self, photoid, tagid=None, tagname=None): - tag = self.get_tag(tagid=tagid, tagname=tagname, resolve_synonyms=True) - if tag is None: - raise_nosuchtag(tagid=tagid, tagname=tagname) - - exe = self.cur.execute - exe('SELECT * FROM photo_tag_rel WHERE photoid == ? AND tagid == ?', [photoid, tag.id]) - fetch = self.cur.fetchone() - has_tag = fetch is not None - return has_tag - - @not_implemented - def rename_tag(self, tagname, newname, apply_to_synonyms): - pass - - def tuple_to_photo(self, tu): - ''' - Given a tuple like the ones from an sqlite query, - create a Photo object. - ''' - photoid = tu[SQL_PHOTO['id']] - tags = self.get_tags_by_photo(photoid) - - photo = Photo( - photodb = self, - photoid = photoid, - filepath = tu[SQL_PHOTO['filepath']], - extension = tu[SQL_PHOTO['extension']], - width = tu[SQL_PHOTO['width']], - height = tu[SQL_PHOTO['height']], - created = tu[SQL_PHOTO['created']], - bytes = tu[SQL_PHOTO['bytes']], - tags = tags, - ) - return photo - - def tuple_to_tag(self, tu): - ''' - Given a tuple like the ones from an sqlite query, - create a Tag object. - ''' - tag = Tag( - photodb = self, - tagid = tu[SQL_TAG['id']], - name = tu[SQL_TAG['name']] - ) - return tag - -class Photo: - ''' - This class represents a PhotoDB entry containing information about an image file. - Photo objects cannot exist without a corresponding PhotoDB object, because - Photos are not the actual files, just the database entry. - ''' - int_properties = set(['area', 'bytes', 'created', 'height', 'id', 'width']) - def __init__( - self, - photodb, - photoid, - filepath, - extension, - width, - height, - bytes, - created, - tags=None, - ): - if tags is None: - tags = [] - - self.photodb = photodb - self.id = photoid - self.filepath = filepath - self.extension = extension - self.width = int(width) - self.height = int(height) - self.ratio = self.width / self.height - self.area = self.width * self.height - self.bytes = int(bytes) - self.created = int(created) - self.tags = tags - - def __eq__(self, other): - if not isinstance(other, Photo): - return False - return self.id == other.id - - def __hash__(self): - return hash(self.id) - - def __repr__(self): - r = ('Photo(photodb={photodb}, ', - 'photoid={photoid}, ', - 'filepath={filepath}, ', - 'extension={extension}, ', - 'width={width}, ', - 'height={height}, ', - 'created={created})' - ) - r = ''.join(r) - r = r.format( - photodb = repr(self.photodb), - photoid = repr(self.id), - filepath = repr(self.filepath), - extension = repr(self.extension), - width = repr(self.width), - height = repr(self.height), - bytes = repr(self.bytes), - area = repr(self.area), - created = repr(self.created), - ) - return r - - def __str__(self): - return 'Photo: %s' % self.id - - def apply_photo_tag(self, tagname): - return self.photodb.apply_photo_tag(self.id, tagname=tagname, commit=True) - - def photo_has_tag(self, tagname): - return self.photodb.photo_has_tag(self.id, tagname=tagname) - -class Tag: - ''' - This class represents a Tag, which can be applied to Photos for - organization. - ''' - def __init__(self, photodb, tagid, name): - self.photodb = photodb - self.id = tagid - self.name = name - - def __eq__(self, other): - if not isinstance(other, Tag): - return False - return self.id == other.id - - def __hash__(self): - return hash(self.id) - - def __repr__(self): - r = 'Tag(photodb={photodb}, name={name}, tagid={tagid})' - r = r.format( - photodb = repr(self.photodb), - name = repr(self.name), - tagid = repr(self.id), - ) - return r - - def __str__(self): - return 'Tag: %s : %s' % (self.id, self.name) - -if __name__ == '__main__': - p = PhotoDB() \ No newline at end of file diff --git a/RateMeter/speedtest.py b/RateMeter/speedtest.py index 456a324..8223c76 100644 --- a/RateMeter/speedtest.py +++ b/RateMeter/speedtest.py @@ -19,25 +19,28 @@ g.total = 0 g.start = None g.last = time.time() -def callback_progress(bytes_downloaded, bytes_total): - if g.start is None: - g.start = time.time() - percent = 100 * bytes_downloaded / bytes_total - percent = '%07.3f%%:' % percent - chunk = bytes_downloaded - g.total - g.total = bytes_downloaded - METER.digest(chunk) - METER_2.digest(chunk) - now = round(time.time(), 1) - if now > g.last or (bytes_downloaded >= bytes_total): - g.last = now - percent = percent.rjust(9, ' ') - rate = bytestring.bytestring(METER.report()[2]).rjust(15, ' ') - rate2 = bytestring.bytestring(METER_2.report()[2]).rjust(15, ' ') - elapsed = str(round(now-g.start, 1)).rjust(10, ' ') - print(percent, rate, rate2, elapsed, end='\r', flush=True) - #print(METER.report(), METER_2.report()) +class P: + def __init__(self, bytes_total): + self.bytes_total = bytes_total + def step(self, bytes_downloaded): + if g.start is None: + g.start = time.time() + percent = 100 * bytes_downloaded / self.bytes_total + percent = '%07.3f%%:' % percent + chunk = bytes_downloaded - g.total + g.total = bytes_downloaded + METER.digest(chunk) + METER_2.digest(chunk) + now = round(time.time(), 1) + if now > g.last or (bytes_downloaded >= self.bytes_total): + g.last = now + percent = percent.rjust(9, ' ') + rate = bytestring.bytestring(METER.report()[2]).rjust(15, ' ') + rate2 = bytestring.bytestring(METER_2.report()[2]).rjust(15, ' ') + elapsed = str(round(now-g.start, 1)).rjust(10, ' ') + print(percent, rate, rate2, elapsed, end='\r', flush=True) + #print(METER.report(), METER_2.report()) print(URL) print('Progress'.rjust(9, ' '), 'bps over 5s'.rjust(15, ' '), 'bps overall'.rjust(15, ' '), 'elapsed'.rjust(10, ' ')) -downloady.download_file(URL, 'nul', callback_progress=callback_progress) \ No newline at end of file +downloady.download_file(URL, 'nul', callback_progress=P) \ No newline at end of file diff --git a/Ratelimiter/ratelimiter.py b/Ratelimiter/ratelimiter.py index f7953c1..a2ccc35 100644 --- a/Ratelimiter/ratelimiter.py +++ b/Ratelimiter/ratelimiter.py @@ -2,13 +2,13 @@ import time class Ratelimiter: - def __init__(self, allowance_per_period, period=1, operation_cost=1, mode='sleep'): + def __init__(self, allowance, period=1, operation_cost=1, mode='sleep'): ''' - allowance_per_period: + allowance: Our spending balance per `period` seconds. period: - The number of seconds over which we can perform `allowance_per_period` operations. + The number of seconds over which we can perform `allowance` operations. operation_cost: The default amount to remove from our balance after each operation. @@ -26,7 +26,7 @@ class Ratelimiter: if mode not in ('sleep', 'reject'): raise ValueError('Invalid mode %s' % repr(mode)) - self.allowance_per_period = allowance_per_period + self.allowance = allowance self.period = period self.operation_cost = operation_cost self.mode = mode @@ -36,7 +36,7 @@ class Ratelimiter: @property def gain_rate(self): - return self.allowance_per_period / self.period + return self.allowance / self.period def limit(self, cost=None): ''' @@ -47,7 +47,7 @@ class Ratelimiter: time_diff = time.time() - self.last_operation self.balance += time_diff * self.gain_rate - self.balance = min(self.balance, self.allowance_per_period) + self.balance = min(self.balance, self.allowance) if self.balance >= cost: self.balance -= cost diff --git a/RenameMap/rename_map.py b/RenameMap/rename_map.py new file mode 100644 index 0000000..df72ac1 --- /dev/null +++ b/RenameMap/rename_map.py @@ -0,0 +1,65 @@ +import os +import tkinter + + +class Application: + def __init__(self): + self.windowtitle = 'Rename Map' + + self.t = tkinter.Tk() + self.t.title(self.windowtitle) + w = 800 + h = 525 + screenwidth = self.t.winfo_screenwidth() + screenheight = self.t.winfo_screenheight() + windowwidth = w + windowheight = h + windowx = (screenwidth-windowwidth) / 2 + windowy = ((screenheight-windowheight) / 2) - 27 + self.geometrystring = '%dx%d+%d+%d' % (windowwidth, windowheight, windowx, windowy) + self.t.geometry(self.geometrystring) + + self.create_mainframe() + self.build_gui_mainmenu() + + def annihilate(self): + self.mainframe.destroy() + self.create_mainframe() + + def build_gui_mainmenu(self): + self.annihilate() + font = ('Consolas', 15) + + self.entry_left = tkinter.Entry(self.mainframe, font=font) + self.entry_right = tkinter.Entry(self.mainframe, font=font) + self.entry_left.grid(row=0, column=0, sticky='ew') + self.entry_right.grid(row=1, column=0, sticky='ew') + self.entry_left.insert(0, 'Left path') + self.entry_right.insert(0, 'Right path') + + button_start = tkinter.Button(self.mainframe, text='Go', command=self.build_gui_mapper) + button_start.grid(row=2, column=0, sticky='ew') + self.mainframe.rowconfigure(0, weight=1) + self.mainframe.rowconfigure(1, weight=1) + self.mainframe.columnconfigure(0, weight=1) + + def build_gui_mapper(self): + left_path = self.entry_left.get() + right_path = self.entry_right.get() + self.annihilate() + left_files = os.listdir(left_path) + right_files = os.listdir(right_path) + print(left_files) + print(right_files) + + + def create_mainframe(self): + self.mainframe = tkinter.Frame(self.t) + self.mainframe.pack(fill='both', expand=True) + + def mainloop(self): + self.t.mainloop() + +if __name__ == '__main__': + a = Application() + a.mainloop() \ No newline at end of file diff --git a/ServerReference/simpleserver.py b/ServerReference/simpleserver.py index 28ae196..477fb52 100644 --- a/ServerReference/simpleserver.py +++ b/ServerReference/simpleserver.py @@ -48,6 +48,7 @@ class Path(pathclass.Path): return any(self in okay for okay in OKAY_PATHS) def anchor(self, display_name=None): + self.correct_case() if display_name is None: display_name = self.basename @@ -59,11 +60,15 @@ class Path(pathclass.Path): icon = '\U0001F48E' #print('anchor', path) - a = '{icon} {display}'.format( + if display_name.endswith('.placeholder'): + a = '{icon} {display}' + else: + a = '{icon} {display}' + a = a.format( full=self.url_path, icon=icon, display=display_name, - ) + ) return a def table_row(self, display_name=None, shaded=False): diff --git a/SpinalTap/spinal.py b/SpinalTap/spinal.py index ea1cb3c..bb4912b 100644 --- a/SpinalTap/spinal.py +++ b/SpinalTap/spinal.py @@ -189,7 +189,6 @@ def copy_dir( Returns: [destination path, number of bytes written to destination] (Written bytes is 0 if all files already existed.) ''' - # Prepare parameters if not is_xor(destination, destination_new_root): m = 'One and only one of `destination` and ' @@ -199,6 +198,7 @@ def copy_dir( source = str_to_fp(source) if destination_new_root is not None: + source.correct_case() destination = new_root(source, destination_new_root) destination = str_to_fp(destination) @@ -279,10 +279,10 @@ def copy_file( destination_new_root=None, bytes_per_second=None, callback=None, + callback_permission_denied=None, callback_verbose=None, dry_run=False, overwrite_old=True, - callback_permission_denied=None, ): ''' Copy a file from one place to another. @@ -412,8 +412,9 @@ def copy_file( callback(destination, written_bytes, source_bytes) # Fin - callback_verbose('Closing handles.') + callback_verbose('Closing source handle.') source_file.close() + callback_verbose('Closing dest handle.') destination_file.close() callback_verbose('Copying metadata') shutil.copystat(source.absolute_path, destination.absolute_path) @@ -461,7 +462,7 @@ def limiter_or_none(value): if isinstance(value, ratelimiter.Ratelimiter): limiter = value elif value is not None: - limiter = ratelimiter.Ratelimiter(allowance_per_period=value, period=1) + limiter = ratelimiter.Ratelimiter(allowance=value, period=1) else: limiter = None return limiter diff --git a/ThreadedDL/threaded_dl.py b/ThreadedDL/threaded_dl.py index fcf6263..bc0b6a9 100644 --- a/ThreadedDL/threaded_dl.py +++ b/ThreadedDL/threaded_dl.py @@ -30,16 +30,18 @@ def listget(li, index, fallback): except IndexError: return fallback -def threaded_dl(urls, thread_count=4): +def threaded_dl(urls, thread_count, prefix=None): threads = [] prefix_digits = len(str(len(urls))) - prefix_text = '%0{digits}d_'.format(digits=prefix_digits) + if prefix is None: + prefix = now = int(time.time()) + prefix_text = '{prefix}_{{index:0{digits}d}}_'.format(prefix=prefix, digits=prefix_digits) for (index, url) in enumerate(urls): while len(threads) == thread_count: threads = remove_finished(threads) time.sleep(0.1) - prefix = prefix_text % index + prefix = prefix_text.format(index=index) t = threading.Thread(target=download_thread, args=[url, prefix]) t.daemon = True threads.append(t) @@ -47,6 +49,7 @@ def threaded_dl(urls, thread_count=4): while len(threads) > 0: threads = remove_finished(threads) + print('%d threads remaining\r' % len(threads), end='', flush=True) time.sleep(0.1) def main(): @@ -55,11 +58,13 @@ def main(): f = open(filename, 'r') with f: urls = f.read() - urls = urls.split() + urls = urls.split('\n') else: urls = clipext.resolve(filename) - urls = urls.split() - threaded_dl(urls) + urls = urls.split('\n') + thread_count = int(listget(sys.argv, 2, 4)) + prefix = listget(sys.argv, 3, None) + threaded_dl(urls, thread_count=thread_count, prefix=prefix) if __name__ == '__main__': main() \ No newline at end of file