else

.gitignore

@@ -1,5 +1,7 @@
 AwfulCrateBox/
 Classifieds/
+Toddo/toddo.db
+Meal/meal.db
 
 # Windows image file caches
 Thumbs.db

DynamicColumnFormatter/dynamic_column_formatter.py

@@ -0,0 +1,102 @@
+import shlex
+
+HEADER_TEXT = 'number name age address misc. "blank column?"'
+BODY_TEXT = '''
+4 "John Smith" 26 "123 north street"
+17 "Jenny Smith" 8 "123 north street"
+55 "Veronica Dove" 20 "456 west street"
+77 "Austin Texas" 33 "789 south avenue"
+89 "Mister Super Long Name" 123 "planet earth" "he's really old"
+120 "Bill" "" "" "Deceased"
+999 "Nine Nine" 999 "999 ninth boulevard" "favorite number is 9"
+'''
+BODY_TEXT = BODY_TEXT.strip()
+DELIMITER = ' | '
+
+output_text = ''
+# Keep track of the longest entry in each column, to determine
+# how wide we should make them.
+# Also, look for numerical columns so we can right-justify them.
+header = shlex.split(HEADER_TEXT)
+column_widths = {index:len(item) for (index, item) in enumerate(header)}
+column_types = {}
+document_lines = [shlex.split(line) for line in BODY_TEXT.splitlines()]
+for line in document_lines:
+    for (index, word) in enumerate(line):
+        current_width = column_widths.get(index, 0)
+        column_widths[index] = max(len(word), current_width)
+
+        try:
+            float(word)
+        except ValueError:
+            # It only takes one failure to make the whole column
+            # string type.
+            if word != "":
+                column_types[index] = 's'
+
+# Move the dictionary into a list where the index is the column
+# number, and the value is how wide it should be.
+column_widths = list(column_widths.items())
+column_widths.sort(key=lambda x: x[0])
+column_widths = [x[1] for x in column_widths]
+
+# Format column widths into a string which will become the basis
+# for each row.
+column_format = '{:%s%d%s}'
+column_formats = []
+for (index, width) in enumerate(column_widths):
+    formtype = column_types.get(index, 'g')
+    justify = '<' if formtype == 's' else '>'
+    form = column_format % (justify, width, formtype)
+    column_formats.append(form)
+
+# Format the header.
+column_count = len(column_widths)
+diff = len(header) - column_count
+if diff > 0:
+    # We have labels for columns that were empty.
+    column_count = len(header)
+else:
+    diff *= -1
+header += [''] * diff
+
+for (index, label) in enumerate(header):
+    form = '{:<%ds}' % column_widths[index]
+    header[index] = form.format(label)
+header = DELIMITER.join(header)
+output_text += header + '\n'
+
+# Format the rows.
+for (rowindex, line) in enumerate(document_lines):
+    # Does this row need any blank columns?
+    diff = column_count - len(line)
+    if diff > 0:
+        line += [''] * diff
+        document_lines[rowindex] = line
+
+    # Format and replace it into the list.
+    for (columnindex, word) in enumerate(line):
+        if word == '':
+            line[columnindex] = ' ' * column_widths[columnindex]
+        else:
+            if column_types.get(columnindex, 'g') == 'g':
+                word = float(word)
+            else:
+                word = str(word)
+            line[columnindex] = column_formats[columnindex].format(word)
+
+document_lines = [DELIMITER.join(line) for line in document_lines]
+document_lines = '\n'.join(document_lines)
+output_text += document_lines
+print(output_text)
+
+'''
+number | name                   | age | address             | misc.                | blank column?
+     4 | John Smith             |  26 | 123 north street    |                      |              
+    17 | Jenny Smith            |   8 | 123 north street    |                      |              
+    55 | Veronica Dove          |  20 | 456 west street     |                      |              
+    77 | Austin Texas           |  33 | 789 south avenue    |                      |              
+    89 | Mister Super Long Name | 123 | planet earth        | he's really old      |              
+   120 | Bill                   |     |                     | Deceased             |              
+   999 | Nine Nine              | 999 | 999 ninth boulevard | favorite number is 9 |              
+'''

MassStitching/README.md

@@ -0,0 +1,4 @@
+Mass Stitch
+===========
+
+Given the name of a directory, stich together all the images in that directory into one large iamge.

MassStitching/massstitch.py

@@ -0,0 +1,58 @@
+from PIL import Image
+import hashlib
+import os
+import sys
+
+def load_all_images(iterable):
+    images = []
+    for filename in iterable:
+        print('Loading "%s"' % filename)
+        try:
+            image = Image.open(filename)
+            image.filename = filename
+            print('Loaded "%s"' % filename)
+            images.append(image)
+        except OSError:
+            print('Could not load "%s"' % filename)
+    return images
+
+def listfiles(directory):
+    files = [name for name in os.listdir(directory)]
+    files = [os.path.join(directory, name) for name in files]
+    files = [name for name in files if os.path.isfile(name)]
+    return files
+
+def stitch(images, outputfilename):
+    largest_width = max(image.size[0] for image in images)
+    largest_height = max(image.size[1] for image in images)
+    print('Using cell size of %dx%dpx' % (largest_width, largest_height))
+    grid_width = round(len(images) ** 0.5)
+    # overflow adds an extra line for nonperfect squares.
+    overflow = 1 if (len(images) % grid_width != 0) else 0
+    grid_height = (len(images) // grid_width) + overflow
+    grid_width_pixels = grid_width * largest_width
+    grid_height_pixels = grid_height * largest_height
+
+    print('Creating image of size: %dx%d (%dx%dpx)' % (grid_width, grid_height, grid_width_pixels, grid_height_pixels))
+    stitched_image = Image.new('RGBA', [grid_width_pixels, grid_height_pixels])
+    print('Pasting components')
+    for (index, image) in enumerate(images):
+        pad_x = int((largest_width - image.size[0]) / 2)
+        pad_y = int((largest_height - image.size[1]) / 2)
+        gridspot_x = index % grid_width
+        gridspot_y = index // grid_width
+        pixel_x = (gridspot_x * largest_width) + pad_x
+        pixel_y = (gridspot_y * largest_height) + pad_y
+        print(index, image.filename, gridspot_x, gridspot_y, pixel_x, pixel_y)
+        stitched_image.paste(image, (pixel_x, pixel_y))
+    print('Saving "%s"' % outputfilename)
+    stitched_image.save(outputfilename)
+
+
+directory = sys.argv[1]
+images = listfiles(directory)
+directory_id = 'massstitch_%s.png' % directory
+if directory_id in images:
+    images.remove(directory_id)
+images = load_all_images(images)
+stitch(images, directory_id)

Meal/README.md

@@ -0,0 +1,6 @@
+Meal
+=======
+
+Just how much pizza do you eat, anyway?
+
+Read HELP_TEXT inside the meal.py file.

Meal/meal.py

@@ -0,0 +1,330 @@
+import datetime
+import math
+import os
+import sqlite3
+import sys
+import time
+
+
+UID_CHARACTERS = 16
+RECENT_COUNT = 12
+
+STRFTIME = '%Y %m %d %H:%M'
+
+SQL_MEAL_ID = 0
+SQL_MEAL_CREATED = 1
+SQL_MEAL_HUMAN = 2
+
+SQL_REL_FOOD = 1
+
+SQL_GROUP_FOOD = 0
+SQL_GROUP_GROUP = 1
+
+DB_INIT = '''
+CREATE TABLE IF NOT EXISTS meals(id TEXT, created INT, human TEXT);
+CREATE TABLE IF NOT EXISTS meal_foods(mealid TEXT, food TEXT);
+CREATE TABLE IF NOT EXISTS food_groups(food TEXT, foodgroup TEXT);
+
+CREATE INDEX IF NOT EXISTS index_meal_id on meals(id);
+CREATE INDEX IF NOT EXISTS index_meal_created on meals(created);
+CREATE INDEX IF NOT EXISTS index_food_mealid on meal_foods(mealid);
+CREATE INDEX IF NOT EXISTS index_food_food on meal_foods(food);
+CREATE INDEX IF NOT EXISTS index_group_food on food_groups(food);
+'''.strip()
+
+HELP_TEXT = '''
+> meal add pizza, soda     : Add a new meal with the foods "pizza" and "soda".
+> meal adjust ec2 +10      : Adjust the timestamp of the meal starting with "ec2" by +10 seconds.
+> meal adjust ec2 +10*60   : Adjusting timestamps supports math operations.
+> meal group water drinks  : Add "water" to foodgroup "drinks". Used for organization & reports.
+> meal group water         : Display the name of the group "water" belongs to.
+> meal recent              : Display info and foods for recent meals. Default {recent_count}.
+> meal recent 4            : Display the last 4 meals.
+> meal recent all          : Display ALL meals.
+> meal remove ec2          : Remove the meal whose ID starts with "ec2".
+> meal show ec2            : Display info and foods for the meal whose ID starts with "ec2".
+> meal ungroup water       : Remove "water" from its foodgroup.
+'''.format(recent_count=RECENT_COUNT)
+
+def listget(li, index, fallback=None):
+    try:
+        return li[index]
+    except IndexError:
+        return fallback
+
+def uid(length=None):
+    '''
+    Generate a u-random hex string..
+    '''
+    if length is None:
+        length = UID_CHARACTERS
+    identifier = ''.join('{:02x}'.format(x) for x in os.urandom(math.ceil(length / 2)))
+    if len(identifier) > length:
+        identifier = identifier[:length]
+    return identifier
+
+
+class MealDB():
+    def __init__(self, dbname='C:/Git/else/Meal/meal.db'):
+        self.dbname = dbname
+        self._sql = None
+        self._cur = None
+
+    @property
+    def sql(self):
+        if self._sql is None:
+            self._sql = sqlite3.connect(self.dbname)
+        return self._sql
+
+    @property
+    def cur(self):
+        if self._cur is None:
+            self._cur = self.sql.cursor()
+            statements = DB_INIT.split(';')
+            for statement in statements:
+                #print(statement)
+                self._cur.execute(statement)
+        return self._cur
+
+    def add_meal(self, foods=None):
+        if foods is None:
+            raise Exception('Empty meal!')
+        assert isinstance(foods, (list, tuple))
+        foods = set(foods)
+
+        if ''.join(foods).replace(' ', '') == '':
+            raise Exception('Empty meal!')
+
+        mealid = self.new_uid('meals')
+        now = datetime.datetime.now()
+        now_stamp = int(now.timestamp())
+        now_string = now.strftime(STRFTIME)
+
+        self.normalized_query('INSERT INTO meals VALUES(?, ?, ?)', [mealid, now_stamp, now_string])
+        for food in foods:
+            self.normalized_query('INSERT INTO meal_foods VALUES(?, ?)', [mealid, food])
+        self.sql.commit()
+        foods = ', '.join(foods)
+        print('Added meal %s at %s with %s' % (mealid, now_string, foods))
+        return mealid
+
+    def adjust_timestamp(self, mealid, adjustment):
+        '''
+        Move a certain meal by `adjustment` seconds. This is useful when you need to
+        report a meal that happened a while ago, rather than the current timestamp.
+        '''
+        meal = self.get_meal_by_id(mealid)
+        mealid = meal[SQL_MEAL_ID]
+        meal_time = meal[SQL_MEAL_CREATED]
+        meal_time += adjustment
+        time_string = datetime.datetime.fromtimestamp(meal_time).strftime(STRFTIME)
+        self.normalized_query('UPDATE meals SET created=?, human=? WHERE id=?', [meal_time, time_string, mealid])
+        self.sql.commit()
+        print('Adjusted %s to %s' % (mealid, time_string))
+
+    def normalized_query(self, query, bindings):
+        nbindings = []
+        for binding in bindings:
+            if isinstance(binding, str):
+                nbindings.append(binding.lower())
+                continue
+            nbindings.append(binding)
+        self.cur.execute(query, nbindings)
+
+    def get_foods_for_meal(self, mealid):
+        meal = self.get_meal_by_id(mealid)
+        mealid = meal[SQL_MEAL_ID]
+        self.normalized_query('SELECT food FROM meal_foods WHERE mealid == ?', [mealid])
+        items = self.cur.fetchall()
+        items = [item[0] for item in items]
+        return items
+
+    def get_meal_by_id(self, mealid):
+        if len(mealid) == UID_CHARACTERS:
+            meal_q = mealid
+            self.normalized_query('SELECT * FROM meals WHERE id == ?', [meal_q])
+        else:
+            meal_q = mealid + '%'
+            self.normalized_query('SELECT * FROM meals WHERE id LIKE ?', [meal_q])        
+
+        items = self.cur.fetchall()
+        if len(items) > 1:
+            items = [str(item) for item in items]
+            items = '\n'.join(items)
+            raise Exception('Found multiple meals for id "%s"\n%s' % (meal_q, items))
+        if len(items) == 0:
+            raise Exception('Found no meal for id "%s"' % (meal_q))
+
+        meal = items[0]
+        return meal
+
+    def group(self, food, groupname):
+        '''
+        Insert `food` into the foodgroup `groupname`. This is used for organization,
+        normalization, and creating dietary reports.
+        '''
+        self.normalized_query('SELECT * FROM food_groups WHERE food == ?', [food])
+        belongs = self.cur.fetchone()
+
+        if groupname is None:
+            self.normalized_query('SELECT * FROM food_groups where foodgroup == ?', [food])
+            contains = self.cur.fetchall()
+
+            if belongs is not None:
+                print('"%s" belongs to group "%s".' % (food, belongs[1]))
+            else:
+                print('"%s" is not in any group.' % (food))
+
+            if contains is not None and len(contains) > 0:
+                contains = [x[0] for x in contains]
+                contains = [repr(x) for x in contains]
+                contains = ', '.join(contains)
+                print('The "%s" group contains: %s' % (food, contains))
+            return
+
+        if belongs is not None:
+            raise Exception('"%s" is already in group "%s"' % (f[0], f[1]))
+        self.normalized_query('INSERT INTO food_groups VALUES(?, ?)', [food, groupname])
+        self.sql.commit()
+        print('Added "%s" to group "%s"' % (food, groupname))
+
+    def new_uid(self, table):
+        '''
+        Create a new UID that is unique to the given table.
+        '''
+        result = None
+        query = 'SELECT * FROM {table} WHERE id == ?'.format(table=table)
+        while result is None:
+            i = uid()
+            # Just gotta be sure, man.
+            self.normalized_query(query, [i])
+            if self.cur.fetchone() is None:
+                result = i
+        return result
+
+    def remove_meal(self, mealid):
+        meal = self.get_meal_by_id(mealid)
+        mealid = meal[SQL_MEAL_ID]
+        self.normalized_query('DELETE FROM meals WHERE id == ?', [mealid])
+        self.normalized_query('DELETE FROM meal_foods WHERE mealid == ?', [mealid])
+        self.sql.commit()
+        print('Removed meal %s' % (mealid))
+
+    def show_meal(self, mealid):
+        '''
+        Display:
+            id
+            timestamp
+            foods
+        for the meal with the given ID.
+        '''
+        meal = self.get_meal_by_id(mealid)
+        foods = self.get_foods_for_meal(mealid)
+        print(meal[SQL_MEAL_ID])
+        print(meal[SQL_MEAL_HUMAN])
+        foods = ', '.join(foods)
+        print(foods)
+
+    def show_recent(self, count=RECENT_COUNT):
+        '''
+        Display:
+            id : timestamp : foods
+        for the `count` most recent meals. If count is "all" or "*", show ALL meals.
+        '''
+        if count in ('all', '*'):
+            self.normalized_query('SELECT * FROM meals ORDER BY created DESC', [])
+        else:
+            self.normalized_query('SELECT * FROM meals ORDER BY created DESC LIMIT ?', [count])
+        meals = self.cur.fetchall()
+        output = []
+        for meal in meals:
+            mealid = meal[SQL_MEAL_ID]
+            human = meal[SQL_MEAL_HUMAN]
+            foods = self.get_foods_for_meal(mealid)
+            foods = ', '.join(foods)
+            output.append('%s : %s : %s' % (mealid, human, foods))
+        output = '\n'.join(output)
+        print(output)
+
+    def ungroup(self, food):
+        '''
+        Remove `food` from whatever group it is in.
+        '''
+        self.normalized_query('SELECT * FROM food_groups WHERE food == ?', [food])
+        f = self.cur.fetchone()
+        if f is None:
+            raise Exception('"%s" is not part of a group' % (food))
+        groupname = f[1]
+        self.normalized_query('DELETE FROM food_groups WHERE food == ?', [food])
+        self.sql.commit()
+        print('Removed "%s" from group "%s"' % (food, groupname))
+
+
+if __name__ == '__main__':
+    mealdb = MealDB()
+
+    args = sys.argv[1:]
+    if len(args) == 0:
+        command = ''
+    else:
+        command = args[0].lower()
+
+    if command == 'add':
+        args = args[1:]
+
+    elif command == 'adjust':
+        mealid = args[1]
+        adjustment = args[2]
+        adjustment = eval(adjustment)
+        mealdb.adjust_timestamp(mealid, adjustment)
+        quit()
+
+    elif command == 'group':
+        food = args[1]
+        groupname = listget(args, 2, None)
+        mealdb.group(food, groupname)
+        quit()
+
+    elif command == 'recent':
+        count = listget(args, 1, RECENT_COUNT)
+        mealdb.show_recent(count)
+        quit()
+
+    elif command == 'remove':
+        mealids = args[1]
+        mealids = mealids.replace(' ', '')
+        mealids = mealids.split(',')
+        for mealid in mealids:
+            mealdb.remove_meal(mealid)
+        quit()
+
+    elif command == 'show':
+        mealid = args[1]
+        mealdb.show_meal(mealid)
+        quit()
+
+    elif command == 'ungroup':
+        food = args[1]
+        mealdb.ungroup(food)
+        quit()
+
+    else:
+        print(HELP_TEXT)
+        quit()
+
+    args = ' '.join(args)
+
+    if ';' in args:
+        (args, adjustment) = args.split(';')
+        adjustment = adjustment.strip()
+        adjustment = eval(adjustment)
+    else:
+        adjustment = 0
+
+    args = args.strip()
+    args = args.split(',')
+    args = [food.strip() for food in args]
+    meal = mealdb.add_meal(args)
+    if adjustment != 0:
+        mealdb.adjust_timestamp(meal, adjustment)
+    quit()

MeshGenerator/meshgenerator.py

@@ -0,0 +1,89 @@
+from PIL import Image
+import sys
+
+def chunk_iterable(iterable, chunk_length, allow_incomplete=True):
+    '''
+    Given an iterable, divide it into chunks of length `chunk_length`.
+    If `allow_incomplete` is True, the final element of the returned list may be shorter
+    than `chunk_length`. If it is False, those items are discarded.
+    '''
+    if len(iterable) % chunk_length != 0 and allow_incomplete:
+        overflow = 1
+    else:
+        overflow = 0
+
+    steps = (len(iterable) // chunk_length) + overflow
+    return [iterable[chunk_length * x : (chunk_length * x) + chunk_length] for x in range(steps)]
+
+def hex_to_rgb(x):
+    x = x.replace('#', '')
+    x = chunk_iterable(x, 2)
+    print(x)
+    x = tuple(int(i, 16) for i in x)
+    return x
+
+def mesh_generator(image_width, image_height, square_size, mode):
+    square = square_size * 2
+    print(mode)
+    x_space = square_size * mode[0]
+    y_space = square_size * mode[1]
+    odd_x = False
+    odd_y = False
+    for y in range(0, image_height, y_space):
+        odd_y = not odd_y
+        for x in range(0, image_width, x_space):
+            odd_x = not odd_x
+            boost_x = int(odd_y) * square_size * mode[2]
+            boost_y = int(odd_x) * square_size * mode[3]
+            #print(boost_x, boost_y)
+            #x += boost_x
+            #y += boost_y
+            yield (x + boost_x, y + boost_y)
+
+def make_image(image_width, image_height, square_size, mode, bgcolor='#00000000', fgcolor='#000000'):
+    pattern_repeat_width = (2 * square_size * mode[0])
+    pattern_repeat_height = (2 * square_size * mode[1])
+    print(pattern_repeat_width, pattern_repeat_height)
+
+    bgcolor = hex_to_rgb(bgcolor)
+    fgcolor = hex_to_rgb(fgcolor)
+
+    pattern = Image.new('RGBA', (pattern_repeat_width, pattern_repeat_height), bgcolor)
+    #image = Image.new('RGBA', (image_width, image_height))
+    blackbox = Image.new('RGBA', (square_size, square_size), fgcolor)
+    for pair in mesh_generator(pattern_repeat_width, pattern_repeat_height, square_size, mode):
+        pattern.paste(blackbox, pair)
+    while pattern.size[0] < image_width or pattern.size[1] < image_height:
+        p = pattern
+        (w, h) = p.size
+        print('expanding from', w, h)
+        pattern = Image.new('RGBA', (w * 2, h * 2))
+        for y in range(2):
+            for x in range(2):
+                pattern.paste(p, (w * x, h * y))
+    image = pattern.crop((0, 0, image_width, image_height))
+
+    mode = [str(x) for x in mode]
+    mode = ''.join(mode)
+    filename = 'mesh_%dx%d_%d_%s.png' % (image_width, image_height, square_size, mode)
+    image.save(filename)
+    print('Saved %s' % filename)
+
+def listget(li, index, fallback=None):
+    try:
+        return li[index]
+    except IndexError:
+        return fallback
+
+if __name__ == '__main__':
+    image_width = int(sys.argv[1])
+    image_height = int(sys.argv[2])
+    square_size = int(listget(sys.argv, 3, 1))
+    x_spacing = int(listget(sys.argv, 4, 2))
+    y_spacing = int(listget(sys.argv, 5, 2))
+    x_alternator = int(listget(sys.argv, 6, 0))
+    y_alternator = int(listget(sys.argv, 7, 0))
+    bgcolor = listget(sys.argv, 8, '#00000000')
+    fgcolor = listget(sys.argv, 9, '#000000')
+    mode = (x_spacing, y_spacing, x_alternator, y_alternator)
+    make_image(image_width, image_height, square_size, mode, bgcolor, fgcolor)

OpenDirDL/opendirdl.py

@@ -0,0 +1,332 @@
+import bs4
+import hashlib
+import json
+import os
+import re
+import requests
+import string
+import sys
+import time
+import traceback
+import urllib.parse
+
+FILENAME_BADCHARS = '/\\:*?"<>|'
+DOWNLOAD_CHUNK = 2048
+
+# When doing a basic scan, we will not send HEAD requests to URLs that end in these strings,
+# because they're probably files.
+# This isn't meant to be a comprehensive filetype library, but it covers enough of the
+# typical opendir to speed things up.
+SKIPPABLE_FILETYPES = [
+'.avi',
+'.bmp',
+'.epub',
+'.db',
+'.flac',
+'.ico',
+'.iso',
+'.jpg',
+'.m4a',
+'.mkv',
+'.mov',
+'.mp3',
+'.mp4',
+'.pdf',
+'.png',
+'.srt',
+'.txt',
+'.webm',
+'.zip',
+]
+
+SKIPPABLE_FILETYPES = [x.lower() for x in SKIPPABLE_FILETYPES]
+
+class Downloader:
+    def __init__(self, urlfile, outputdir=None, headers=None):
+        jdict = file_to_dict(urlfile)
+        self.urls = [item[0] for item in jdict.items()]
+        self.urls.sort(key=str.lower)
+        self.outputdir = outputdir
+
+        if self.outputdir is None or self.outputdir == "":
+            # returns (root, path, filename). Keep root.
+            self.outputdir = url_to_filepath(self.urls[0])[0]
+
+    def download(self, overwrite=False):
+        overwrite = bool(overwrite)
+        for url in self.urls:
+            ''' Creating the Path '''
+            (root, folder, filename) = url_to_filepath(url)
+            # In case the user has set a custom download directory,
+            # ignore the above value of `root`.
+            root = self.outputdir
+            folder = os.path.join(root, folder)
+            if not os.path.exists(folder):
+                os.makedirs(folder)
+            localname = os.path.join(folder, filename)
+            temporary_basename = hashit(url, 16) + '.oddltemporary'
+            temporary_localname = os.path.join(folder, temporary_basename)
+
+            ''' Managing overwrite '''
+            if os.path.isfile(localname):
+                if overwrite is True:
+                    os.remove(localname)
+                else:
+                    safeprint('Skipping "%s". Use `overwrite=True`' % localname)
+                    continue
+
+            safeprint('Downloading "%s" as "%s"' % (localname, temporary_basename))
+            filehandle = open(temporary_localname, 'wb')
+            try:
+                download_file(url, filehandle, hookfunction=hook1)
+                os.rename(temporary_localname, localname)
+            except:
+                filehandle.close()
+                raise
+
+class Walker:
+    def __init__(self, website, outputfile, fullscan=False):
+        self.website = website
+        self.fullscan = bool(fullscan)
+        if os.path.exists(outputfile):
+            self.results = file_to_dict(outputfile)
+        else:
+            self.results = {}
+        self.already_seen = set()
+
+    def add_head_to_results(self, head):
+        if isinstance(head, str):
+            # For when we're doing a basic scan, which skips urls that
+            # look like a file.
+            self.results[head] = {
+                'Content-Length': -1,
+                'Content-Type': '?',
+            }
+            self.already_seen.add(head)
+        else:
+            # For when we're doing a full scan, which does a HEAD request
+            # for all urls.
+            self.results[head.url] = {
+                'Content-Length': int(head.headers.get('Content-Length', -1)),
+                'Content-Type': head.headers.get('Content-Type', '?'),
+            }
+            self.already_seen.add(head.url)
+
+    def extract_hrefs(self, response):
+        soup = bs4.BeautifulSoup(response.text)
+        elements = soup.findAll('a')
+        hrefs = []
+        for element in elements:
+            try:
+                href = element['href']
+            except KeyError:
+                continue
+            href = urllib.parse.urljoin(response.url, href)
+            if not href.startswith(self.website):
+                # Don't go to other sites or parent directories
+                continue
+            if 'C=' in href and 'O=' in href:
+                # Alternative sort modes for index pages
+                continue
+            if href.endswith('desktop.ini'):
+                # I hate these things
+                continue
+            hrefs.append(href)
+        return hrefs
+
+    def walk(self, url=None):
+        if url is None:
+            url = self.website
+        else:
+            url = urllib.parse.urljoin(self.website, url)
+
+        results = []
+
+        urll = url.lower()
+        if self.fullscan is False and any(urll.endswith(ext) for ext in SKIPPABLE_FILETYPES):
+            print('Skipping "%s" due to extension' % url)
+            self.add_head_to_results(url)
+            return results
+
+        if not url.startswith(self.website):
+            # Don't follow external links or parent directory.
+            return results
+
+        head = requests.head(url)
+        head.raise_for_status()
+
+        safeprint('HEAD: %s : %s' % (url, head))
+        content_type = head.headers.get('Content-Type', '?')
+        self.already_seen.add(head.url)
+
+        if content_type.startswith('text/html') and head.url.endswith('/'):
+            # This is an index page, let's get recursive.
+            page = requests.get(url)
+            safeprint(' GET: %s : %s' % (url, page))
+            hrefs = self.extract_hrefs(page)
+            for url in hrefs:
+                if url not in self.results and url not in self.already_seen:
+                    results += self.walk(url)
+        else:
+            # Don't add index pages to the results.
+            self.add_head_to_results(head)
+
+        return results
+
+
+def dict_to_file(jdict, filename):
+    filehandle = open(filename, 'wb')
+    text = json.dumps(jdict, indent=4, sort_keys=True)
+    text = text.encode('utf-8')
+    filehandle.write(text)
+    filehandle.close()
+
+def download_file(url, filehandle, getsizeheaders=True, hookfunction=None, headers={}, auth=None):
+    if getsizeheaders:
+        totalsize = requests.head(url, headers=headers, auth=auth)
+        totalsize = int(totalsize.headers['content-length'])
+    else:
+        totalsize = 1
+    currentblock = 0
+    downloading = requests.get(url, stream=True, headers=headers, auth=auth)
+    for chunk in downloading.iter_content(chunk_size=DOWNLOAD_CHUNK):
+        if chunk:
+            currentblock += 1
+            filehandle.write(chunk)
+            if hookfunction is not None:
+                hookfunction(currentblock, DOWNLOAD_CHUNK, totalsize)
+    filehandle.close()
+    size = os.path.getsize(filehandle.name)
+    if size < totalsize:
+        raise Exception('Did not receive expected total size. %d / %d' % (size, totalsize))
+    return True
+
+def file_to_dict(filename):
+    filehandle = open(filename, 'rb')
+    jdict = json.loads(filehandle.read().decode('utf-8'))
+    filehandle.close()
+    return jdict
+
+def filepath_sanitize(text, exclusions=''):
+    bet = FILENAME_BADCHARS.replace(exclusions, '')
+    for char in bet:
+        text = text.replace(char, '')
+    return text
+
+def hashit(text, length=None):
+    h = hashlib.sha512(text.encode('utf-8')).hexdigest()
+    if length is not None:
+        h = h[:length]
+    return h
+
+def hook1(currentblock, chunksize, totalsize):
+    currentbytes = currentblock * chunksize
+    if currentbytes > totalsize:
+        currentbytes = totalsize
+    currentbytes = '{:,}'.format(currentbytes)
+    totalsize = '{:,}'.format(totalsize)
+    currentbytes = currentbytes.rjust(len(totalsize), ' ')
+    print('%s / %s bytes' % (currentbytes, totalsize), end='\r')
+    if currentbytes == totalsize:
+        print()
+
+def safeprint(text, **kwargs):
+    text = str(text)
+    text = text.encode('ascii', 'replace').decode()
+    text = text.replace('?', '_')
+    print(text, **kwargs)
+
+def url_to_filepath(text):
+    text = urllib.parse.unquote(text)
+    parts = urllib.parse.urlsplit(text)
+    root = parts.netloc
+    (folder, filename) = os.path.split(parts.path)
+    while folder.startswith('/'):
+        folder = folder[1:]
+
+    # Folders are allowed to have slashes
+    folder = filepath_sanitize(folder, exclusions='/\\')
+    folder = folder.replace('\\', os.path.sep)
+    folder = folder.replace('/', os.path.sep)
+    # But Files are not.
+    filename = filepath_sanitize(filename)
+    return (root, folder, filename)
+
+## Commandline functions ####################################################\\
+def digest(website, outputfile, fullscan, *trash):
+    fullscan = bool(fullscan)
+    if website[-1] != '/':
+        website += '/'
+    walker = Walker(website, outputfile, fullscan=fullscan)
+    try:
+        walker.walk()
+        dict_to_file(walker.results, outputfile)
+    except:
+        dict_to_file(walker.results, outputfile)
+        traceback.print_exc()
+        print('SAVED PROGRESS SO FAR')
+
+def download(urlfile, outputdir, overwrite, *trash):
+    downloader = Downloader(urlfile, outputdir)
+    downloader.download(overwrite)
+
+def filter_pattern(urlfile, patterns, negative=False, *trash):
+    '''
+    When `negative` is True, items are kept when they do NOT match the pattern,
+    allowing you to delete trash files.
+
+    When `negative` is False, items are keep when they DO match the pattern,
+    allowing you to keep items of interest.
+    '''
+    if isinstance(patterns, str):
+        patterns = [patterns]
+    jdict = file_to_dict(urlfile)
+    keys = list(jdict.keys())
+    for key in keys:
+        for pattern in patterns:
+            contains = re.search(pattern, key) is not None
+            if contains ^ negative:
+                safeprint('Removing "%s"' % key)
+                del jdict[key]
+    dict_to_file(jdict, urlfile)
+
+def keep_pattern(urlfile, patterns, *trash):
+    filter_pattern(urlfile=urlfile, patterns=patterns, negative=True)
+
+def measure(urlfile, *trash):
+    jdict = file_to_dict(urlfile)
+    totalbytes = 0
+    for (url, info) in jdict.items():
+        bytes = info['Content-Length']
+        if bytes > 0:
+            totalbytes += bytes
+    bytestring = '{:,}'.format(totalbytes)
+    print(bytestring)
+    return totalbytes
+
+def remove_pattern(urlfile, patterns, *trash):
+    filter_pattern(urlfile=urlfile, patterns=patterns, negative=False)
+
+def listget(l, index, default=None):
+    try:
+        return l[index]
+    except IndexError:
+        return default
+cmdfunctions = [digest, download, keep_pattern, measure, remove_pattern]
+## End of commandline functions #############################################//
+    
+if __name__ == '__main__':
+    command = listget(sys.argv, 1, None)
+    arg1 = listget(sys.argv, 2, None)
+    arg2 = listget(sys.argv, 3, None)
+    arg3 = listget(sys.argv, 4, None)
+    if command is None:
+        quit()
+    did_something = False
+    for function in cmdfunctions:
+        if command == function.__name__:
+            function(arg1, arg2, arg3)
+            did_something = True
+            break
+    if not did_something:
+        print('No matching function')

Passwordy/passwordy.py

@@ -5,116 +5,181 @@ import sys
 DEFAULT_LENGTH = 32
 DEFAULT_SENTENCE = 5
 HELP_MESSAGE = '''
- ---------------------------------------------------------------
-|Generates a randomized password.                               |
-|                                                               |
-|> passwordy [length] ["p"] ["d"]                               |
-|                                                               |
-|   length : How many characters. Default %03d.                  |
-|   p      : If present, the password will contain punctuation  |
-|            characters. Otherwise not.                         |
-|   d      : If present, the password will contain digits.      |
-|            Otherwise not.                                     |
-|                                                               |
-|   The password can always contain upper and lowercase         |
-|   letters.                                                    |
- ---------------------------------------------------------------
-'''[1:-1] % (DEFAULT_LENGTH)
+===============================================================================
+Generates a randomized password.
 
-HELP_SENTENCE = '''
- ---------------------------------------------------------------
-|Generates a randomized sentence                                |
-|                                                               |
-|> passwordy sent [length] [join]                               |
-|                                                               |
-|   length : How many words to retrieve. Default %03d.           |
-|   join   : The character that will join the words together.   |
-|            Default space.                                     |
- ---------------------------------------------------------------
- '''[1:-1] % (DEFAULT_SENTENCE)
+> passwordy [length] [options]
 
-def make_password(length=None, allowpunctuation=False, allowdigits=False, digits_only=False, binary=False):
-	'''
-	Returns a string of length `length` consisting of a random selection
-	of uppercase and lowercase letters, as well as punctuation and digits
-	if parameters permit
-	'''
-	if length is None:
-		length = DEFAULT_LENGTH
-	
-	if digits_only is False and binary is False:
-		s = string.ascii_letters
-		if allowpunctuation is True:
-			s += string.punctuation
-		if allowdigits is True:
-			s += string.digits
-	elif digits_only:
-		s = string.digits
-	elif binary:
-		s = '01'
+    length: How many characters. Default %03d.
+    options:
+        h  : consist entirely of hexadecimal characters.
+        b  : consist entirely of binary characters.
+        dd : consist entirely of decimal characters.
+        default : consist entirely of upper+lower letters.
 
-	password = ''.join([random.choice(s) for x in range(length)])
-	return password
+        p  : allow punctuation in conjunction with above.
+        d  : allow digits in conjunction with above.
+
+        l  : convert to lowercase.
+        u  : convert to uppercase.
+        nd : no duplicates. Each character can only appear once.
+
+Examples:
+> passwordy 32 h l
+98f17b6016cf08cc00f2aeecc8d8afeb
+
+> passwordy 32 h u
+2AA706866BF7A5C18328BF866136A261
+
+> passwordy 32 u
+JHEPTKCEFZRFXILMASHNPSTFFNWQHTTN
+
+> passwordy 32 p
+Q+:iSKX!Nt)ewUvlE*!+^D}hp+|<wpJ}
+
+> passwordy 32 l p
+m*'otz/"!qo?-^wwdu@fasf:|ldkosi`
+
+===============================================================================
+
+Generates a randomized sentence of words.
+
+> passwordy sent [length] [join]
+
+    length : How many words. Default %03d.
+    join   : The character that will join words together.
+             Default space.
+
+Examples:
+> passwordy sent
+arrowroot sheared rustproof undo propionic acid
+
+> passwordy sent 8
+cipher competition solid angle rigmarole lachrymal social class critter consequently
+
+> passwordy sent 8 _
+Kahn_secondary_emission_unskilled_superior_court_straight_ticket_voltameter_hopper_crass
+
+===============================================================================
+ '''.strip() % (DEFAULT_LENGTH, DEFAULT_SENTENCE)
+
+
+def listget(li, index, fallback=None):
+    try:
+        return li[index]
+    except IndexError:
+        return fallback
+
+def make_password(length=None, passtype='standard'):
+    '''
+    Returns a string of length `length` consisting of a random selection
+    of uppercase and lowercase letters, as well as punctuation and digits
+    if parameters permit
+    '''
+    if length is None:
+        length = DEFAULT_LENGTH
+
+    alphabet = ''
+
+    if 'standard' in passtype:
+        alphabet = string.ascii_letters
+    elif 'digit_only' in passtype:
+        alphabet = string.digits
+    elif 'hex' in passtype:
+        alphabet = '0123456789abcdef'
+    elif 'binary' in passtype:
+        alphabet = '01'
+
+    if '+digits' in passtype:
+        alphabet += string.digits
+    if '+punctuation' in passtype:
+        alphabet += string.punctuation
+    if '+lowercase' in passtype:
+        alphabet = alphabet.lower()
+    elif '+uppercase' in passtype:
+        alphabet = alphabet.upper()
+
+    alphabet = list(set(alphabet))
+
+    if '+noduplicates' in passtype:
+        if len(alphabet) < length:
+            message = 'Alphabet "%s" is not long enough to support no-dupe password of length %d'
+            message = message % (alphabet, length)
+            raise Exception(message)
+        password = ''
+        for x in range(length):
+            random.shuffle(alphabet)
+            password += alphabet.pop(0)
+    else:
+        password = ''.join([random.choice(alphabet) for x in range(length)])
+    return password
 
 def make_sentence(length=None, joiner=' '):
-	'''
-	Returns a string containing `length` words, which come from
-	dictionary.common.
-	'''
-	import dictionary.common as common
-	if length is None:
-		length = DEFAULT_LENGTH
-	words = [random.choice(common.words) for x in range(length)]
-	words = [w.replace(' ', joiner) for w in words]
-	result = joiner.join(words)
-	return result
+    '''
+    Returns a string containing `length` words, which come from
+    dictionary.common.
+    '''
+    import dictionary.common as common
+    if length is None:
+        length = DEFAULT_LENGTH
+    words = [random.choice(common.words) for x in range(length)]
+    words = [w.replace(' ', joiner) for w in words]
+    result = joiner.join(words)
+    return result
 
 if __name__ == '__main__':
-	args = sys.argv
-	argc = len(args) - 1
+    args = sys.argv[1:]
+    argc = len(args)
 
-	if argc == 0:
-		mode = 'password'
-		length = DEFAULT_LENGTH
+    mode = listget(args, 0, 'password')
+    if 'help' in mode:
+        print(HELP_MESSAGE)
+        quit()
 
-	elif args[1].isdigit():
-		mode = 'password'
-		length = int(args[1])
+    if 'sent' not in mode:
+        length  = listget(args, 0, str(DEFAULT_LENGTH))
+        options = [a.lower() for a in args[1:]]
 
-	elif args[1] in 'DdPp':
-		mode = 'password'
-		length = DEFAULT_LENGTH
+        if '-' in length:
+            length = length.replace(' ', '')
+            length = [int(x) for x in length.split('-', 1)]
+            length = random.randint(*length)
 
-	elif 'sent' in args[1].lower() and argc == 1:
-		mode = 'sentence'
-		length = DEFAULT_SENTENCE
+        elif not length.isdigit() and options == []:
+            options = [length]
+            length = DEFAULT_LENGTH
 
-	elif argc == 1:
-		mode = None
-		print(HELP_MESSAGE)
-		print(HELP_SENTENCE)
+        length = int(length)
 
-	elif 'sent' in args[1].lower() and args[2].isdigit():
-		mode = 'sentence'
-		length = int(args[2])
+        passtype = 'standard'
+        if 'dd' in options:
+            passtype = 'digit_only'
+        if 'b' in options:
+            passtype = 'binary'
+        if 'h' in options:
+            passtype = 'hex'
 
-	elif 'sent' in args[1].lower():
-		mode = 'sentence'
-		length = DEFAULT_SENTENCE
+        if 'l' in options:
+            passtype += '+lowercase'
+        elif 'u' in options:
+            passtype += '+uppercase'
+        if 'p' in options:
+            passtype += '+punctuation'
+        if 'd' in options:
+            passtype += '+digits'
+        if 'nd' in options:
+            passtype += '+noduplicates'
 
-	if mode == 'password':
-		punc = 'p' in args
-		digi = 'd' in args
-		digi_only = 'dd' in args
-		binary = 'b' in args
-		print(make_password(length, punc, digi, digi_only, binary))
-		
-	elif mode == 'sentence':
-		if argc == 3:
-			joiner = args[3]
-		else:
-			joiner = ' '
-		print(make_sentence(length, joiner))
+        print(make_password(length, passtype=passtype))
+        
+    else:
+        length = listget(args, 1, str(DEFAULT_SENTENCE))
+        joiner = listget(args, 2, ' ')
 
-	else:
-		pass
+        if not length.isdigit():
+            joiner = length
+            length = DEFAULT_SENTENCE
+
+        length = int(length)
+
+        print(make_sentence(length, joiner))

Phototagger/phototagger.py

@@ -0,0 +1,624 @@
+import datetime
+import os
+import PIL.Image
+import random
+import sqlite3
+import string
+import warnings
+
+# UIDs consist of hex characters, so keyspace is 16 ** UID_CHARACTERS.
+UID_CHARACTERS = 16
+VALID_TAG_CHARS = string.ascii_lowercase + string.digits + '_-'
+MAX_TAG_NAME_LENGTH = 32
+
+SQL_PHOTO_COLUMNCOUNT = 8
+SQL_PHOTO_ID = 0
+SQL_PHOTO_FILEPATH = 1
+SQL_PHOTO_EXTENSION = 2
+SQL_PHOTO_WIDTH = 3
+SQL_PHOTO_HEIGHT = 4
+SQL_PHOTO_AREA = 5
+SQL_PHOTO_BYTES = 6
+SQL_PHOTO_CREATED = 7
+
+SQL_PHOTOTAG_COLUMNCOUNT = 2
+SQL_PHOTOTAG_PHOTOID = 0
+SQL_PHOTOTAG_TAGID = 1
+
+SQL_SYN_COLUMNCOUNT = 2
+SQL_SYN_NAME = 0
+SQL_SYN_MASTER = 1
+
+SQL_TAG_COLUMNCOUNT = 2
+SQL_TAG_ID = 0
+SQL_TAG_NAME = 1
+
+DB_INIT = '''
+CREATE TABLE IF NOT EXISTS photos(
+    id TEXT,
+    filepath TEXT,
+    extension TEXT,
+    width INT,
+    height INT,
+    area INT,
+    bytes INT,
+    created INT
+    );
+CREATE TABLE IF NOT EXISTS tags(
+    id TEXT,
+    name TEXT
+    );
+CREATE TABLE IF NOT EXISTS photo_tag_rel(
+    photoid TEXT,
+    tagid TEXT
+    );
+CREATE TABLE IF NOT EXISTS tag_synonyms(
+    name TEXT,
+    mastername TEXT
+    );
+CREATE INDEX IF NOT EXISTS index_photo_id on photos(id);
+CREATE INDEX IF NOT EXISTS index_photo_path on photos(filepath);
+CREATE INDEX IF NOT EXISTS index_photo_created on photos(created);
+
+CREATE INDEX IF NOT EXISTS index_tag_id on tags(id);
+CREATE INDEX IF NOT EXISTS index_tag_name on tags(name);
+
+CREATE INDEX IF NOT EXISTS index_tagrel_photoid on photo_tag_rel(photoid);
+CREATE INDEX IF NOT EXISTS index_tagrel_tagid on photo_tag_rel(tagid);
+
+CREATE INDEX IF NOT EXISTS index_tagsyn_name on tag_synonyms(name);
+'''
+
+def basex(number, base, alphabet='0123456789abcdefghijklmnopqrstuvwxyz'):
+    '''
+    Converts an integer to a different base string.
+    Based on http://stackoverflow.com/a/1181922/5430534
+    '''
+    if base > len(alphabet):
+        raise Exception('alphabet %s does not support base %d' % (
+                         repr(alphabet), base))
+    alphabet = alphabet[:base]
+    if not isinstance(number, (int, str)):
+        raise TypeError('number must be an integer')
+    number = int(number)
+    based = ''
+    sign = ''
+    if number < 0:
+        sign = '-'
+        number = -number
+    if 0 <= number < len(alphabet):
+        return sign + alphabet[number]
+    while number != 0:
+        number, i = divmod(number, len(alphabet))
+        based = alphabet[i] + based
+    return sign + based
+
+def getnow(timestamp=True):
+    '''
+    Return the current UTC timestamp or datetime object.
+    '''
+    now = datetime.datetime.now(datetime.timezone.utc)
+    if timestamp:
+        return now.timestamp()
+    return now
+
+def is_xor(x, y):
+    '''
+    Return True if and only if one of (x, y) is truthy.
+    '''
+    same = (bool(x) == bool(y))
+    return not same
+
+def normalize_tagname(tagname):
+    '''
+    Tag names can only consist of lowercase letters, underscores,
+    and hyphens. The given tagname is lowercased, gets its spaces
+    replaced by underscores, and is stripped of any not-whitelisted
+    characters.
+    '''
+    tagname = tagname.lower()
+    tagname = tagname.replace(' ', '_')
+    tagname = (c for c in tagname if c in VALID_TAG_CHARS)
+    tagname = ''.join(tagname)
+    if len(tagname) == 0:
+        raise ValueError('Normalized tagname of length 0.')
+    return tagname
+
+def not_implemented(function):
+    '''
+    Great for keeping track of which functions still need to be filled out.
+    '''
+    warnings.warn('%s is not implemented' % function.__name__)
+    return function
+
+def uid(length=None):
+    '''
+    Generate a u-random hex string..
+    '''
+    if length is None:
+        length = UID_CHARACTERS
+    identifier = ''.join('{:02x}'.format(x) for x in os.urandom(math.ceil(length / 2)))
+    if len(identifier) > length:
+        identifier = identifier[:length]
+    return identifier
+
+
+class NoSuchPhoto(Exception):
+    pass
+
+class NoSuchTag(Exception):
+    pass
+
+class PhotoExists(Exception):
+    pass
+
+class TagExists(Exception):
+    pass
+
+class XORException(Exception):
+    pass
+
+class PhotoDB:
+    '''
+    This class represents an SQLite3 database containing the following tables:
+
+    photos: 
+        Rows represent image files on the local disk.
+        Entries contain a unique ID, the image's filepath, and metadata
+        like dimensions and filesize.
+
+    tags:
+        Rows represent labels, which can be applied to an arbitrary number of
+        photos. Photos may be selected by which tags they contain.
+        Entries contain a unique ID and a name.
+
+    photo_tag_rel:
+        Rows represent a Photo's ownership of a particular Tag.
+
+    tag_synonyms:
+        Rows represent relationships between two tag names, so that they both
+        resolve to the same Tag object when selected. Entries contain the
+        subordinate name and master name.
+        The master name MUST also exist in the `tags` table.
+        If a new synonym is created referring to another synoym, the master name
+        will be resolved and used instead, so a synonym is never in the master
+        column.
+        Tag objects will ALWAYS represent the master tag.
+
+        Note that the entries in this table do not contain ID numbers.
+        The rationale here is that "coco" is a synonym for "chocolate" regardless
+        of the "chocolate" tag's ID, or the fact that you decided to rename your
+        "chocolate" tag to "candy" after applying it to a few photos.
+        The `rename_tag` method includes a parameter `apply_to_synonyms` if you do
+        want them to follow.
+    '''
+    def __init__(self, databasename='phototagger.db'):
+        self.databasename = databasename
+        self.sql = sqlite3.connect(databasename)
+        self.cur = self.sql.cursor()
+        statements = DB_INIT.split(';')
+        for statement in statements:
+            self.cur.execute(statement)
+
+    def __repr__(self):
+        return 'PhotoDB(databasename={dbname})'.format(dbname=repr(self.databasename))
+
+    def add_photo_tag(self, photoid, tag=None, commit=True):
+        '''
+        Apply a tag to a photo. `tag` may be the name of the tag or a Tag
+        object from the same PhotoDB.
+
+        `tag` may NOT be the tag's ID, since an ID would also have been a valid name.
+        '''
+        if isinstance(tag, Tag) and tag.photodb is self:
+            tagid = tag.id
+        else:
+            tag = self.get_tag_by_name(tag)
+            if tag is None:
+                raise NoSuchTag(tag)
+            tagid = tag.id
+
+        self.cur.execute('SELECT * FROM photos WHERE id == ?', [photoid])
+        if self.cur.fetchone() is None:
+            raise NoSuchPhoto(photoid)
+
+        self.cur.execute('SELECT * FROM photo_tag_rel WHERE photoid == ? AND tagid == ?', [photoid, tagid])
+        if self.cur.fetchone() is not None:
+            warning = 'Photo {photoid} already has tag {tagid}'.format(photoid=photoid, tagid=tagid)
+            warnings.warn(warning)
+            return
+
+        self.cur.execute('INSERT INTO photo_tag_rel VALUES(?, ?)', [photoid, tagid])
+        if commit:
+            self.sql.commit()
+
+    @not_implemented
+    def convert_tag_to_synonym(self, tagname, mastertag):
+        '''
+        Convert an independent tag into a synonym for a different tag.
+        All photos which possess the current tag will have it replaced
+        with the master tag.
+        '''
+        photos = self.get_photos_by_tag(musts=[tagname])
+
+    def get_photo_by_id(self, photoid):
+        '''
+        Return this Photo object, or None if it does not exist.
+        '''
+        self.cur.execute('SELECT * FROM photos WHERE id == ?', [photoid])
+        photo = cur.fetchone()
+        if photo is None:
+            return None
+        photo = self.tuple_to_photo(photo)
+        return photo
+
+    def get_photo_by_path(self, filepath):
+        '''
+        Return this Photo object, or None if it does not exist.
+        '''
+        filepath = os.path.abspath(filepath)
+        self.cur.execute('SELECT * FROM photos WHERE filepath == ?', [filepath])
+        photo = self.cur.fetchone()
+        if photo is None:
+            return None
+        photo = self.tuple_to_photo(photo)
+        return photo
+
+    def get_photos_by_recent(self):
+        '''
+        Yield photo objects in order of creation time.
+        '''
+        # We're going to use a second cursor because the first one may
+        # get used for something else, deactivating this query.
+        cur2 = self.sql.cursor()
+        cur2.execute('SELECT * FROM photos ORDER BY created DESC')
+        while True:
+            f = cur2.fetchone()
+            if f is None:
+                return
+            photo = self.tuple_to_photo(f)
+            yield photo
+
+    @not_implemented
+    def get_photos_by_tag(
+        self,
+        musts=None,
+        mays=None,
+        forbids=None,
+        forbid_unspecified=False,
+        ):
+        '''
+        Given one or multiple tags, yield photos possessing those tags.
+
+        Parameters:
+            musts :
+                A list of strings or Tag objects.
+                Photos MUST have ALL tags in this list.
+            mays :
+                A list of strings or Tag objects.
+                If `forbid_unspecified` is True, then Photos MUST have AT LEAST ONE tag in this list.
+                If `forbid_unspecified` is False, then Photos MAY or MAY NOT have ANY tag in this list.
+            forbids :
+                A list of strings or Tag objects.
+                Photos MUST NOT have ANY tag in the list.
+            forbid_unspecified :
+                True or False.
+                If False, Photos need only comply with the `musts`.
+                If True, Photos need to comply with both `musts` and `mays`.
+        '''
+        if all(arg is None for arg in (musts, mays, forbids)):
+            raise TypeError('All arguments cannot be None')
+
+    def get_tag_by_id(self, tagid):
+        self.cur.execute('SELECT * FROM tags WHERE id == ?', [tagid])
+        tag = self.cur.fetchone()
+        if tag is None:
+            return None
+        tag = self.tuple_to_tag(tag)
+        return tag
+
+    def get_tag_by_name(self, tagname):
+        '''
+        Return the Tag object that the given tagname resolves to.
+
+        If the given tagname is a synonym, the master tag will be returned.
+        '''
+        if isinstance(tagname, Tag):
+            return tagname
+
+        self.cur.execute('SELECT * FROM tag_synonyms WHERE name == ?', [tagname])
+        fetch = self.cur.fetchone()
+        if fetch is not None:
+            mastertagid = fetch[SQL_SYN_MASTER]
+            tag = self.get_tag_by_id(mastertagid)
+            return tag
+
+        self.cur.execute('SELECT * FROM tags WHERE name == ?', [tagname])
+        fetch = self.cur.fetchone()
+        if fetch is None:
+            return None
+
+        tag = self.tuple_to_tag(fetch)
+        return tag
+        
+    def get_tags_by_photo(self, photoid):
+        '''
+        Return the tags assigned to the given photo.
+        '''
+        self.cur.execute('SELECT * FROM photo_tag_rel WHERE photoid == ?', [photoid])
+        tags = self.cur.fetchall()
+        tagobjects = []
+        for tag in tags:
+            tagid = tag[SQL_PHOTOTAG_TAGID]
+            tagobj = self.get_tag_by_id(tagid)
+            if tagobj is None:
+                warnings.warn('Photo {photid} contains unkown tagid {tagid}'.format(photoid=photoid, tagid=tagid))
+                continue
+            tagobjects.append(tagobj)
+        return tagobjects
+
+    def new_photo(self, filename, tags=[], allow_duplicates=False):
+        '''
+        Given a filepath, determine its attributes and create a new Photo object in the
+        database. Tags may be applied now or later.
+
+        If `allow_duplicates` is False, we will first check the database for any files
+        with the same path and raise PhotoExists if found.
+        '''
+        filename = os.path.abspath(filename)
+        if not allow_duplicates:
+            existing = self.get_photo_by_path(filename)
+            if existing is not None:
+                raise PhotoExists(filename, existing)
+
+        # I want the caller to receive any exceptions this raises.
+        image = PIL.Image.open(filename)
+
+        extension = os.path.splitext(filename)[1]
+        extension = extension.replace('.', '')
+        (width, height) = image.size
+        area = width * height
+        bytes = os.path.getsize(filename)
+        created = int(getnow())
+        photoid = self.new_uid('photos')
+        data = [None] * SQL_PHOTO_COLUMNCOUNT
+        data[SQL_PHOTO_ID] = photoid
+        data[SQL_PHOTO_FILEPATH] = filename
+        data[SQL_PHOTO_EXTENSION] = extension
+        data[SQL_PHOTO_WIDTH] = width
+        data[SQL_PHOTO_HEIGHT] = height
+        data[SQL_PHOTO_AREA] = area
+        data[SQL_PHOTO_BYTES] = bytes
+        data[SQL_PHOTO_CREATED] = created
+        photo = self.tuple_to_photo(data)
+        self.cur.execute('INSERT INTO photos VALUES(?, ?, ?, ?, ?, ?, ?, ?)', data)
+        for tag in tags:
+            try:
+                self.add_photo_tag(photoid, tag, commit=False)
+            except NoSuchTag:
+                self.sql.rollback()
+                raise
+        self.sql.commit()
+        return photo
+
+    def new_tag(self, tagname):
+        '''
+        Register a new tag.
+        '''
+        tagname = normalize_tagname(tagname)
+        if self.get_tag_by_name(tagname) is not None:
+            raise TagExists(tagname)
+        tagid = self.new_uid('tags')
+        self.cur.execute('INSERT INTO tags VALUES(?, ?)', [tagid, tagname])
+        self.sql.commit()
+        tag = self.tuple_to_tag([tagid, tagname])
+        return tag
+
+    def new_tag_synonym(self, tagname, mastertagname):
+        '''
+        Register a new synonym for an existing tag.
+        '''
+        tagname = normalize_tagname(tagname)
+        mastertagname = normalize_tagname(mastertagname)
+
+        if tagname == mastertagname:
+            raise TagExists(tagname)
+
+        tag = self.get_tag_by_name(tagname)
+        if tag is not None:
+            raise TagExists(tagname)
+
+        mastertag = self.get_tag_by_name(mastertagname)
+        if mastertag is None:
+            raise NoSuchTag(mastertagname)
+        mastertagname = mastertag.name
+
+        self.cur.execute('INSERT INTO tag_synonyms VALUES(?, ?)', [tagname, mastertagname])
+        self.sql.commit()
+
+    def new_uid(self, table):
+        '''
+        Create a new UID that is unique to the given table.
+        '''
+        result = None
+        # Well at least we won't get sql injection this way.
+        table = normalize_tagname(table)
+        query = 'SELECT * FROM {table} WHERE id == ?'.format(table=table)
+        while result is None:
+            i = uid()
+            # Just gotta be sure, man.
+            self.cur.execute(query, [i])
+            if self.cur.fetchone() is None:
+                result = i
+        return result
+
+    @not_implemented
+    def remove_photo(self):
+        pass
+
+    @not_implemented
+    def remove_tag(self, tagid=None, tagname=None):
+        '''
+        Delete a tag and its relation to any photos.
+        '''
+        if not is_xor(tagid, tagname):
+            raise XORException('One and only one of `tagid`, `tagname` can be passed.')
+
+        if tagid is not None:
+            self.cur.execute('SELECT * FROM tags WHERE id == ?', [tagid])
+            tag = self.cur.fetchone()
+
+        elif tagname is not None:
+            tagname = normalize_tagname(tagname)
+            self.cur.execute('SELECT * from tags WHERE name == ?', [tagname])
+            tag = self.cur.fetchone()
+
+        if tag is None:
+            raise NoSuchTag(tagid or tagname)
+
+        tag = self.tuple_to_tag(tag)
+
+        self.cur.execute('DELETE FROM tags WHERE id == ?', [tag.id])
+        self.cur.execute('DELETE FROM photo_tag_rel WHERE tagid == ?', [tag.id])
+        self.cur.execute('DELETE FROM tag_synonyms WHERE mastername == ?', [tag.name])
+        self.sql.commit()
+
+    @not_implemented
+    def remove_tag_synonym(self, tagname):
+        pass
+
+    def tuple_to_photo(self, tu):
+        '''
+        Given a tuple like the ones from an sqlite query,
+        create a Photo object.
+        '''
+        photoid = tu[SQL_PHOTO_ID]
+        tags = self.get_tags_by_photo(photoid)
+
+        photo = Photo(
+            photodb = self,
+            photoid = photoid,
+            filepath = tu[SQL_PHOTO_FILEPATH],
+            extension = tu[SQL_PHOTO_EXTENSION],
+            width = tu[SQL_PHOTO_WIDTH],
+            height = tu[SQL_PHOTO_HEIGHT],
+            area = tu[SQL_PHOTO_AREA],
+            created = tu[SQL_PHOTO_CREATED],
+            bytes = tu[SQL_PHOTO_BYTES],
+            tags = tags,
+            )
+        return photo
+
+    def tuple_to_tag(self, tu):
+        '''
+        Given a tuple like the ones from an sqlite query,
+        create a Tag object.
+        '''
+        tag = Tag(
+            photodb = self,
+            tagid = tu[SQL_TAG_ID],
+            name = tu[SQL_TAG_NAME]
+            )
+        return tag
+
+class Photo:
+    '''
+    This class represents a PhotoDB entry containing information about an image file.
+    Photo objects cannot exist without a corresponding PhotoDB object, because
+    Photos are not the actual files, just the database entry. 
+    '''
+    def __init__(
+        self,
+        photodb,
+        photoid,
+        filepath,
+        extension,
+        width,
+        height,
+        area,
+        bytes,
+        created,
+        tags=[],
+        ):
+        self.photodb = photodb
+        self.id = photoid
+        self.filepath = filepath
+        self.extension = extension
+        self.width = width
+        self.height = height
+        self.area = area
+        self.bytes = bytes
+        self.created = created
+        self.tags = tags
+
+    def __eq__(self, other):
+        if not isinstance(other, Photo):
+            return False
+        return self.id == other.id
+
+    def __hash__(self):
+        return hash(self.id)
+
+    def __repr__(self):
+        r = ('Photo(photodb={photodb}, ',
+            'photoid={photoid}, ',
+            'filepath={filepath}, ',
+            'extension={extension}, ',
+            'width={width}, ',
+            'height={height}, ',
+            'area={area}, ',
+            'bytes={bytes} ',
+            'created={created})'
+            )
+        r = ''.join(r)
+        r = r.format(
+            photodb = repr(self.photodb),
+            photoid = repr(self.id),
+            filepath = repr(self.filepath),
+            extension = repr(self.extension),
+            width = repr(self.width),
+            height = repr(self.height),
+            bytes = repr(self.bytes),
+            area = repr(self.area),
+            created = repr(self.created),
+            )
+        return r
+
+    def __str__(self):
+        return 'Photo: %s' % self.id
+
+    def add_photo_tag(self, tagname):
+        return self.photodb.add_photo_tag(self.id, tagname, commit=True)
+
+class Tag:
+    '''
+    This class represents a Tag, which can be applied to Photos for
+    organization.
+    '''
+    def __init__(self, photodb, tagid, name):
+        self.photodb = photodb
+        self.id = tagid
+        self.name = name
+
+    def __eq__(self, other):
+        if not isinstance(other, Tag):
+            return False
+        return self.id == other.id
+
+    def __hash__(self):
+        return hash(self.id)
+
+    def __repr__(self):
+        r = 'Tag(photodb={photodb}, name={name}, tagid={tagid})'
+        r = r.format(
+            photodb = repr(self.photodb),
+            name = repr(self.name),
+            tagid = repr(self.id),
+            )
+        return r
+
+    def __str__(self):
+        return 'Tag: %s : %s' % (self.id, self.name)
+
+if __name__ == '__main__':
+    p = PhotoDB()

Phototagger/phototagger_tests.py

@@ -0,0 +1,45 @@
+import os
+import phototagger
+import unittest
+
+DB_NAME = ':memory:'
+
+#try:
+#    os.remove(DB_NAME)
+#    print('Deleted old database.')
+#except FileNotFound:
+#    pass
+
+class PhotoDBTest(unittest.TestCase):
+    def setUp(self):
+        self.p = phototagger.PhotoDB(DB_NAME)
+
+    def tearDown(self):
+        pass
+
+    def test_add_and_remove_tag(self):
+        tag = self.p.new_tag('trains')
+        self.assertEqual(tag.name, 'trains')
+        self.assertEqual(len(tag.id), phototagger.UID_CHARACTERS)
+
+        tag2 = self.p.get_tag_by_id(tag.id)
+        self.assertEqual(tag, tag2)
+
+        tag3 = self.p.get_tag_by_name(tag.name)
+        self.assertEqual(tag, tag3)
+
+        self.assertEqual(tag2, tag3)
+
+        self.p.remove_tag(tagid=tag.id)
+
+        tag4 = self.p.get_tag_by_id(tag.id)
+        self.assertIsNone(tag4)
+
+    def test_new_tag_invalid_name(self):
+        print('NOT IMPLEMENTED')
+
+    def test_new_tag_too_long(self):
+        print('NOT IMPLEMENTED')
+
+if __name__ == '__main__':
+    unittest.main()

RGBLayers/rgblayers.py

@@ -0,0 +1,52 @@
+import os
+import sys
+import PIL.Image
+
+def listget(li, index, fallback=None):
+    try:
+        return li[index]
+    except IndexError:
+        return fallback
+
+def onechannel(tu, index):
+    ret = [0, 0, 0, 255]
+    ret[index] = listget(tu, index, 0)
+    ret[3] = listget(tu, 3, 255)
+    return tuple(ret)
+
+def splitchannels(filename):
+    image = PIL.Image.open(filename)
+    (base, extension) = os.path.splitext(filename)
+    image_r = PIL.Image.new(image.mode, image.size)
+    image_g = PIL.Image.new(image.mode, image.size)
+    image_b = PIL.Image.new(image.mode, image.size)
+    width = image.size[0]
+    height = image.size[1]
+    pixels = list(image.getdata())
+    print('Extracting R')
+    image_r.putdata([onechannel(x, 0) for x in pixels])
+    print('Extracting G')
+    image_g.putdata([onechannel(x, 1) for x in pixels])
+    print('Extracting B')
+    image_b.putdata([onechannel(x, 2) for x in pixels])
+    #for (index, pixel) in enumerate(iter(image.getdata())):
+    #    x = index % width
+    #    y = index // height
+    #    co = (x, y)
+    #    pixel = image.getpixel(co)
+    #    r = listget(pixel, 0, 0)
+    #    g = listget(pixel, 1, 0)
+    #    b = listget(pixel, 2, 0)
+    #    o = listget(pixel, 3, 255)
+    #    image_r.putpixel(co, (r, 0, 0, o))
+    #    image_g.putpixel(co, (0, g, 0, o))
+    #    image_b.putpixel(co, (0, 0, b, 0))
+    #    if x == 0:
+    #        print(y)
+    image_r.save('%s_R%s' % (base, extension), quality=100)
+    image_g.save('%s_G%s' % (base, extension), quality=100)
+    image_b.save('%s_B%s' % (base, extension), quality=100)
+
+filename = sys.argv[1]
+splitchannels(filename)
+i = PIL.Image.open(filename)

Steganographic/README.md

@@ -1,6 +1,13 @@
 Steganographic
 ==============
 
+    2015 01 15:
+        Now supports variable "bitness", the number of bits per color channel to overwrite.
+        Previously, bitness was always 1, to maximize transparency.
+        Now, bitness can be 1-8 to favor transparency or information density.
+
+ 
+
 Let's be honest, this was really just an excuse to make big Terminal headers out of hashmarks.
 
  
@@ -22,11 +29,13 @@ Each Image pixel holds 3 Secret bits, so the Image must have at least `((secretb
 An Image can hold `((3 * (pixels - 14)) / 8)` Secret bytes.  
 
     Usage:
-    > steganographic.py encode imagefilename.png secretfilename.ext
-    > steganographic.py decode lacedimagename.png
+    > steganographic.py encode imagefilename.png secretfilename.ext bitness
+    > steganographic.py decode lacedimagename.png bitness
+
+where bitness defaults to 1 in both cases.
 
 
-Reference table for files with NO EXTENSION.
+Reference table for files with NO EXTENSION and bitness of 1.
 For each extension character, subtract 1 byte from secret size
 
          pixels |       example dimensions | Secret file size
@@ -55,5 +64,5 @@ For each extension character, subtract 1 byte from secret size
      89,478,500 | 9500 x 9500 (90,250,000) | 33,554,432 bytes (32 mb)
 
 <p align="center">
-  <img src="https://github.com/voussoir/else/blob/master/.GitImages/steganographic_logo.png?raw=true" alt="steganographic"/>
+    <img src="https://github.com/voussoir/else/blob/master/.GitImages/steganographic_logo.png?raw=true" alt="steganographic"/>
 </p>

Steganographic/examples/trespassing (thats_advertising_mp4).png

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0a9c758425f9185dd207183b283f639cd6499d042ad75b51e3f7407624aeb8aa
-size 6002882

Steganographic/steganographic.py

@@ -49,31 +49,99 @@ import math
 import os
 import sys
 
-# 11 for the content length
+# 11 pixels for the secret file size
 HEADER_SIZE = 11
 
+FILE_READ_SIZE = 4 * 1024
+
 class StegError(Exception):
     pass
 
+class BitsToImage:
+    def __init__(self, image, bitness):
+        self.image = image
+        self.bitness = bitness
+        self.width = image.size[0]
+        self.pixel_index = -1
+        self.channel_index = 0
+        self.bit_index = self.bitness - 1
+        self.active_pixel = None
+        self.x = 0
+        self.y = 0
+
+    def _write(self, bit):
+        if self.active_pixel is None:
+            self.pixel_index += 1
+            self.channel_index = 0
+            self.bit_index = self.bitness - 1
+            (self.x, self.y) = index_to_xy(self.pixel_index, self.width)
+            self.active_pixel = list(self.image.getpixel((self.x, self.y)))
+
+        channel = self.active_pixel[self.channel_index]
+        channel = set_bit(channel, self.bit_index, int(bit))
+        self.active_pixel[self.channel_index] = channel
+        self.bit_index -= 1
+
+        if self.bit_index < 0:
+            # We have exhausted our bitness for this channel.
+            self.bit_index = self.bitness - 1
+            self.channel_index += 1
+        if self.channel_index == 3:
+            # We have exhausted the channels for this pixel.
+            self.image.putpixel((self.x, self.y), tuple(self.active_pixel))
+            self.active_pixel = None
+
+    def write(self, bits):
+        for bit in bits:
+            self._write(bit)
+
+
+class ImageToBits:
+    def __init__(self, image, bitness):
+        self.image = image
+        self.width = image.size[0]
+        self.pixel_index = -1
+        self.active_byte = []
+
+    def _read(self):
+        if len(self.active_byte) == 0:
+            self.pixel_index += 1
+            (x, y) = index_to_xy(self.pixel_index, self.width)
+            self.active_byte = list(self.image.getpixel((x, y)))
+            self.active_byte = self.active_byte[:3]
+            self.active_byte = [binary(channel) for channel in self.active_byte]
+            self.active_byte = [channel[-bitness:] for channel in self.active_byte]
+            self.active_byte = ''.join(self.active_byte)
+            self.active_byte = list(self.active_byte)
+
+        ret = self.active_byte.pop(0)
+        return ret
+
+    def read(self, bits=1):
+        return ''.join(self._read() for x in range(bits)) 
+
+
 def binary(i):
     return bin(i)[2:].rjust(8, '0')
 
-def increment_pixel(save=True):
+def chunk_iterable(iterable, chunk_length, allow_incomplete=True):
     '''
-    Increment the active channel, and roll to the next pixel when appropriate.
+    Given an iterable, divide it into chunks of length `chunk_length`.
+    If `allow_incomplete` is True, the final element of the returned list may be shorter
+    than `chunk_length`. If it is False, those items are discarded.
     '''
-    global pixel
-    global pixel_index
-    global channel_index
-    channel_index += 1
-    if channel_index == 3:
-        channel_index = 0
-        if save:
-            image.putpixel((pixel_index % image.size[0], pixel_index // image.size[0]), tuple(pixel))
-            #print('wrote', pixel)
-        pixel_index += 1
-        pixel = list(image.getpixel( (pixel_index % image.size[0], pixel_index // image.size[0]) ))
-        #print('opend', pixel)
+    if len(iterable) % chunk_length != 0 and allow_incomplete:
+        overflow = 1
+    else:
+        overflow = 0
+
+    steps = (len(iterable) // chunk_length) + overflow
+    return [iterable[chunk_length * x : (chunk_length * x) + chunk_length] for x in range(steps)]
+
+def index_to_xy(index, width):
+    x = index % width
+    y = index // width
+    return (x, y)
 
 def bytes_to_pixels(bytes):
     return ((bytes * (8 / 3)) + 14)
@@ -81,6 +149,15 @@ def bytes_to_pixels(bytes):
 def pixels_to_bytes(pixels):
     return ((3 * (pixels - 14)) / 8)
 
+def set_bit(number, index, newvalue):
+    # Thanks unwind
+    # http://stackoverflow.com/a/12174051/5430534
+    mask = 1 << index
+    number &= ~mask
+    if newvalue:
+        number |= mask
+    return number    
+
 ##############    ####      ####        ########          ######        ##########        ##############
   ####      ##    ####      ####      ####    ####      ####  ####        ####  ####        ####      ##
   ####            ######    ####    ####      ####    ####      ####      ####    ####      ####        
@@ -90,7 +167,7 @@ def pixels_to_bytes(pixels):
   ####            ####    ######    ####      ####    ####      ####      ####    ####      ####        
   ####      ##    ####      ####      ####    ####      ####  ####        ####  ####        ####      ##
 ##############    ####      ####        ########          ######        ##########        ##############
-def encode(imagefilename, secretfilename):
+def encode(imagefilename, secretfilename, bitness=1):
     global image
     global pixel
     global pixel_index
@@ -98,83 +175,88 @@ def encode(imagefilename, secretfilename):
     pixel_index = 0
     channel_index = 0
 
-    def modify_pixel(bit):
-        global pixel
-        global channel_index
-        #print(channel_index, bit)
-        #print(pixel_index, channel_index, bit)
-        channel = pixel[channel_index]
-        channel = binary(channel)[:7] + bit
-        channel = int(channel, 2)
-        pixel[channel_index] = channel
-        #print(pixel)
-
+    if bitness < 1:
+        raise ValueError('Cannot modify less than 1 bit per channel')
+    if bitness > 8:
+        raise ValueError('Cannot modify more than 8 bits per channel')
 
     print('Hiding "%s" within "%s"' % (secretfilename, imagefilename))
+    secret_size = os.path.getsize(secretfilename)
+    if secret_size == 0:
+        raise StegError('The Secret can\'t be 0 bytes.')
+
     image = Image.open(imagefilename)
+    image_steg = BitsToImage(image, bitness)
 
     totalpixels = image.size[0] * image.size[1]
     if totalpixels < HEADER_SIZE:
         raise StegError('Image cannot have fewer than %d pixels. They are used to store Secret\'s length' % HEADER_SIZE)
 
-    secretfile = open(secretfilename, 'rb')
-    secret = secretfile.read()
-    secret = list(secret)
-
-    if secret == []:
-        raise StegError('The Secret can\'t be 0 bytes.')
-
     secret_extension = os.path.splitext(secretfilename)[1][1:]
-    secret_content_length = (len(secret)) + (len(secret_extension)) + 1
-    requiredpixels = math.ceil(((secret_content_length * 8) + 32) / 3)
+    secret_content_length = (secret_size) + (len(secret_extension)) + 1
+    requiredpixels = math.ceil(((secret_content_length * 8) + 32) / (3 * bitness))
     if totalpixels < requiredpixels:
         raise StegError('Image does not have enough pixels to store the Secret'
                         'Must have at least %d pixels' % requiredpixels)
 
-    print('%d available pixels, %d required' % (totalpixels, requiredpixels))
+    print('%d pixels available, %d required' % (totalpixels, requiredpixels))
 
     # --> YOU ARE HERE <--
+
+    # Because bitness may be between 1 and 8, we need to create a writing buffer
+    # called `binary_write_buffer`, so that we're always writing the same amount
+    # of data per color channel.
+    # If we were to write the secret length / extension on the fly, we might end
+    # up using the wrong number of bits for the final channel of some pixel.
+    # Example: 10010101 broken into groups of 3 is [100, 101, 01]
+    # Note that the last group is not the same size as the desired bitness, and
+    # will cause decode errors.
+
     pixel = list(image.getpixel((0, 0)))
+    binary_write_buffer = ''
 
     # Write secret length
     secret_content_length_b = binary(secret_content_length).rjust(32, '0')
-    for x in range(32):
-        modify_pixel(secret_content_length_b[x])
-        increment_pixel()
+    print('Content bytes:', secret_content_length)
+    image_steg.write(secret_content_length_b)
 
     # Write the secret extension
     for character in (secret_extension + chr(0)):
-        character = ord(character)
-        character = binary(character)
-        for bit in character:
-            modify_pixel(bit)
-            increment_pixel()
+        image_steg.write(binary(ord(character)))
 
     # Write the secret data
-    for (index, byte) in enumerate(secret):
-        if index % 1024 == 0:
-            percentage = (index + 1) / len(secret)
+    bytes_written = 0
+    done = False
+    secretfile = open(secretfilename, 'rb')
+    while not done:
+        if bytes_written % 1024 == 0:
+            percentage = (bytes_written + 1) / secret_size
             percentage = '%07.3f%%\r' % (100 * percentage)
             print(percentage, end='')
-        # Convert byte integer to a binary string, and loop through characters
-        byte = binary(byte)
-        for (bindex, bit) in enumerate(byte):
-            modify_pixel(bit)
-            if not (index == secret_content_length -1 and bindex == 7):
-                # If your Image dimensions are at the extreme limit of the Secret size,
-                # this would otherwise raise IndexError as it tries to grab the next pixel
-                # off the canvas.
-                increment_pixel()
-    print('100.000%')  # you know it
+
+        bytes = secretfile.read(FILE_READ_SIZE)
+
+        done = len(bytes) == 0
+
+        bytes = list(bytes)
+        bytes = [binary(byte) for byte in bytes]
+        bytes_written += len(bytes)
+        bytes = ''.join(bytes)
+        image_steg.write(bytes)
+
+    # haha
+    print('100.000%')
 
     if channel_index != 0:
         # The Secret data has finished, but we still have an unsaved pixel
         # (because channel_index is set to 0 when we save the active pixel above)
-        image.putpixel((pixel_index % image.size[0], pixel_index // image.size[0]), tuple(pixel))
+        (x, y) = index_to_xy(pixel_index, image.size[0])
+        image.putpixel((x, y), tuple(pixel))
 
-    newname = os.path.splitext(imagefilename)[0]
-    newname = '%s (%s).png' % (newname, os.path.basename(secretfilename).replace('.', '_'))
-    print(newname)
+    new_name = os.path.splitext(imagefilename)[0]
+    original_name = os.path.basename(secretfilename).replace('.', '_')
+    newname = '%s (%s) (%d).png' % (new_name, original_name, bitness)
+    print('Writing:', newname)
     image.save(newname)
 
 
@@ -188,49 +270,29 @@ def encode(imagefilename, secretfilename):
   ####    ####      ####            ####      ####    ####      ####      ####    ####      ####        
   ####  ####        ####      ##      ####    ####      ####  ####        ####  ####        ####      ##
 ##########        ##############        ########          ######        ##########        ##############
-def decode(imagefilename):
-    global image
-    global pixel
-    global pixel_index
-    global channel_index
-    pixel_index = 0
-    channel_index = 0
-    
+def decode(imagefilename, bitness=1):
+
     print('Extracting content from "%s"' % imagefilename)
     image = Image.open(imagefilename)
+    image_steg = ImageToBits(image, bitness)
 
     # determine the content length
-    content_length = ''
-    for x in range(11):
-        pixel = list(image.getpixel( (pixel_index % image.size[0], pixel_index // image.size[0]) ))
-        pixel = pixel[:3]
-        #print(pixel)
-        content_length += ''.join([bin(i)[-1] for i in pixel])
-        pixel_index += 1
-    content_length = content_length[:-1]
+    content_length = image_steg.read(32)
     content_length = int(content_length, 2)
     print('Content bytes:', content_length)
-    
-    # Continue from the end of the header
-    # This would have been automatic if I used `increment_pixel`
-    pixel_index = 10
-    channel_index = 2
 
     # determine secret extension
     extension = ''
     while extension[-8:] != '00000000' or len(extension) % 8 != 0:
-        channel = pixel[channel_index]
-        channel = binary(channel)
-        channel = channel[-1]
-        extension += channel
-        increment_pixel(save=False)
-    extension = extension[:-8]
-    extension = [extension[8*x: (8*x)+8] for x in range(len(extension)//8)]
+        extension += image_steg.read()
+
+    extension = chunk_iterable(extension, 8)
+    extension.remove('00000000')
     extension = [chr(int(x, 2)) for x in extension]
     extension = ''.join(extension)
     print('Extension:', extension)
 
-    # Remove the extension length
+    # Remove the extension length, and null byte
     content_length -= 1
     content_length -= len(extension)
 
@@ -242,38 +304,43 @@ def decode(imagefilename):
     outfile = open(newname, 'wb')
 
     # extract data
-    for byte in range(content_length):
-        if byte % 1024 == 0:
-            percentage = (byte + 1) / content_length
+    bytes_written = 0
+    while bytes_written < content_length:
+        if bytes_written % 1024 == 0:
+            percentage = (bytes_written + 1) / content_length
             percentage = '%07.3f%%\r' % (100 * percentage)
             print(percentage, end='')
 
-        activebyte = ''
-        for bit in range(8):
-            channel = pixel[channel_index]
-            channel = binary(channel)[-1]
-            activebyte += channel
-            if not (byte == content_length - 1 and bit == 7):
-                # If your Image dimensions are at the extreme limit of the Secret size,
-                # this would otherwise raise IndexError as it tries to grab the next pixel
-                # off the canvas.
-                increment_pixel(save=False)
-        activebyte = '%02x' % int(activebyte, 2)
-        outfile.write(binascii.a2b_hex(activebyte))
-    print('100.000%')  # I'm on fire
-    print(newname)
+        byte = image_steg.read(8)
+        byte = '%02x' % int(byte, 2)
+        outfile.write(binascii.a2b_hex(byte))
+        bytes_written += 1
+
+    # I'm on fire
+    print('100.000%')
+    print('Wrote', newname)
     outfile.close()
 
+def listget(li, index, fallback=None):
+    try:
+        return li[index]
+    except IndexError:
+        return fallback
+
 if __name__ == '__main__':
-    if (len(sys.argv) == 1) or (sys.argv[1] not in ['encode', 'decode']):
+    command = listget(sys.argv, 1, '').lower()
+    if command not in ['encode', 'decode']:
         print('Usage:')
-        print('> 3bitspixel.py encode imagefilename.png secretfilename.ext')
-        print('> 3bitspixel.py decode lacedimagename.png')
+        print('> steganographic.py encode imagefilename.png secretfilename.ext')
+        print('> steganographic.py decode lacedimagename.png')
+        quit()
 
     imagefilename = sys.argv[2]
 
-    if sys.argv[1] == 'encode':
+    if command == 'encode':
         secretfilename = sys.argv[3]
-        encode(imagefilename, secretfilename)
+        bitness = int(listget(sys.argv, 4, 1))
+        encode(imagefilename, secretfilename, bitness)
     else:
-        decode(imagefilename)
+        bitness = int(listget(sys.argv, 3, 1))
+        decode(imagefilename, bitness)

TKCube/README.md

@@ -1,4 +0,0 @@
-TKCube
-=========
-
-Not done yet.

TKCube/tkcube.py

@@ -1,117 +0,0 @@
-import copy
-import math
-import random
-import tkinter
-
-class TKCube:
-    def __init__(self):
-        self.t = tkinter.Tk()
-        self.FACES = [
-        [[2, 2, 1],    [2, -2, 1],   [-2, -2, 1], [-2, 2, 1]],
-        [[2, -2, 1],   [-2, -2, -1], [-2, 2, -1], [2, 2, -1]],
-        [[-2, -2, 1],  [2, -2, 1],   [2, -2, -1], [-2, -2, -1]],
-        [[-2, 2, 1],   [2, 2, 1],    [2, 2, -1],  [-2, 2, -1]],
-        [[-2, -2, -1], [-2, 2, -1],  [-2, 2, 1],  [-2, -2, -1]],
-        [[2, -2, 1],   [2, 2, 1],    [2, 2, -1],  [2, -2, -1]],
-        ]
-        self.INFLATE_SCALE = 8
-
-        self.c = tkinter.Canvas(self.t, width=600, height=600, bg='#444')
-        self.c.pack(fill='both', expand=True)
-        self.t.bind('<Return>', self.render)
-        self.is_mouse_down = False
-        self.prev_mouse_x = None
-        self.prev_mouse_y = None
-        self.t.bind('<ButtonPress-1>', self.mouse_down)
-        self.t.bind('<ButtonRelease-1>', self.mouse_up)
-        self.t.bind('<Motion>', self.mouse_motion)
-        self.t.bind('<Up>', lambda event: self.arbitrarymove(0, -1))
-        self.t.bind('<Down>', lambda event: self.arbitrarymove(0, 1))
-        self.t.bind('<Left>', lambda event: self.arbitrarymove(-1, 0))
-        self.t.bind('<Right>', lambda event: self.arbitrarymove(1, 0))
-        self.render()
-        self.t.mainloop()
-
-    def arbitrarymove(self, deltax, deltay):
-        for face in self.FACES:
-            for point in face:
-                point[0] += deltax
-                point[1] += deltay
-        self.render()
-
-    def mouse_down(self, event):
-        self.is_mouse_down = True
-
-    def mouse_up(self, event):
-        self.is_mouse_down = False
-
-    def mouse_motion(self, event):
-        if not self.is_mouse_down:
-            return
-        if self.prev_mouse_x is None:
-            self.prev_mouse_x = event.x
-            self.prev_mouse_y = event.y
-        distance = math.sqrt( ((event.x - self.prev_mouse_x) ** 2) + ((event.y - self.prev_mouse_y) ** 2) )
-        self.prev_mouse_x = event.x
-        self.prev_mouse_y = event.y
-        print(distance)
-
-    def center_of_square(self, face):
-        x = 0; y = 0; z = 0
-        for point in face:
-            x += point[0]
-            y += point[1]
-            z += point[2]
-        return [x/4, y/4, z/4]
-
-    def plot_point_screen(self, x, y, diameter=4):
-        radius = diameter / 2
-        x1 = x - radius
-        y1 = y - radius
-        x2 = x + radius
-        y2 = y + radius
-        self.c.create_oval(x1, y1, x2, y2, fill='#000')
-
-    def render(self, *event):
-        self.c.delete('all')
-        rendered_faces = copy.deepcopy(self.FACES)
-
-        # Sort by depth from camera
-        # The sort key is the z value of the coordinate
-        # in the center of the face
-        rendered_faces.sort(key=lambda face: self.center_of_square(face)[2])
-
-        canvas_width_half = self.c.winfo_width() / 2
-        canvas_height_half = self.c.winfo_height() / 2
-        highest_z = max([max([point[2] for point in face]) for face in rendered_faces])
-        for face in self.FACES:
-            for point in face:
-                x = point[0]
-                y = point[1]
-                z = point[2]
-
-                # Push everything away from the camera so all z are <= 0
-                z -= highest_z
-
-                # Create vanishing point.
-                distance_camera = math.sqrt((x**2) + (y**2) + (z**2))
-                #if z != 0:
-                #    factor = (abs(z) ** 0.2) - 1
-                #    print(factor)
-                #else:
-                #    factor = 0
-                x += x * factor
-                y += y * factor
-                
-                # Inflate for display
-                x *= self.INFLATE_SCALE
-                y *= self.INFLATE_SCALE
-                z *= self.INFLATE_SCALE
-
-                # Shift the coordinates into the screen
-                x += canvas_width_half
-                y += canvas_height_half
-                self.plot_point_screen(x, y)
-
-        #print(rendered_faces)
-t = TKCube()

Toddo/toddo.py

@@ -4,6 +4,13 @@ import sqlite3
 import sys
 import textwrap
 
+try:
+    import colorama
+    colorama.init()
+    HAS_COLORAMA = True
+except:
+    HAS_COLORAMA = False
+
 SQL_ID = 0
 SQL_TODOTABLE = 1
 SQL_CREATED = 2
@@ -28,6 +35,7 @@ Use `toddo all` to see if there are entries for other tables.'''
 
 HELP_REMOVE = '''Provide an ID number to remove.'''
 
+# The newline at the top of this message is intentional
 DISPLAY_INDIVIDUAL = '''
      ID: _id_
   Table: _table_
@@ -58,6 +66,24 @@ class Toddo():
             self._cur.execute('CREATE INDEX IF NOT EXISTS todoindex on todos(id)')
         return self._cur
 
+    def _install_default_lastid(self):
+        self.cur.execute('SELECT val FROM meta WHERE key == "lastid"')
+        f = cur.fetchone()
+        if f is not None:
+            return int(f[0])
+        self.cur.execute('INSERT INTO meta VALUES("lastid", 1)')
+        self.sql.commit()
+        return 1
+
+    def _install_default_todotable(self):
+        self.cur.execute('SELECT val FROM meta WHERE key == "todotable"')
+        f = cur.fetchone()
+        if f is not None:
+            return f[0]
+        self.cur.execute('INSERT INTO meta VALUES("todotable", "default")')
+        self.sql.commit()
+        return 'default'
+
     def add_todo(self, message=None):
         '''
         Create new entry in the database on the active todotable.
@@ -113,7 +139,6 @@ class Toddo():
         output = output.replace('_human_', human(todo[SQL_CREATED]))
         output = output.replace('_message_', message)
 
-
         return output
 
     def display_active_todos(self):
@@ -154,13 +179,38 @@ class Toddo():
             if '\n' in message:
                 message = message.split('\n')[0] + ' ...'
 
-            total = '%s : %s : %s : %s' % (todoid, todotable, timestamp, message)
             terminal_width = shutil.get_terminal_size()[0]
+            total = '%s : %s : %s' % (timestamp, todoid, message)
+            space_remaining = terminal_width - len(total)
             if len(total) > terminal_width:
                 total = total[:(terminal_width-(len(total)+4))] + '...'
             display.append(total)
 
         return '\n'.join(display)
+
+    def get_todotable(self):
+        self.cur.execute('SELECT val FROM meta WHERE key="todotable"')
+        todotable = self.cur.fetchone()
+        if todotable is None:
+            self._install_default_todotable()
+            todotable = 'default'
+        else:
+            todotable = todotable[0]
+        return todotable
+
+    def increment_lastid(self, increment=False):
+        '''
+        Increment the lastid in the meta table, THEN return it.
+        '''
+        self.cur.execute('SELECT val FROM meta WHERE key="lastid"')
+        lastid = self.cur.fetchone()
+        if lastid is None:
+            self._install_default_lastid()
+            return 1
+        else:
+            lastid = int(lastid[0]) + 1
+            self.cur.execute('UPDATE meta SET val=? WHERE key="lastid"', [lastid])
+            return lastid
     
     def switch_todotable(self, newtable=None):
         '''
@@ -178,47 +228,24 @@ class Toddo():
         self.sql.commit()
         return newtable
 
-    def increment_lastid(self, increment=False):
-        '''
-        Increment the lastid in the meta table, THEN return it.
-        '''
-        self.cur.execute('SELECT val FROM meta WHERE key="lastid"')
-        lastid = self.cur.fetchone()
-        if lastid is None:
-            self._install_default_lastid()
-            return 1
-        else:
-            lastid = int(lastid[0]) + 1
-            self.cur.execute('UPDATE meta SET val=? WHERE key="lastid"', [lastid])
-            return lastid
-
-    def get_todotable(self):
-        self.cur.execute('SELECT val FROM meta WHERE key="todotable"')
-        todotable = self.cur.fetchone()
-        if todotable is None:
-            self._install_default_todotable()
-            todotable = 'default'
-        else:
-            todotable = todotable[0]
-        return todotable
-
-    def _install_default_lastid(self):
-        '''
-        This method assumes that "lastid" does not already exist.
-        If it does, it's your fault for calling this.
-        '''
-        self.cur.execute('INSERT INTO meta VALUES("lastid", 1)')
-        self.sql.commit()
-        return 1
-
-    def _install_default_todotable(self):
-        '''
-        This method assumes that "todotable" does not already exist.
-        If it does, it's your fault for calling this.
-        '''
-        self.cur.execute('INSERT INTO meta VALUES("todotable", "default")')
-        self.sql.commit()
-        return 'default'
+def colorama_print(text):
+    alternator = False
+    terminal_size = shutil.get_terminal_size()[0]
+    for line in text.split('\n'):
+        line += ' ' * (terminal_size - (len(line)+1))
+        if HAS_COLORAMA:
+            if alternator:
+                sys.stdout.write(colorama.Fore.BLACK)
+                sys.stdout.write(colorama.Back.WHITE)
+            else:
+                sys.stdout.write(colorama.Fore.WHITE)
+                sys.stdout.write(colorama.Back.BLACK)
+        alternator = not alternator
+        print(line)
+    if HAS_COLORAMA:
+        sys.stdout.write(colorama.Back.RESET)
+        sys.stdout.write(colorama.Fore.RESET)
+        sys.stdout.flush()
 
 def human(timestamp):
     timestamp = datetime.datetime.utcfromtimestamp(timestamp)
@@ -282,14 +309,14 @@ if __name__ == '__main__':
             table = toddo.get_todotable()
             print(HELP_NOACTIVE % table)
         else:
-            print(message)
+            colorama_print(message)
 
     elif sys.argv[1] == 'all':
         message = toddo.display_todos_from_table(None)
         if message is None:
             print(HELP_NOENTRIES)
         else:
-            print(message)
+            colorama_print(message)
 
     elif sys.argv[1] == 'add':
         args = list(filter(None, sys.argv))

Toolbox/filenameorderedrandomness.pyw

@@ -4,22 +4,23 @@ name randomly scrambled into 12 digits. The others will increment that number b
 1.
 '''
 
+print('hi')
 import os
 import random
 import string
 import sys
 
 argv = sys.argv[1:]
-print(argv)
+print(''.join(c for c in argv if c in string.printable))
 
 randname = [random.choice(string.digits) for x in range(12)]
 randname = int(''.join(randname))
-for originalname in argv:
-    folder = os.path.dirname(originalname)
-    basename = os.path.basename(originalname)
-    extension = basename.split('.')[-1]
-    newname = randname
+for filepath in argv:
+    folder = os.path.dirname(filepath)
+    basename = os.path.basename(filepath)
+    extension = os.path.splitext(basename)[1]
+    newname = str(randname).rjust(12, '0')
     randname += 1
-    newname = '%s/%d.%s' % (folder, newname, extension)
-    print('%s -> %s' % (originalname, newname))
-    os.rename(originalname, newname)
+    newname = '%s\\%s%s' % (folder, newname, extension)
+    os.rename(filepath, newname)
+    print('%s -> %s' % (filepath, newname))

Toolbox/filenamescramble.pyw

@@ -10,13 +10,13 @@ import string
 import sys
 
 argv = sys.argv[1:]
-print(argv)
-for originalname in argv:
-    folder = os.path.dirname(originalname)
-    basename = os.path.basename(originalname)
-    extension = basename.split('.')[-1]
+print(''.join(c for c in argv if c in string.printable))
+for filepath in argv:
+    folder = os.path.dirname(filepath)
+    basename = os.path.basename(filepath)
+    extension = os.path.splitext(basename)[1]
     newname = [random.choice(string.ascii_letters) for x in range(16)]
     newname = ''.join(newname)
-    newname = '%s/%s.%s' % (folder, newname, extension)
-    print('%s -> %s' % (originalname, newname))
-    os.rename(originalname, newname)
+    newname = '%s\\%s%s' % (folder, newname, extension)
+    os.rename(filepath, newname)
+    print('%s -> %s' % (filepath, newname))

Toolbox/filenamescrambleint.pyw

@@ -9,13 +9,13 @@ import string
 import sys
 
 argv = sys.argv[1:]
-print(argv)
-for originalname in argv:
-    folder = os.path.dirname(originalname)
-    basename = os.path.basename(originalname)
-    extension = basename.split('.')[-1]
+print(''.join(c for c in argv if c in string.printable))
+for filepath in argv:
+    folder = os.path.dirname(filepath)
+    basename = os.path.basename(filepath)
+    extension = os.path.splitext(basename)[1]
     newname = [random.choice(string.digits) for x in range(12)]
     newname = ''.join(newname)
-    newname = '%s/%s.%s' % (folder, newname, extension)
-    print('%s -> %s' % (originalname, newname))
-    os.rename(originalname, newname)
+    newname = '%s\\%s%s' % (folder, newname, extension)
+    os.rename(filepath, newname)
+    print('%s -> %s' % (filepath, newname))

Toolbox/fileprefix.py

@@ -31,7 +31,6 @@ zeropadding = max(2, zeropadding)
 zeropadding = str(zeropadding)
 
 format = '%s%0{pad}d%s'.format(pad=zeropadding)
-print(format)
 
 def natural_sort(l):
     '''
@@ -55,5 +54,6 @@ for (fileindex, filename) in enumerate(files):
     else:
         extension = ''
     newname = format % (prefix, fileindex, extension)
-    print(''.join([c for c in filename if c in string.printable]), '->', newname)
-    os.rename(filename, newname)
+    if os.path.basename(filename) != newname:
+        print(''.join([c for c in (filename + ' -> ' + newname) if c in string.printable]))
+        os.rename(filename, newname)

Toolbox/timestampfilename.pyw

@@ -0,0 +1,25 @@
+'''
+Drag a file on top of this .py file, and it will
+be renamed to the current timestamp.
+'''
+
+import datetime
+import os
+import sys
+
+STRFTIME = '%Y%m%d %H%M%S'
+UTC = True
+
+filename = sys.argv[1]
+folder = os.path.dirname(filename)
+if folder == '':
+    folder = os.getcwd()
+basename = os.path.basename(filename)
+extension = os.path.splitext(basename)[1]
+
+now = datetime.datetime.now(datetime.timezone.utc if UTC else None)
+newname = now.strftime(STRFTIME)
+
+newname = '%s\\%s%s' % (folder, newname, extension)
+print(filename, '-->', newname)
+os.rename(filename, newname)