voussoir/else
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

else

Browse source
This commit is contained in:
Ethan Dalool 2017-01-15 22:04:20 -08:00
parent a9f661ddb4
commit fa2c2bda76
18 changed files with 2085 additions and 1667 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
BaseNumber/basenumber.py
View file

@ -9,7 +9,7 @@ def from_base(number, base, alphabet=None):
raise TypeError('base must be an int.')
if base == 10:
return number
return int(number)
if alphabet is None:
alphabet = ALPHABET

138
Fusker/fusker.py Normal file
View file

@ -0,0 +1,138 @@
import collections
import itertools
import string
import sys
from voussoirkit import basenumber
class Landmark:
def __init__(self, opener, closer, parser):
self.opener = opener
self.closer = closer
self.parser = parser
def barsplit(chars):
wordlist = []
wordbuff = []
def flush():
if not wordbuff:
return
word = fusk_join(wordbuff)
wordlist.append(word)
wordbuff.clear()
for item in chars:
if item == '|':
flush()
else:
wordbuff.append(item)
flush()
return wordlist
def fusk_join(items):
form = ''
fusks = []
result = []
for item in items:
if isinstance(item, str):
form += item
else:
form += '{}'
fusks.append(item)
product = itertools.product(*fusks)
for group in product:
f = form.format(*group)
result.append(f)
return result
def fusk_spinner(items):
for item in items:
if isinstance(item, str):
yield item
else:
yield from item
def parse_spinner(characters):
words = barsplit(characters)
spinner = fusk_spinner(words)
return spinner
def fusk_range(lo, hi, padto=0, base=10, lower=False):
for x in range(lo, hi+1):
x = basenumber.to_base(x, base)
x = x.rjust(padto, '0')
if lower:
x = x.lower()
yield x
def parse_range(characters):
r = ''.join(characters)
(lo, hi) = r.split('-')
lo = lo.strip()
hi = hi.strip()
lowers = string.digits + string.ascii_lowercase
uppers = string.digits + string.ascii_uppercase
lohi = lo + hi
lower = False
if all(c in string.digits for c in lohi):
base = 10
elif all(c in lowers for c in lohi):
lower = True
base = 36
elif all(c in uppers for c in lohi):
base = 36
else:
base = 62
if (not lo) or (not hi):
raise ValueError('Invalid range', r)
if len(lo) > 1 and lo.startswith('0'):
padto = len(lo)
if len(hi) != padto:
raise ValueError('Inconsistent padding', lo, hi)
else:
padto = 0
lo = basenumber.from_base(lo, base)
hi = basenumber.from_base(hi, base)
frange = fusk_range(lo, hi, padto=padto, base=base, lower=lower)
return frange
landmarks = {
'{': Landmark('{', '}', parse_spinner),
'[': Landmark('[', ']', parse_range),
}
def fusker(fstring, landmark=None, depth=0):
escaped = False
result = []
buff = []
if isinstance(fstring, str):
fstring = collections.deque(fstring)
while fstring:
character = fstring.popleft()
if escaped:
buff.append('\\' + character)
escaped = False
elif character == '\\':
escaped = True
elif landmark and character == landmark.closer:
buff = [landmark.parser(buff)]
break
elif character in landmarks:
subtotal = fusker(fstring, landmark=landmarks[character])
buff.extend(subtotal)
else:
buff.append(character)
if not landmark:
buff = parse_spinner(buff)
return buff
return result
if __name__ == '__main__':
pattern = sys.argv[1]
fusk = fusker(pattern)
for result in fusk:
print(result)

9
OpenDirDL/README.md
View file

@ -3,9 +3,9 @@ Open Dir DL
The open directory downloader.
## Installation
## Installing requirements
pip install requirements.txt
pip install -r requirements.txt
## Usage
@ -17,12 +17,15 @@ See inside opendirdl.py for usage instructions.
- **[addition]** A new feature was added.
- **[bugfix]** Incorrect behavior was fixed.
- **[change]** An existing feature was slightly modified or parameters were renamed.
- **[change]** An existing feature was modified or parameters were renamed.
- **[cleanup]** Code was improved, comments were added, or other changes with minor impact on the interface.
- **[removal]** An old feature was removed.
 
- 2017 01 06
- **[cleanup]** Much of the file tree builder has been moved to a new file in the voussoirkit, `pathtree.py`. The goal is to generalize the concept of a pathtree so I can start using it in other local-disk applications instead of having it locked into opendirdl's URL database. They were entangled very badly so it's still messy until I get the division of labor sorted out. Running the `tree` command on the commandline still works exactly the same as before.
- 2016 11 11
- **[addition]** You can now call opendirdl using the database filename as the first argument, and the subcommand as the second. Previously, the subcommand always had to come first, but now they are interchangeable when the system detects that argv[0] is a file. This makes it much easier to do multiple operations on a single database because you can just backspace the previous command rather than having to hop over the database name to get to it.
- **[addition]** `measure` now takes an argument `--threads x` to use `x` threads during the head requests.

336
OpenDirDL/opendirdl.py
View file

@ -133,17 +133,20 @@ import requests
import shutil
import sqlite3
import sys
import time
## import tkinter
import urllib.parse
# pip install voussoirkit
from voussoirkit import bytestring
from voussoirkit import downloady
from voussoirkit import fusker
from voussoirkit import treeclass
from voussoirkit import pathtree
DOWNLOAD_CHUNK = 16 * bytestring.KIBIBYTE
FILENAME_BADCHARS = '/\\:*?"<>|'
TERMINAL_WIDTH = shutil.get_terminal_size().columns
UNKNOWN_SIZE_STRING = '???'
# When doing a basic scan, we will not send HEAD requests to URLs that end in
# these strings, because they're probably files.
@ -192,6 +195,10 @@ SKIPPABLE_FILETYPES = [
'.zip',
]
SKIPPABLE_FILETYPES = set(x.lower() for x in SKIPPABLE_FILETYPES)
SKIPPABLE_FILETYPES.update(fusker.fusker('.r[0-99]'))
SKIPPABLE_FILETYPES.update(fusker.fusker('.r[00-99]'))
SKIPPABLE_FILETYPES.update(fusker.fusker('.r[000-099]'))
SKIPPABLE_FILETYPES.update(fusker.fusker('.[00-99]'))
# Will be ignored completely. Are case-sensitive
BLACKLISTED_FILENAMES = [
@ -199,62 +206,6 @@ BLACKLISTED_FILENAMES = [
'thumbs.db',
]
# oh shit
HTML_TREE_HEAD = '''
<head>
<meta charset="UTF-8">
<script type="text/javascript">
function collapse(div)
{
if (div.style.display != "none")
{
div.style.display = "none";
}
else
{
div.style.display = "block";
}
}
</script>
<style>
*
{
font-family: Consolas;
}
.directory_even, .directory_odd
{
padding: 10px;
padding-left: 15px;
margin-bottom: 10px;
border: 1px solid #000;
box-shadow: 1px 1px 2px 0px rgba(0,0,0,0.3);
}
.directory_even
{
background-color: #fff;
}
.directory_odd
{
background-color: #eee;
}
</style>
</head>
'''
HTML_FORMAT_DIRECTORY = '''
<div class="buttonbox">
<button onclick="collapse(this.parentElement.nextElementSibling)">{name} ({size})</button>
{directory_anchor}
</div>
<div class="{css}" style="display:none">
'''.replace('\n', '')
HTML_FORMAT_FILE = '<a href="{url}">{name} ({size})</a><br>'
DB_INIT = '''
CREATE TABLE IF NOT EXISTS urls(
url TEXT,
@ -435,92 +386,6 @@ class Walker:
## WALKER ##########################################################################################
## OTHER CLASSES ###################################################################################
## ##
class TreeExistingChild(Exception):
pass
class TreeInvalidIdentifier(Exception):
pass
class TreeNode:
def __init__(self, identifier, data=None):
assert isinstance(identifier, str)
assert '\\' not in identifier
self.identifier = identifier
self.data = data
self.parent = None
self.children = {}
def __eq__(self, other):
return isinstance(other, TreeNode) and self.abspath() == other.abspath()
def __getitem__(self, key):
return self.children[key]
def __hash__(self):
return hash(self.abspath())
def __repr__(self):
return 'TreeNode %s' % self.abspath()
def abspath(self):
node = self
nodes = [node]
while node.parent is not None:
node = node.parent
nodes.append(node)
nodes.reverse()
nodes = [node.identifier for node in nodes]
return '\\'.join(nodes)
def add_child(self, other_node, overwrite_parent=False):
self.check_child_availability(other_node.identifier)
if other_node.parent is not None and not overwrite_parent:
raise ValueError('That node already has a parent. Try `overwrite_parent=True`')
other_node.parent = self
self.children[other_node.identifier] = other_node
return other_node
def check_child_availability(self, identifier):
if identifier in self.children:
raise TreeExistingChild('Node %s already has child %s' % (self.identifier, identifier))
def detach(self):
del self.parent.children[self.identifier]
self.parent = None
def list_children(self, customsort=None):
children = list(self.children.values())
if customsort is None:
children.sort(key=lambda node: node.identifier.lower())
else:
children.sort(key=customsort)
return children
def merge_other(self, othertree, otherroot=None):
newroot = None
if ':' in othertree.identifier:
if otherroot is None:
raise Exception('Must specify a new name for the other tree\'s root')
else:
newroot = otherroot
else:
newroot = othertree.identifier
othertree.identifier = newroot
othertree.parent = self
self.check_child_availability(newroot)
self.children[newroot] = othertree
def walk(self, customsort=None):
yield self
for child in self.list_children(customsort=customsort):
yield from child.walk(customsort=customsort)
## ##
## OTHER CLASSES ###################################################################################
## GENERAL FUNCTIONS ###############################################################################
## ##
def build_file_tree(databasename):
@ -533,67 +398,22 @@ def build_file_tree(databasename):
if len(fetch_all) == 0:
return
path_form = '{domain}\\{folder}\\{filename}'
all_items = []
path_datas = []
# :|| is my temporary (probably not temporary) hack for including the URL
# scheme without causing the pathtree processor to think there's a top
# level directory called 'http'.
# It will be replaced with :// in the calling `tree` function.
path_form = '{scheme}:||{domain}\\{folder}\\{filename}'
for item in fetch_all:
url = item[SQL_URL]
size = item[SQL_CONTENT_LENGTH]
path_parts = url_split(item[SQL_URL])
path_parts = path_form.format(**path_parts)
#path_parts = urllib.parse.unquote(path_parts)
path_parts = path_parts.split('\\')
item = {'url': url, 'size': size, 'path_parts': path_parts}
all_items.append(item)
path = path_form.format(**path_parts)
path = urllib.parse.unquote(path)
path_data = {'path': path, 'size': size, 'data': url}
path_datas.append(path_data)
all_items.sort(key=lambda x: x['url'])
root_data = {
'item_type': 'directory',
'name': databasename,
}
scheme = url_split(all_items[0]['url'])['scheme']
tree_root = TreeNode(databasename, data=root_data)
tree_root.unsorted_children = all_items
node_queue = set()
node_queue.add(tree_root)
# In this process, URLs are divided up into their nodes one directory layer at a time.
# The root receives all URLs, and creates nodes for each of the top-level
# directories. Those nodes receive all subdirectories, and repeat.
while len(node_queue) > 0:
node = node_queue.pop()
for new_child_data in node.unsorted_children:
path_parts = new_child_data['path_parts']
# Create a new node for the directory, path_parts[0]
# path_parts[1:] is assigned to that node to be divided next.
child_identifier = path_parts.pop(0)
#child_identifier = child_identifier.replace(':', '#')
child = node.children.get(child_identifier, None)
if not child:
child = TreeNode(child_identifier, data={})
child.unsorted_children = []
node.add_child(child)
child.data['name'] = child_identifier
if len(path_parts) > 0:
child.data['item_type'] = 'directory'
child.unsorted_children.append(new_child_data)
node_queue.add(child)
else:
child.data['item_type'] = 'file'
child.data['size'] = new_child_data['size']
child.data['url'] = new_child_data['url']
if node.parent is None:
continue
elif node.parent == tree_root:
node.data['url'] = scheme + '://' + node.identifier
else:
node.data['url'] = node.parent.data['url'] + '/' + node.identifier
del node.unsorted_children
return tree_root
return pathtree.from_paths(path_datas, root_name=databasename)
def db_init(sql, cur):
lines = DB_INIT.split(';')
@ -652,83 +472,15 @@ def int_none(x):
return x
return int(x)
def recursive_get_size(node):
'''
Calculate the size of the Directory nodes by summing the sizes of all children.
Modifies the nodes in-place.
'''
return_value = {
'size': 0,
'unmeasured': 0,
}
if node.data['item_type'] == 'file':
if node.data['size'] is None:
return_value['unmeasured'] = 1
return_value['size'] = node.data['size']
else:
for child in node.list_children():
child_details = recursive_get_size(child)
return_value['size'] += child_details['size'] or 0
return_value['unmeasured'] += child_details['unmeasured']
node.data['size'] = return_value['size']
return return_value
def recursive_print_node(node, depth=0, use_html=False, output_file=None):
'''
Given a tree node (presumably the root), print it and all of its children.
'''
size = node.data['size']
if size is None:
size = UNKNOWN_SIZE_STRING
else:
size = bytestring.bytestring(size)
if use_html:
css_class = 'directory_even' if depth % 2 == 0 else 'directory_odd'
if node.data['item_type'] == 'directory':
directory_url = node.data.get('url')
directory_anchor = '<a href="{url}">►</a>' if directory_url else ''
directory_anchor = directory_anchor.format(url=directory_url)
line = HTML_FORMAT_DIRECTORY.format(
css=css_class,
directory_anchor=directory_anchor,
name=node.data['name'],
size=size,
)
else:
line = HTML_FORMAT_FILE.format(
name=node.data['name'],
size=size,
url=node.data['url'],
)
else:
line = '{space}{bar}{name} : ({size})'
line = line.format(
space='| ' * (depth-1),
bar='|---' if depth > 0 else '',
name=node.data['name'],
size=size
)
write(line, output_file)
# Sort by type (directories first) then subsort by lowercase path
customsort = lambda node: (
node.data['item_type'] == 'file',
node.data['url'].lower(),
)
for child in node.list_children(customsort=customsort):
recursive_print_node(child, depth=depth+1, use_html=use_html, output_file=output_file)
if node.data['item_type'] == 'directory':
if use_html:
# Close the directory div
write('</div>', output_file)
else:
# This helps put some space between sibling directories
write('| ' * (depth), output_file)
def promise_results(promises):
promises = promises[:]
while len(promises) > 0:
for (index, promise) in enumerate(promises):
if not promise.done():
continue
yield promise.result()
promises.pop(index)
break
def safeindex(sequence, index, fallback=None):
try:
@ -1085,18 +837,14 @@ def measure(databasename, fullscan=False, new_only=False, threads=4):
else:
totalsize += size
while len(thread_promises) > 0:
# If that thread is done, `result()` will return immediately
# Otherwise, it will wait, which is okay because the threads themselves
# are not blocked.
head = thread_promises.pop(0).result()
for head in promise_results(thread_promises):
fetch = smart_insert(sql, cur, head=head, commit=True)
size = fetch[SQL_CONTENT_LENGTH]
if size is None:
write('"%s" is not revealing Content-Length' % url)
size = 0
totalsize += size
except KeyboardInterrupt:
except (Exception, KeyboardInterrupt):
for promise in thread_promises:
promise.cancel()
raise
@ -1141,6 +889,11 @@ def tree(databasename, output_filename=None):
be a plain text drawing.
'''
tree_root = build_file_tree(databasename)
tree_root.path = None
for node in tree_root.walk():
if node.path:
node.path = node.path.replace(':||', '://')
node.display_name = node.display_name.replace(':||', '://')
if output_filename is not None:
output_file = open(output_filename, 'w', encoding='utf-8')
@ -1149,19 +902,20 @@ def tree(databasename, output_filename=None):
output_file = None
use_html = False
if use_html:
write('<!DOCTYPE html>\n<html>', output_file)
write(HTML_TREE_HEAD, output_file)
write('<body>', output_file)
size_details = pathtree.recursive_get_size(tree_root)
size_details = recursive_get_size(tree_root)
recursive_print_node(tree_root, use_html=use_html, output_file=output_file)
if size_details['unmeasured'] > 0:
write(UNMEASURED_WARNING % size_details['unmeasured'], output_file)
footer = UNMEASURED_WARNING % size_details['unmeasured']
else:
footer = None
line_generator = pathtree.recursive_print_node(tree_root, use_html=use_html, footer=footer)
for line in line_generator:
write(line, output_file)
if output_file is not None:
if use_html:
write('</body>\n</html>', output_file)
output_file.close()
return tree_root

2
OpenDirDL/requirements.txt
View file

@ -1,3 +1,3 @@
https://github.com/voussoir/else/raw/master/_voussoirkit/voussoirkit.zip
voussoirkit
bs4
requests

4
Piecewise/piecewise.py
View file

@ -8,6 +8,7 @@ import traceback
from voussoirkit import bytestring
from voussoirkit import downloady
from voussoirkit import clipext
DEFAULT_PIECE_SIZE = bytestring.MIBIBYTE
DEFAULT_THREAD_COUNT = 10
@ -133,7 +134,8 @@ def init_argparse(args):
piece_size = bytestring.parsebytes(args.piece_size)
else:
piece_size = args.piece_size
init(args.url, localname=args.localname, piece_size=piece_size)
url = clipext.resolve(args.url)
init(url, localname=args.localname, piece_size=piece_size)
def reset_argparse(args):
reset(args.databasename)

2
RateMeter/speedtest.py
View file

@ -1,5 +1,5 @@
from voussoirkit import bytestring
import downloady
from voussoirkit import downloady
import ratemeter
import requests
import sys

1470
Steganographic/asciibet.py Normal file
View file

File diff suppressed because it is too large Load diff

1310
Steganographic/asciibet.txt
View file

File diff suppressed because it is too large Load diff

9
ThreadedDL/threaded_dl.py
View file

@ -35,10 +35,11 @@ def threaded_dl(urls, thread_count, filename_format=None):
if filename_format is None:
filename_format = '{now}_{index}_{basename}'
filename_format = filename_format.replace('{index}', '{index:0%0dd}' % index_digits)
if '{' not in filename_format and len(urls) > 1:
filename_format += '_{index}'
if '{extension}' not in filename_format:
filename_format += '{extension}'
if filename_format != os.devnull:
if '{' not in filename_format and len(urls) > 1:
filename_format += '_{index}'
if '{extension}' not in filename_format:
filename_format += '{extension}'
now = int(time.time())
for (index, url) in enumerate(urls):
while len(threads) == thread_count:

46
Toolbox/ffrename.py
View file

@ -1,46 +0,0 @@
import converter
import glob
import os
import re
import subprocess
import sys
import time
def main(filename):
ffmpeg = converter.Converter()
probe = ffmpeg.probe(filename)
new_name = filename
if '_x_' in filename:
dimensions = '%dx%d' % (probe.video.video_width, probe.video.video_height)
new_name = new_name.replace('_x_', dimensions)
if '___' in filename:
video_codec = probe.video.codec
audios = [stream for stream in probe.streams if stream.type == 'audio' and stream.bitrate]
if audios:
audio = max(audios, key=lambda x: x.bitrate)
audio_codec = probe.audio.codec
else:
audio_codec = None
if any(not x for x in [video_codec, probe.video.bitrate, audio_codec, probe.audio.bitrate]):
print('Could not identify media info')
else:
video_bitrate = probe.video.bitrate // 1000
audio_bitrate = probe.audio.bitrate // 1000
video = '%s-%d' % (video_codec, video_bitrate)
audio = '%s-%d' % (audio_codec, audio_bitrate)
video = video.upper()
audio = audio.upper()
video = video.replace('H264', 'h264')
video = video.replace('HEVC', 'h265')
info = '{v}, {a}'.format(v=video, a=audio)
new_name = new_name.replace('___', info)
print(new_name)
if input('Okay?').lower() in ['y', 'yes']:
os.rename(filename, new_name)
if __name__ == '__main__':
for filename in glob.glob(sys.argv[1]):
main(filename)

32
Toolbox/search.py
View file

@ -7,9 +7,15 @@ import sys
from voussoirkit import safeprint
from voussoirkit import spinal
def search(terms, match_any=False, do_regex=False, do_glob=False):
search_terms = [term.lower() for term in terms]
def search(
terms,
*,
case_sensitive=False,
do_regex=False,
do_glob=False,
local_only=False,
match_any=False,
):
def term_matches(text, term):
return (
(term in text) or
@ -17,11 +23,23 @@ def search(terms, match_any=False, do_regex=False, do_glob=False):
(do_glob and fnmatch.fnmatch(text, term))
)
if case_sensitive:
search_terms = terms
else:
search_terms = [term.lower() for term in terms]
anyall = any if match_any else all
generator = spinal.walk_generator(depth_first=False, yield_directories=True)
generator = spinal.walk_generator(
depth_first=False,
recurse=not local_only,
yield_directories=True,
)
for filepath in generator:
basename = filepath.basename.lower()
basename = filepath.basename
if not case_sensitive:
basename = basename.lower()
if anyall(term_matches(basename, term) for term in search_terms):
safeprint.safeprint(filepath.absolute_path)
@ -29,8 +47,10 @@ def search(terms, match_any=False, do_regex=False, do_glob=False):
def search_argparse(args):
return search(
terms=args.search_terms,
case_sensitive=args.case_sensitive,
do_glob=args.do_glob,
do_regex=args.do_regex,
local_only=args.local_only,
match_any=args.match_any,
)
@ -41,6 +61,8 @@ def main(argv):
parser.add_argument('--any', dest='match_any', action='store_true')
parser.add_argument('--regex', dest='do_regex', action='store_true')
parser.add_argument('--glob', dest='do_glob', action='store_true')
parser.add_argument('--case', dest='case_sensitive', action='store_true')
parser.add_argument('--local', dest='local_only', action='store_true')
parser.set_defaults(func=search_argparse)
args = parser.parse_args(argv)

281
Treeclass/pathtree.py Normal file
View file

@ -0,0 +1,281 @@
import argparse
import sys
import os
from voussoirkit import bytestring
from voussoirkit import treeclass
HTML_TREE_HEAD = '''
<head>
<meta charset="UTF-8">
<script type="text/javascript">
function collapse(div)
{
if (div.style.display != "none")
{
div.style.display = "none";
}
else
{
div.style.display = "block";
}
}
</script>
<style>
*
{
font-family: Consolas;
}
.directory_even, .directory_odd
{
padding: 10px;
padding-left: 15px;
margin-bottom: 10px;
border: 1px solid #000;
box-shadow: 1px 1px 2px 0px rgba(0,0,0,0.3);
}
.directory_even
{
background-color: #fff;
}
.directory_odd
{
background-color: #eee;
}
</style>
</head>
'''
HTML_FORMAT_DIRECTORY = '''
<div class="buttonbox">
<button onclick="collapse(this.parentElement.nextElementSibling)">{name} ({size})</button>
{directory_anchor}
</div>
<div class="{css}" style="display:none">
'''.replace('\n', '')
HTML_FORMAT_FILE = '<a href="{url}">{name} ({size})</a><br>'
class PathTree(treeclass.Tree):
def __init__(
self,
path,
display_name=None,
item_type=None,
size=None,
**kwargs,
):
self.path = normalize_slash(path)
if display_name is None:
self.display_name = self.path.split(os.sep)[-1]
else:
self.display_name = display_name
kwargs['identifier'] = self.display_name
super(PathTree, self).__init__(**kwargs)
self.size = size
self.item_type = item_type
def normalize_slash(path):
path = path.replace('/', '\\')
path = path.rstrip(os.sep)
return path
def from_paths(path_datas, root_name):
all_datas = []
for data in path_datas:
if isinstance(data, str):
data = {'path': data}
elif isinstance(data, dict):
pass
else:
raise TypeError(data)
data['parts'] = data['path'].split(os.sep)
all_datas.append(data)
#path_parts = path_parts.split('\\')
#item = {'url': url, 'size': size, 'path_parts': path_parts}
#all_items.append(item)
#scheme = url_split(all_items[0]['url'])['scheme']
all_datas.sort(key=lambda x: x['path'])
tree_root = PathTree(root_name, item_type='directory')
tree_root.unsorted_children = all_datas
node_queue = set()
node_queue.add(tree_root)
# In this process, URLs are divided up into their nodes one directory layer at a time.
# The root has all URLs as its `unsorted_children` attribute, and creates
# nodes for each of the top-level directories.
# Those nodes receive all subdirectories, and repeat.
while len(node_queue) > 0:
node = node_queue.pop()
for new_child_data in node.unsorted_children:
# Create a new node for the subdirectory, which is path_parts[0]
# The rest of the child path is assigned to that node to be further divided.
# By popping, we modify the path_parts in place so that the next cycle
# only deals with the remaining subpath.
path_parts = new_child_data['parts']
child_identifier = path_parts.pop(0)
child = node.children.get(child_identifier)
if not child:
child = PathTree(child_identifier)
child.unsorted_children = []
node.add_child(child)
if len(path_parts) > 0:
child.item_type = 'directory'
child.unsorted_children.append(new_child_data)
else:
child.item_type = 'file'
child.size = new_child_data.get('size')
child.data = new_child_data.get('data')
node_queue.add(child)
if node.parent is not None and node.parent != tree_root:
node.path = node.parent.path + os.sep + node.path
del node.unsorted_children
return tree_root
def recursive_get_size(node):
'''
Calculate the size of the Directory nodes by summing the sizes of all children.
Modifies the nodes in-place.
'''
return_value = {
'size': 0,
'unmeasured': 0,
}
if node.item_type == 'file':
if node.size is None:
return_value['unmeasured'] = 1
# = instead of += because if the node.size is None, we want to propogate
# that to the caller, rather than normalizing it to 0.
return_value['size'] = node.size
else:
for child in node.list_children():
child_details = recursive_get_size(child)
return_value['size'] += child_details['size'] or 0
return_value['unmeasured'] += child_details['unmeasured']
node.size = return_value['size']
return return_value
def recursive_print_node(node, depth=0, use_html=False, header=None, footer=None):
'''
Given a tree node (presumably the root), print it and all of its children.
use_html:
Generate a neat HTML page instead of plain text.
header:
This text goes at the top of the file, or just below the <body> tag.
footer:
This text goes at the end of the file, or just above the </body> tag.
'''
if depth == 0:
if use_html:
yield '<!DOCTYPE html>\n<html>'
yield HTML_TREE_HEAD
yield '<body>'
if header is not None:
yield header
size = node.size
if size is None:
size = '???'
else:
size = bytestring.bytestring(size)
if use_html:
css_class = 'directory_even' if depth % 2 == 0 else 'directory_odd'
if node.item_type == 'directory':
directory_url = node.path
directory_anchor = '<a href="{url}">&rightarrow;</a>' if directory_url else ''
directory_anchor = directory_anchor.format(url=directory_url)
line = HTML_FORMAT_DIRECTORY.format(
css=css_class,
directory_anchor=directory_anchor,
name=node.display_name,
size=size,
)
else:
line = HTML_FORMAT_FILE.format(
name=node.display_name,
size=size,
url=node.path,
)
else:
line = '{space}{bar}{name} : ({size})'
line = line.format(
space='| ' * (depth-1),
bar='|---' if depth > 0 else '',
name=node.display_name,
size=size
)
yield line
# Sort by type (directories first) then subsort by lowercase path
customsort = lambda node: (
node.item_type == 'file',
node.path.lower(),
)
for child in node.list_children(customsort=customsort):
yield from recursive_print_node(child, depth=depth+1, use_html=use_html)
if node.item_type == 'directory':
if use_html:
# Close the directory div
yield '</div>'
else:
# This helps put some space between sibling directories
yield '| ' * (depth)
if depth == 0:
if footer is not None:
yield footer
if use_html:
yield '</body>\n</html>'
def pathtree_argparse(args):
from voussoirkit import safeprint
from voussoirkit import spinal
paths = spinal.walk_generator()
paths = [{'path': path.absolute_path, 'size': path.size} for path in paths]
tree = from_paths(paths, '.')
recursive_get_size(tree)
if args.output_file:
output_file = open(args.output_file, 'w', encoding='utf-8')
else:
output_file = None
for line in recursive_print_node(tree, use_html=args.use_html):
if output_file:
print(line, file=output_file)
else:
safeprint.safeprint(line)
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument('output_file', nargs='?', default=None)
parser.add_argument('--html', dest='use_html', action='store_true')
parser.set_defaults(func=pathtree_argparse)
args = parser.parse_args(argv)
args.func(args)
if __name__ == '__main__':
main(sys.argv[1:])

99
Treeclass/treeclass.py Normal file
View file

@ -0,0 +1,99 @@
import os
class ExistingChild(Exception):
pass
class InvalidIdentifier(Exception):
pass
class Tree:
def __init__(self, identifier, data=None):
if not isinstance(identifier, str):
print(identifier)
raise InvalidIdentifier('Identifiers must be strings')
identifier = identifier.replace('/', os.sep)
identifier = identifier.replace('\\', os.sep)
if os.sep in identifier:
raise InvalidIdentifier('Identifier cannot contain slashes')
self.identifier = identifier
self.data = data
self.parent = None
self.children = {}
def __eq__(self, other):
return isinstance(other, Tree) and self.abspath() == other.abspath()
def __getitem__(self, key):
return self.children[key]
def __hash__(self):
return hash(self.abspath())
def __repr__(self):
return 'Tree(%s)' % self.identifier
def abspath(self):
node = self
nodes = [node]
while node.parent is not None:
node = node.parent
nodes.append(node)
nodes.reverse()
nodes = [node.identifier for node in nodes]
return '\\'.join(nodes)
def add_child(self, other_node, overwrite_parent=False):
self.check_child_availability(other_node.identifier)
if other_node.parent is not None and not overwrite_parent:
raise ValueError('That node already has a parent. Try `overwrite_parent=True`')
other_node.parent = self
self.children[other_node.identifier] = other_node
return other_node
def check_child_availability(self, identifier):
if identifier in self.children:
raise ExistingChild('Node %s already has child %s' % (self.identifier, identifier))
def detach(self):
if self.parent is None:
return
del self.parent.children[self.identifier]
self.parent = None
def list_children(self, customsort=None):
children = list(self.children.values())
if customsort is None:
children.sort(key=lambda node: node.identifier.lower())
else:
children.sort(key=customsort)
return children
def merge_other(self, othertree, otherroot=None):
newroot = None
if ':' in othertree.identifier:
if otherroot is None:
raise Exception('Must specify a new name for the other tree\'s root')
else:
newroot = otherroot
else:
newroot = othertree.identifier
othertree.identifier = newroot
othertree.detach()
othertree.parent = self
self.check_child_availability(newroot)
self.children[newroot] = othertree
def walk(self, customsort=None):
yield self
for child in self.list_children(customsort=customsort):
yield from child.walk(customsort=customsort)
def walk_parents(self):
parent = self.parent
while parent is not None:
yield parent
parent = parent.parent

2
_voussoirkit/localkit.bat
View file

@ -1,3 +1,3 @@
phase1
xcopy voussoirkit C:\Python35\Lib\site-packages\voussoirkit /y
xcopy voussoirkit C:\Python36\Lib\site-packages\voussoirkit /y
phase2

4
_voussoirkit/phase1.py
View file

@ -2,13 +2,17 @@ import shutil
import os
PATHS = [
'C:\\git\\else\\BaseNumber\\basenumber.py',
'C:\\git\\else\\Bytestring\\bytestring.py',
'C:\\git\\else\\Clipext\\clipext.py',
'C:\\git\\else\\Downloady\\downloady.py',
'C:\\git\\else\\Fusker\\fusker.py',
'C:\\git\\else\\Pathclass\\pathclass.py',
'C:\\git\\else\\Ratelimiter\\ratelimiter.py',
'C:\\git\\else\\RateMeter\\ratemeter.py',
'C:\\git\\else\\Safeprint\\safeprint.py',
'C:\\git\\else\\Treeclass\\treeclass.py',
'C:\\git\\else\\Treeclass\\pathtree.py',
'C:\\git\\else\\SpinalTap\\spinal.py',
]

4
_voussoirkit/setup.py
View file

@ -3,9 +3,9 @@ import setuptools
setuptools.setup(
name='voussoirkit',
packages=['voussoirkit'],
version='0.0.8',
version='0.0.9',
author='voussoir',
author_email='_',
author_email='ethan@voussoir.net',
description='voussoir\'s toolkit',
url='https://github.com/voussoir/else',
install_requires=['pyperclip']

2
_voussoirkit/voussoirkit.bat
View file

@ -1,4 +1,4 @@
phase1
py setup.py register -r pypi
rem py setup.py register -r pypi
py setup.py sdist upload -r pypi
phase2