Upload voussoirkit to pypi

AESFile/aesfile.py

@@ -4,8 +4,7 @@ from Crypto.Cipher import AES
 import sys
 import os
 
-# pip install
-# https://raw.githubusercontent.com/voussoir/else/master/_voussoirkit/voussoirkit.zip
+# pip install voussoirkit
 from voussoirkit import bytestring
 
 

Downloady/downloady.py

@@ -1,14 +1,13 @@
 import argparse
 import os
-import pyperclip # pip install pyperclip
+import pyperclip
 import requests
 import sys
 import time
 import urllib
 import warnings
 
-# pip install
-# https://raw.githubusercontent.com/voussoir/else/master/_voussoirkit/voussoirkit.zip
+# pip install voussoirkit
 from voussoirkit import bytestring
 from voussoirkit import ratelimiter
 from voussoirkit import clipext

Instathief/instathief.py

@@ -6,8 +6,7 @@ import os
 import requests
 import sys
 
-# pip install
-# https://raw.githubusercontent.com/voussoir/else/master/_voussoirkit/voussoirkit.zip
+# pip install voussoirkit
 from voussoirkit import clipext
 from voussoirkit import downloady
 

OpenDirDL/opendirdl.py

@@ -136,8 +136,7 @@ import sys
 ## import tkinter
 import urllib.parse
 
-# pip install
-# https://raw.githubusercontent.com/voussoir/else/master/_voussoirkit/voussoirkit.zip
+# pip install voussoirkit
 from voussoirkit import bytestring
 from voussoirkit import downloady
 

Pathclass/pathclass.py

@@ -55,6 +55,11 @@ class Path:
     def is_link(self):
         return os.path.islink(self.absolute_path)
 
+    def join(self, subpath):
+        if not isinstance(subpath, str):
+            raise TypeError('subpath must be a string')
+        return Path(os.path.join(self.absolute_path, subpath))
+
     @property
     def normcase(self):
         return os.path.normcase(self.absolute_path)
@@ -95,8 +100,7 @@ class Path:
         return os.stat(self.absolute_path)
 
     def with_child(self, basename):
-        basename = os.path.basename(basename)
-        return Path(os.path.join(self.absolute_path, basename))
+        return self.join(os.path.basename(basename))
 
 
 def common_path(paths, fallback):
@@ -167,6 +171,7 @@ def get_path_casing(path):
         imaginary_portion = imaginary_portion.replace(real_portion, '')
         imaginary_portion = imaginary_portion.lstrip(os.sep)
         cased = os.path.join(cased, imaginary_portion)
+        cased = cased.rstrip(os.sep)
         return cased
     except IndexError:
         return input_path

Piecewise/piecewise.py

@@ -0,0 +1,170 @@
+import argparse
+import os
+import sqlite3
+import sys
+import threading
+import time
+import traceback
+
+from voussoirkit import bytestring
+from voussoirkit import downloady
+
+DEFAULT_PIECE_SIZE = bytestring.MIBIBYTE
+DEFAULT_THREAD_COUNT = 10
+
+def init(url, localname=None, piece_size=DEFAULT_PIECE_SIZE):
+    localname = localname or downloady.basename_from_url(url) or str(int(time.time()))
+    plan = downloady.prepare_plan(url, localname, headers={'range': 'bytes=0-'})
+    sql_name = localname + '.db'
+    if os.path.exists(sql_name):
+        raise ValueError('database already exists %s' % sql_name)
+    sql = sqlite3.connect(sql_name)
+    cur = sql.cursor()
+    cur.execute('CREATE TABLE IF NOT EXISTS meta (url TEXT)')
+    cur.execute('CREATE TABLE IF NOT EXISTS pieces (indx INT, start INT, end INT, done INT)')
+    cur.execute('INSERT INTO meta VALUES(?)', [url])
+    cur.execute('CREATE INDEX IF NOT EXISTS index_done on pieces(done)')
+    index = 0
+    while True:
+        start = index * piece_size
+        end = start + piece_size
+        end = min(end, plan['remote_total_bytes'])
+        done = 0
+        cur.execute('INSERT INTO pieces VALUES(?, ?, ?, ?)', [index, start, end, done])
+        start += piece_size
+        if start > plan['remote_total_bytes']:
+            break
+        index += 1
+    sql.commit()
+    print('Initialized %s with %d pieces' % (sql_name, index + 1))
+
+def reset(databasename):
+    sql = sqlite3.connect(databasename)
+    cur = sql.cursor()
+    cur.execute('UPDATE pieces SET done = 0 WHERE done == 1')
+    sql.commit()
+
+def piece_thread(entry):
+    try:
+        headers = {'range': 'bytes=%d-%d' % (entry['start'], entry['end'])}
+        x = downloady.download_file(entry['url'], entry['localname'], headers=headers, timeout=10)
+        entry['job'].finish()
+        return x
+    except:
+        traceback.print_exc()
+        entry['job'].reset()
+
+class Downloader:
+    def __init__(self, sql_name, thread_count=10):
+        self.thread_count = thread_count
+        self.sql_name = sql_name
+        self.localname = self.sql_name.rsplit('.db')[0]
+        self.sql = sqlite3.connect(self.sql_name)
+        self.cur = self.sql.cursor()
+        self.url = self.cur.execute('SELECT * FROM meta').fetchone()[0]
+    
+    def start(self):
+        self.cur.execute('SELECT * FROM pieces WHERE done == 0')
+        fetch = self.cur.fetchall()
+        jobs = []
+        for entry in fetch:
+            entry = {
+                'url': self.url,
+                'localname': self.localname,
+                'index': entry[0],
+                'start': entry[1],
+                'end': entry[2],
+            }
+            job = Job(entry)
+            jobs.append(job)
+
+        while len(jobs) > 0:
+            finished_jobs = []
+            active_jobs = []
+            pending_jobs = []
+
+            for job in jobs:
+                if job.state == 'finished':
+                    finished_jobs.append(job)
+                elif job.state == 'in progress':
+                    active_jobs.append(job)
+                else:
+                    pending_jobs.append(job)
+
+            for job in finished_jobs:
+                self.cur.execute('UPDATE pieces SET done == 1 WHERE indx == ?', [job.entry['index']])
+                print('finished #%d' % job.entry['index'])
+            self.sql.commit()
+
+            need = self.thread_count - len(active_jobs)
+            while need > 0 and len(pending_jobs) > 0:
+                job = pending_jobs.pop()
+                print('starting #%d' % job.entry['index'])
+                job.start()
+                active_jobs.append(job)
+                need -= 1
+
+            jobs = pending_jobs + active_jobs
+            print('%d jobs in progress' % len(active_jobs))
+            print('%d jobs waiting' % len(pending_jobs))
+            time.sleep(2)
+
+class Job:
+    def __init__(self, entry):
+        self.state = 'pending'
+        self.entry = entry
+
+    def start(self):
+        self.state = 'in progress'
+        self.entry['job'] = self
+        thread = threading.Thread(target=piece_thread, args=[self.entry])
+        thread.daemon = True
+        thread.start()
+
+    def finish(self):
+        self.state = 'finished'
+
+    def reset(self):
+        self.state = 'pending'
+
+
+def init_argparse(args):
+    if isinstance(args.piece_size, str):
+        piece_size = bytestring.parsebytes(args.piece_size)
+    else:
+        piece_size = args.piece_size
+    init(args.url, localname=args.localname, piece_size=piece_size)
+
+def reset_argparse(args):
+    reset(args.databasename)
+
+def download_argparse(args):
+    downloader = Downloader(args.databasename, int(args.thread_count))
+    downloader.start()
+
+def main(argv):
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers()
+    subparsers.required = True
+    subparsers.dest = 'command'
+
+    p_init = subparsers.add_parser('init')
+    p_init.add_argument('url')
+    p_init.add_argument('localname', nargs='?', default=None)
+    p_init.add_argument('--piece-size', dest='piece_size', default=DEFAULT_PIECE_SIZE)
+    p_init.set_defaults(func=init_argparse)
+
+    p_reset = subparsers.add_parser('reset')
+    p_reset.add_argument('databasename')
+    p_reset.set_defaults(func=reset_argparse)
+
+    p_download = subparsers.add_parser('download')
+    p_download.add_argument('databasename')
+    p_download.add_argument('--threads', dest='thread_count', default=DEFAULT_THREAD_COUNT)
+    p_download.set_defaults(func=download_argparse)
+
+    args = parser.parse_args(argv)
+    args.func(args)
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

README.md

@@ -5,4 +5,4 @@ For anything that isn't Reddit
 
 Note: Many projects in this repository import other projects. If you are having any Import Errors, please:
 
-    pip install https://github.com/voussoir/else/raw/master/_voussoirkit/voussoirkit.zip --upgrade
+    pip install voussoirkit --upgrade

ServerReference/simpleserver.py

@@ -8,8 +8,7 @@ import socketserver
 import sys
 import types
 
-# pip install
-# https://raw.githubusercontent.com/voussoir/else/master/_voussoirkit/voussoirkit.zip
+# pip install voussoirkit
 from voussoirkit import bytestring
 from voussoirkit import pathclass
 from voussoirkit import ratelimiter

SpinalTap/spinal.py

@@ -5,8 +5,7 @@ import os
 import shutil
 import sys
 
-# pip install
-# https://raw.githubusercontent.com/voussoir/else/master/_voussoirkit/voussoirkit.zip
+# pip install voussoirkit
 from voussoirkit import bytestring
 from voussoirkit import pathclass
 from voussoirkit import ratelimiter
@@ -558,6 +557,7 @@ def walk_generator(
         path='.',
         callback_exclusion=None,
         callback_permission_denied=None,
+        depth_first=True,
         exclude_directories=None,
         exclude_filenames=None,
         recurse=True,
@@ -690,6 +690,9 @@ def walk_generator(
         if not recurse:
             break
 
-        # Extendleft causes them to get reversed, so flip it first.
-        directories.reverse()
-        directory_queue.extendleft(directories)
+        if depth_first:
+            # Extendleft causes them to get reversed, so flip it first.
+            directories.reverse()
+            directory_queue.extendleft(directories)
+        else:
+            directory_queue.extend(directories)

ThreadedDL/threaded_dl.py

@@ -3,8 +3,7 @@ import sys
 import threading
 import time
 
-# pip install
-# https://raw.githubusercontent.com/voussoir/else/master/_voussoirkit/voussoirkit.zip
+# pip install voussoirkit
 from voussoirkit import clipext
 from voussoirkit import downloady
 

Toolbox/fileswith.py

@@ -5,6 +5,7 @@ For example:
 fileswith.py *.py "import random"
 '''
 
+import argparse
 import fnmatch
 import glob
 import re
@@ -13,24 +14,57 @@ import sys
 from voussoirkit import safeprint
 from voussoirkit import spinal
 
-filepattern = sys.argv[1]
-searchpattern = sys.argv[2]
 
-for filename in spinal.walk_generator():
-    filename = filename.absolute_path
-    if not fnmatch.fnmatch(filename, filepattern):
-        continue
-    matches = []
-    handle = open(filename, 'r', encoding='utf-8')
-    try:
-        with handle:
-            for (index, line) in enumerate(handle):
-                if re.search(searchpattern, line, flags=re.IGNORECASE):
-                    line = '%d | %s' % (index, line.strip())
-                    matches.append(line)
-    except:
-        pass
-    if matches:
-        print(filename)
-        safeprint.safeprint('\n'.join(matches))
-        print()
+def fileswith(filepattern, terms, do_regex=False, do_glob=False):
+    search_terms = [term.lower() for term in terms]
+
+    def term_matches(text, term):
+        return (
+            (term in text) or
+            (do_regex and re.search(term, text)) or
+            (do_glob and fnmatch.fnmatch(text, term))
+        )
+
+
+    generator = spinal.walk_generator(depth_first=False, yield_directories=True)
+    for filepath in generator:
+        if not fnmatch.fnmatch(filepath.basename, filepattern):
+            continue
+        handle = open(filepath.absolute_path, 'r', encoding='utf-8')
+        matches = []
+        try:
+            with handle:
+                for (index, line) in enumerate(handle):
+                    if all(term_matches(line, term) for term in terms):
+                        line = '%d | %s' % (index, line.strip())
+                        matches.append(line)
+        except:
+            pass
+        if matches:
+            print(filepath.absolute_path)
+            safeprint.safeprint('\n'.join(matches))
+            print()
+
+
+def fileswith_argparse(args):
+    return fileswith(
+        filepattern=args.filepattern,
+        terms=args.search_terms,
+        do_glob=args.do_glob,
+        do_regex=args.do_regex,
+    )
+
+def main(argv):
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('filepattern')
+    parser.add_argument('search_terms', nargs='+', default=None)
+    parser.add_argument('--regex', dest='do_regex', action='store_true')
+    parser.add_argument('--glob', dest='do_glob', action='store_true')
+    parser.set_defaults(func=fileswith_argparse)
+
+    args = parser.parse_args(argv)
+    args.func(args)
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

Toolbox/search.py

@@ -0,0 +1,50 @@
+import argparse
+import fnmatch
+import os
+import re
+import sys
+
+from voussoirkit import safeprint
+from voussoirkit import spinal
+
+def search(terms, match_any=False, do_regex=False, do_glob=False):
+    search_terms = [term.lower() for term in terms]
+
+    def term_matches(text, term):
+        return (
+            (term in text) or
+            (do_regex and re.search(term, text)) or
+            (do_glob and fnmatch.fnmatch(text, term))
+        )
+
+    anyall = any if match_any else all
+
+    generator = spinal.walk_generator(depth_first=False, yield_directories=True)
+    for filepath in generator:
+        basename = filepath.basename.lower()
+        if anyall(term_matches(basename, term) for term in search_terms):
+            safeprint.safeprint(filepath.absolute_path)
+
+
+def search_argparse(args):
+    return search(
+        terms=args.search_terms,
+        do_glob=args.do_glob,
+        do_regex=args.do_regex,
+        match_any=args.match_any,
+    )
+
+def main(argv):
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('search_terms', nargs='+', default=None)
+    parser.add_argument('--any', dest='match_any', action='store_true')
+    parser.add_argument('--regex', dest='do_regex', action='store_true')
+    parser.add_argument('--glob', dest='do_glob', action='store_true')
+    parser.set_defaults(func=search_argparse)
+
+    args = parser.parse_args(argv)
+    args.func(args)
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

VoxelSphereGenerator/voxelspheregenerator.py

@@ -215,32 +215,6 @@ def voxelspheregenerator(WIDTH, HEIGH, DEPTH, WALL_THICKNESS=None):
         layer_image.save(layer_filename)
 
 
-        # Copy to the upper right corner.
-        #for y in range(math.ceil(RAD_Y)):
-        #    #print(y)
-        #    for x in range(math.ceil(RAD_X), WIDTH):
-        #        #print(x, '==', (WIDTH-1) - x)
-        #        mapped_x = (WIDTH - 1) - x
-        #        layer_matrix[x][y] = layer_matrix[mapped_x][y]
-
-        # Copy to the lower semicircle.
-        #for y in range(math.ceil(RAD_Y), HEIGH):
-        #    #print(y)
-        #    for x in range(WIDTH):
-        #        #print(y, '==', (HEIGH-1) - y)
-        #        mapped_y = (HEIGH-1) - y
-        #        layer_matrix[x][y] = layer_matrix[x][mapped_y]
-
-
-
-        #break
-        #layer_matrix = [['▓' if dot == dot_highlight else '░' if dot == dot_normal else ' 'for dot in sublist] for sublist in layer_matrix]
-        #layer_matrix = [''.join(sublist) for sublist in layer_matrix]
-        #layer_matrix = '\n'.join(layer_matrix)
-        #print(layer_matrix)
-        #print()
-
-
 def voxelsphere_argparse(args):
     height_depth_match = bool(args.height) == bool(args.depth)
     if not height_depth_match:

_voussoirkit/phase1.py

@@ -0,0 +1,18 @@
+import shutil
+import os
+
+PATHS = [
+'C:\\git\\else\\Bytestring\\bytestring.py',
+'C:\\git\\else\\Clipext\\clipext.py',
+'C:\\git\\else\\Downloady\\downloady.py',
+'C:\\git\\else\\Pathclass\\pathclass.py',
+'C:\\git\\else\\Ratelimiter\\ratelimiter.py',
+'C:\\git\\else\\RateMeter\\ratemeter.py',
+'C:\\git\\else\\Safeprint\\safeprint.py',
+'C:\\git\\else\\SpinalTap\\spinal.py',
+]
+
+os.makedirs('voussoirkit', exist_ok=True)
+for filename in PATHS:
+    shutil.copy(filename, os.path.join('voussoirkit', os.path.basename(filename)))
+open(os.path.join('voussoirkit', '__init__.py'), 'w').close()

_voussoirkit/phase2.py

@@ -0,0 +1,12 @@
+import shutil
+import os
+
+def delete(folder):
+    try:
+        shutil.rmtree(folder)
+    except:
+        pass
+
+delete('dist')
+delete('voussoirkit')
+delete('voussoirkit.egg-info')

_voussoirkit/setup.py

@@ -0,0 +1,11 @@
+import setuptools
+
+setuptools.setup(
+    name='voussoirkit',
+    packages=['voussoirkit'],
+    version='0.0.7',
+    author='voussoir',
+    author_email='_',
+    description='voussoir\'s toolkit',
+    url='https://github.com/voussoir/else',
+)

_voussoirkit/voussoirkit.bat

@@ -0,0 +1,4 @@
+phase1
+py setup.py register -r pypi
+py setup.py sdist upload -r pypi
+phase2

_voussoirkit/voussoirkit.py

@@ -1,73 +0,0 @@
-import glob
-import shutil
-import os
-
-PACKAGE = 'voussoirkit'
-PATHS = [
-'C:\\git\\else\\Bytestring\\bytestring.py',
-'C:\\git\\else\\Clipext\\clipext.py',
-'C:\\git\\else\\Downloady\\downloady.py',
-'C:\\git\\else\\Pathclass\\pathclass.py',
-'C:\\git\\else\\Ratelimiter\\ratelimiter.py',
-'C:\\git\\else\\RateMeter\\ratemeter.py',
-'C:\\git\\else\\Safeprint\\safeprint.py',
-'C:\\git\\else\\SpinalTap\\spinal.py',
-'C:\\git\\else\\WebstreamZip\\webstreamzip.py',
-]
-
-os.makedirs(PACKAGE, exist_ok=True)
-
-for zipfile in glob.glob('*.zip'):
-    os.remove(zipfile)
-
-py_modules = []
-local_paths = []
-
-for path in PATHS:
-    basename = os.path.basename(path)
-    module_name = '{package}.{module}'.format(package=PACKAGE, module=basename.replace('.py', ''))
-    py_modules.append(module_name)
-    local_path = os.path.join(PACKAGE, basename)
-    local_paths.append(local_path)
-    try:
-        os.link(path, local_path)
-    except FileExistsError:
-        pass
-
-print('Creating setup.py')
-setup_content = '''
-import setuptools
-
-setuptools.setup(
-    author='voussoir',
-    name='{package}',
-    version='0.0.4',
-    description='',
-    py_modules=[{py_modules}],
-)
-'''
-
-py_modules = ["'%s'" % x for x in py_modules]
-py_modules = ', '.join(py_modules)
-setup_content = setup_content.format(package=PACKAGE, py_modules=py_modules)
-
-setup_file = open('setup.py', 'w')
-setup_file.write(setup_content)
-setup_file.close()
-
-print('Executing setup.py')
-os.system('python setup.py sdist')
-
-print('Moving zip file')
-zips = glob.glob('dist\\*.zip')
-for zip_filename in zips:
-    new_zip = os.path.basename(zip_filename)
-    new_zip = os.path.abspath(new_zip)
-    shutil.move(zip_filename, new_zip)
-
-print('Deleting temp')
-shutil.rmtree('dist')
-shutil.rmtree(PACKAGE)
-shutil.rmtree(glob.glob('*.egg-info')[0])
-os.remove('setup.py')
-os.rename(glob.glob('*.zip')[0], 'voussoirkit.zip')