diff --git a/AESFile/aesfile.py b/AESFile/aesfile.py index c63d396..891f9a7 100644 --- a/AESFile/aesfile.py +++ b/AESFile/aesfile.py @@ -4,8 +4,7 @@ from Crypto.Cipher import AES import sys import os -# pip install -# https://raw.githubusercontent.com/voussoir/else/master/_voussoirkit/voussoirkit.zip +# pip install voussoirkit from voussoirkit import bytestring diff --git a/Downloady/downloady.py b/Downloady/downloady.py index 91c8246..9a8828e 100644 --- a/Downloady/downloady.py +++ b/Downloady/downloady.py @@ -1,14 +1,13 @@ import argparse import os -import pyperclip # pip install pyperclip +import pyperclip import requests import sys import time import urllib import warnings -# pip install -# https://raw.githubusercontent.com/voussoir/else/master/_voussoirkit/voussoirkit.zip +# pip install voussoirkit from voussoirkit import bytestring from voussoirkit import ratelimiter from voussoirkit import clipext diff --git a/Instathief/instathief.py b/Instathief/instathief.py index b297f0b..a726269 100644 --- a/Instathief/instathief.py +++ b/Instathief/instathief.py @@ -6,8 +6,7 @@ import os import requests import sys -# pip install -# https://raw.githubusercontent.com/voussoir/else/master/_voussoirkit/voussoirkit.zip +# pip install voussoirkit from voussoirkit import clipext from voussoirkit import downloady diff --git a/OpenDirDL/opendirdl.py b/OpenDirDL/opendirdl.py index 9de6f86..ab28f81 100644 --- a/OpenDirDL/opendirdl.py +++ b/OpenDirDL/opendirdl.py @@ -136,8 +136,7 @@ import sys ## import tkinter import urllib.parse -# pip install -# https://raw.githubusercontent.com/voussoir/else/master/_voussoirkit/voussoirkit.zip +# pip install voussoirkit from voussoirkit import bytestring from voussoirkit import downloady diff --git a/Pathclass/pathclass.py b/Pathclass/pathclass.py index 9e2eb87..80ab47e 100644 --- a/Pathclass/pathclass.py +++ b/Pathclass/pathclass.py @@ -55,6 +55,11 @@ class Path: def is_link(self): return os.path.islink(self.absolute_path) + def join(self, subpath): + if not isinstance(subpath, str): + raise TypeError('subpath must be a string') + return Path(os.path.join(self.absolute_path, subpath)) + @property def normcase(self): return os.path.normcase(self.absolute_path) @@ -95,8 +100,7 @@ class Path: return os.stat(self.absolute_path) def with_child(self, basename): - basename = os.path.basename(basename) - return Path(os.path.join(self.absolute_path, basename)) + return self.join(os.path.basename(basename)) def common_path(paths, fallback): @@ -167,6 +171,7 @@ def get_path_casing(path): imaginary_portion = imaginary_portion.replace(real_portion, '') imaginary_portion = imaginary_portion.lstrip(os.sep) cased = os.path.join(cased, imaginary_portion) + cased = cased.rstrip(os.sep) return cased except IndexError: return input_path diff --git a/Piecewise/piecewise.py b/Piecewise/piecewise.py new file mode 100644 index 0000000..c159763 --- /dev/null +++ b/Piecewise/piecewise.py @@ -0,0 +1,170 @@ +import argparse +import os +import sqlite3 +import sys +import threading +import time +import traceback + +from voussoirkit import bytestring +from voussoirkit import downloady + +DEFAULT_PIECE_SIZE = bytestring.MIBIBYTE +DEFAULT_THREAD_COUNT = 10 + +def init(url, localname=None, piece_size=DEFAULT_PIECE_SIZE): + localname = localname or downloady.basename_from_url(url) or str(int(time.time())) + plan = downloady.prepare_plan(url, localname, headers={'range': 'bytes=0-'}) + sql_name = localname + '.db' + if os.path.exists(sql_name): + raise ValueError('database already exists %s' % sql_name) + sql = sqlite3.connect(sql_name) + cur = sql.cursor() + cur.execute('CREATE TABLE IF NOT EXISTS meta (url TEXT)') + cur.execute('CREATE TABLE IF NOT EXISTS pieces (indx INT, start INT, end INT, done INT)') + cur.execute('INSERT INTO meta VALUES(?)', [url]) + cur.execute('CREATE INDEX IF NOT EXISTS index_done on pieces(done)') + index = 0 + while True: + start = index * piece_size + end = start + piece_size + end = min(end, plan['remote_total_bytes']) + done = 0 + cur.execute('INSERT INTO pieces VALUES(?, ?, ?, ?)', [index, start, end, done]) + start += piece_size + if start > plan['remote_total_bytes']: + break + index += 1 + sql.commit() + print('Initialized %s with %d pieces' % (sql_name, index + 1)) + +def reset(databasename): + sql = sqlite3.connect(databasename) + cur = sql.cursor() + cur.execute('UPDATE pieces SET done = 0 WHERE done == 1') + sql.commit() + +def piece_thread(entry): + try: + headers = {'range': 'bytes=%d-%d' % (entry['start'], entry['end'])} + x = downloady.download_file(entry['url'], entry['localname'], headers=headers, timeout=10) + entry['job'].finish() + return x + except: + traceback.print_exc() + entry['job'].reset() + +class Downloader: + def __init__(self, sql_name, thread_count=10): + self.thread_count = thread_count + self.sql_name = sql_name + self.localname = self.sql_name.rsplit('.db')[0] + self.sql = sqlite3.connect(self.sql_name) + self.cur = self.sql.cursor() + self.url = self.cur.execute('SELECT * FROM meta').fetchone()[0] + + def start(self): + self.cur.execute('SELECT * FROM pieces WHERE done == 0') + fetch = self.cur.fetchall() + jobs = [] + for entry in fetch: + entry = { + 'url': self.url, + 'localname': self.localname, + 'index': entry[0], + 'start': entry[1], + 'end': entry[2], + } + job = Job(entry) + jobs.append(job) + + while len(jobs) > 0: + finished_jobs = [] + active_jobs = [] + pending_jobs = [] + + for job in jobs: + if job.state == 'finished': + finished_jobs.append(job) + elif job.state == 'in progress': + active_jobs.append(job) + else: + pending_jobs.append(job) + + for job in finished_jobs: + self.cur.execute('UPDATE pieces SET done == 1 WHERE indx == ?', [job.entry['index']]) + print('finished #%d' % job.entry['index']) + self.sql.commit() + + need = self.thread_count - len(active_jobs) + while need > 0 and len(pending_jobs) > 0: + job = pending_jobs.pop() + print('starting #%d' % job.entry['index']) + job.start() + active_jobs.append(job) + need -= 1 + + jobs = pending_jobs + active_jobs + print('%d jobs in progress' % len(active_jobs)) + print('%d jobs waiting' % len(pending_jobs)) + time.sleep(2) + +class Job: + def __init__(self, entry): + self.state = 'pending' + self.entry = entry + + def start(self): + self.state = 'in progress' + self.entry['job'] = self + thread = threading.Thread(target=piece_thread, args=[self.entry]) + thread.daemon = True + thread.start() + + def finish(self): + self.state = 'finished' + + def reset(self): + self.state = 'pending' + + +def init_argparse(args): + if isinstance(args.piece_size, str): + piece_size = bytestring.parsebytes(args.piece_size) + else: + piece_size = args.piece_size + init(args.url, localname=args.localname, piece_size=piece_size) + +def reset_argparse(args): + reset(args.databasename) + +def download_argparse(args): + downloader = Downloader(args.databasename, int(args.thread_count)) + downloader.start() + +def main(argv): + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + subparsers.required = True + subparsers.dest = 'command' + + p_init = subparsers.add_parser('init') + p_init.add_argument('url') + p_init.add_argument('localname', nargs='?', default=None) + p_init.add_argument('--piece-size', dest='piece_size', default=DEFAULT_PIECE_SIZE) + p_init.set_defaults(func=init_argparse) + + p_reset = subparsers.add_parser('reset') + p_reset.add_argument('databasename') + p_reset.set_defaults(func=reset_argparse) + + p_download = subparsers.add_parser('download') + p_download.add_argument('databasename') + p_download.add_argument('--threads', dest='thread_count', default=DEFAULT_THREAD_COUNT) + p_download.set_defaults(func=download_argparse) + + args = parser.parse_args(argv) + args.func(args) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/README.md b/README.md index ed6fc38..949dd2a 100644 --- a/README.md +++ b/README.md @@ -5,4 +5,4 @@ For anything that isn't Reddit Note: Many projects in this repository import other projects. If you are having any Import Errors, please: - pip install https://github.com/voussoir/else/raw/master/_voussoirkit/voussoirkit.zip --upgrade + pip install voussoirkit --upgrade diff --git a/ServerReference/simpleserver.py b/ServerReference/simpleserver.py index 36e71fb..b2c910f 100644 --- a/ServerReference/simpleserver.py +++ b/ServerReference/simpleserver.py @@ -8,8 +8,7 @@ import socketserver import sys import types -# pip install -# https://raw.githubusercontent.com/voussoir/else/master/_voussoirkit/voussoirkit.zip +# pip install voussoirkit from voussoirkit import bytestring from voussoirkit import pathclass from voussoirkit import ratelimiter diff --git a/SpinalTap/spinal.py b/SpinalTap/spinal.py index 81f2598..ab1c9e1 100644 --- a/SpinalTap/spinal.py +++ b/SpinalTap/spinal.py @@ -5,8 +5,7 @@ import os import shutil import sys -# pip install -# https://raw.githubusercontent.com/voussoir/else/master/_voussoirkit/voussoirkit.zip +# pip install voussoirkit from voussoirkit import bytestring from voussoirkit import pathclass from voussoirkit import ratelimiter @@ -558,6 +557,7 @@ def walk_generator( path='.', callback_exclusion=None, callback_permission_denied=None, + depth_first=True, exclude_directories=None, exclude_filenames=None, recurse=True, @@ -690,6 +690,9 @@ def walk_generator( if not recurse: break - # Extendleft causes them to get reversed, so flip it first. - directories.reverse() - directory_queue.extendleft(directories) + if depth_first: + # Extendleft causes them to get reversed, so flip it first. + directories.reverse() + directory_queue.extendleft(directories) + else: + directory_queue.extend(directories) diff --git a/ThreadedDL/threaded_dl.py b/ThreadedDL/threaded_dl.py index 7285ef3..4f54ee4 100644 --- a/ThreadedDL/threaded_dl.py +++ b/ThreadedDL/threaded_dl.py @@ -3,8 +3,7 @@ import sys import threading import time -# pip install -# https://raw.githubusercontent.com/voussoir/else/master/_voussoirkit/voussoirkit.zip +# pip install voussoirkit from voussoirkit import clipext from voussoirkit import downloady diff --git a/Toolbox/fileswith.py b/Toolbox/fileswith.py index 598a4bd..2246de2 100644 --- a/Toolbox/fileswith.py +++ b/Toolbox/fileswith.py @@ -5,6 +5,7 @@ For example: fileswith.py *.py "import random" ''' +import argparse import fnmatch import glob import re @@ -13,24 +14,57 @@ import sys from voussoirkit import safeprint from voussoirkit import spinal -filepattern = sys.argv[1] -searchpattern = sys.argv[2] -for filename in spinal.walk_generator(): - filename = filename.absolute_path - if not fnmatch.fnmatch(filename, filepattern): - continue - matches = [] - handle = open(filename, 'r', encoding='utf-8') - try: - with handle: - for (index, line) in enumerate(handle): - if re.search(searchpattern, line, flags=re.IGNORECASE): - line = '%d | %s' % (index, line.strip()) - matches.append(line) - except: - pass - if matches: - print(filename) - safeprint.safeprint('\n'.join(matches)) - print() +def fileswith(filepattern, terms, do_regex=False, do_glob=False): + search_terms = [term.lower() for term in terms] + + def term_matches(text, term): + return ( + (term in text) or + (do_regex and re.search(term, text)) or + (do_glob and fnmatch.fnmatch(text, term)) + ) + + + generator = spinal.walk_generator(depth_first=False, yield_directories=True) + for filepath in generator: + if not fnmatch.fnmatch(filepath.basename, filepattern): + continue + handle = open(filepath.absolute_path, 'r', encoding='utf-8') + matches = [] + try: + with handle: + for (index, line) in enumerate(handle): + if all(term_matches(line, term) for term in terms): + line = '%d | %s' % (index, line.strip()) + matches.append(line) + except: + pass + if matches: + print(filepath.absolute_path) + safeprint.safeprint('\n'.join(matches)) + print() + + +def fileswith_argparse(args): + return fileswith( + filepattern=args.filepattern, + terms=args.search_terms, + do_glob=args.do_glob, + do_regex=args.do_regex, + ) + +def main(argv): + parser = argparse.ArgumentParser() + + parser.add_argument('filepattern') + parser.add_argument('search_terms', nargs='+', default=None) + parser.add_argument('--regex', dest='do_regex', action='store_true') + parser.add_argument('--glob', dest='do_glob', action='store_true') + parser.set_defaults(func=fileswith_argparse) + + args = parser.parse_args(argv) + args.func(args) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/Toolbox/search.py b/Toolbox/search.py new file mode 100644 index 0000000..64c93e2 --- /dev/null +++ b/Toolbox/search.py @@ -0,0 +1,50 @@ +import argparse +import fnmatch +import os +import re +import sys + +from voussoirkit import safeprint +from voussoirkit import spinal + +def search(terms, match_any=False, do_regex=False, do_glob=False): + search_terms = [term.lower() for term in terms] + + def term_matches(text, term): + return ( + (term in text) or + (do_regex and re.search(term, text)) or + (do_glob and fnmatch.fnmatch(text, term)) + ) + + anyall = any if match_any else all + + generator = spinal.walk_generator(depth_first=False, yield_directories=True) + for filepath in generator: + basename = filepath.basename.lower() + if anyall(term_matches(basename, term) for term in search_terms): + safeprint.safeprint(filepath.absolute_path) + + +def search_argparse(args): + return search( + terms=args.search_terms, + do_glob=args.do_glob, + do_regex=args.do_regex, + match_any=args.match_any, + ) + +def main(argv): + parser = argparse.ArgumentParser() + + parser.add_argument('search_terms', nargs='+', default=None) + parser.add_argument('--any', dest='match_any', action='store_true') + parser.add_argument('--regex', dest='do_regex', action='store_true') + parser.add_argument('--glob', dest='do_glob', action='store_true') + parser.set_defaults(func=search_argparse) + + args = parser.parse_args(argv) + args.func(args) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/VoxelSphereGenerator/voxelspheregenerator.py b/VoxelSphereGenerator/voxelspheregenerator.py index 241c92a..c02da6e 100644 --- a/VoxelSphereGenerator/voxelspheregenerator.py +++ b/VoxelSphereGenerator/voxelspheregenerator.py @@ -215,32 +215,6 @@ def voxelspheregenerator(WIDTH, HEIGH, DEPTH, WALL_THICKNESS=None): layer_image.save(layer_filename) - # Copy to the upper right corner. - #for y in range(math.ceil(RAD_Y)): - # #print(y) - # for x in range(math.ceil(RAD_X), WIDTH): - # #print(x, '==', (WIDTH-1) - x) - # mapped_x = (WIDTH - 1) - x - # layer_matrix[x][y] = layer_matrix[mapped_x][y] - - # Copy to the lower semicircle. - #for y in range(math.ceil(RAD_Y), HEIGH): - # #print(y) - # for x in range(WIDTH): - # #print(y, '==', (HEIGH-1) - y) - # mapped_y = (HEIGH-1) - y - # layer_matrix[x][y] = layer_matrix[x][mapped_y] - - - - #break - #layer_matrix = [['▓' if dot == dot_highlight else '░' if dot == dot_normal else ' 'for dot in sublist] for sublist in layer_matrix] - #layer_matrix = [''.join(sublist) for sublist in layer_matrix] - #layer_matrix = '\n'.join(layer_matrix) - #print(layer_matrix) - #print() - - def voxelsphere_argparse(args): height_depth_match = bool(args.height) == bool(args.depth) if not height_depth_match: diff --git a/_voussoirkit/phase1.py b/_voussoirkit/phase1.py new file mode 100644 index 0000000..e978f07 --- /dev/null +++ b/_voussoirkit/phase1.py @@ -0,0 +1,18 @@ +import shutil +import os + +PATHS = [ +'C:\\git\\else\\Bytestring\\bytestring.py', +'C:\\git\\else\\Clipext\\clipext.py', +'C:\\git\\else\\Downloady\\downloady.py', +'C:\\git\\else\\Pathclass\\pathclass.py', +'C:\\git\\else\\Ratelimiter\\ratelimiter.py', +'C:\\git\\else\\RateMeter\\ratemeter.py', +'C:\\git\\else\\Safeprint\\safeprint.py', +'C:\\git\\else\\SpinalTap\\spinal.py', +] + +os.makedirs('voussoirkit', exist_ok=True) +for filename in PATHS: + shutil.copy(filename, os.path.join('voussoirkit', os.path.basename(filename))) +open(os.path.join('voussoirkit', '__init__.py'), 'w').close() diff --git a/_voussoirkit/phase2.py b/_voussoirkit/phase2.py new file mode 100644 index 0000000..fd59f3f --- /dev/null +++ b/_voussoirkit/phase2.py @@ -0,0 +1,12 @@ +import shutil +import os + +def delete(folder): + try: + shutil.rmtree(folder) + except: + pass + +delete('dist') +delete('voussoirkit') +delete('voussoirkit.egg-info') diff --git a/_voussoirkit/setup.py b/_voussoirkit/setup.py new file mode 100644 index 0000000..bf79d3c --- /dev/null +++ b/_voussoirkit/setup.py @@ -0,0 +1,11 @@ +import setuptools + +setuptools.setup( + name='voussoirkit', + packages=['voussoirkit'], + version='0.0.7', + author='voussoir', + author_email='_', + description='voussoir\'s toolkit', + url='https://github.com/voussoir/else', +) diff --git a/_voussoirkit/voussoirkit.bat b/_voussoirkit/voussoirkit.bat new file mode 100644 index 0000000..d0a0224 --- /dev/null +++ b/_voussoirkit/voussoirkit.bat @@ -0,0 +1,4 @@ +phase1 +py setup.py register -r pypi +py setup.py sdist upload -r pypi +phase2 diff --git a/_voussoirkit/voussoirkit.py b/_voussoirkit/voussoirkit.py deleted file mode 100644 index 41fcf97..0000000 --- a/_voussoirkit/voussoirkit.py +++ /dev/null @@ -1,73 +0,0 @@ -import glob -import shutil -import os - -PACKAGE = 'voussoirkit' -PATHS = [ -'C:\\git\\else\\Bytestring\\bytestring.py', -'C:\\git\\else\\Clipext\\clipext.py', -'C:\\git\\else\\Downloady\\downloady.py', -'C:\\git\\else\\Pathclass\\pathclass.py', -'C:\\git\\else\\Ratelimiter\\ratelimiter.py', -'C:\\git\\else\\RateMeter\\ratemeter.py', -'C:\\git\\else\\Safeprint\\safeprint.py', -'C:\\git\\else\\SpinalTap\\spinal.py', -'C:\\git\\else\\WebstreamZip\\webstreamzip.py', -] - -os.makedirs(PACKAGE, exist_ok=True) - -for zipfile in glob.glob('*.zip'): - os.remove(zipfile) - -py_modules = [] -local_paths = [] - -for path in PATHS: - basename = os.path.basename(path) - module_name = '{package}.{module}'.format(package=PACKAGE, module=basename.replace('.py', '')) - py_modules.append(module_name) - local_path = os.path.join(PACKAGE, basename) - local_paths.append(local_path) - try: - os.link(path, local_path) - except FileExistsError: - pass - -print('Creating setup.py') -setup_content = ''' -import setuptools - -setuptools.setup( - author='voussoir', - name='{package}', - version='0.0.4', - description='', - py_modules=[{py_modules}], -) -''' - -py_modules = ["'%s'" % x for x in py_modules] -py_modules = ', '.join(py_modules) -setup_content = setup_content.format(package=PACKAGE, py_modules=py_modules) - -setup_file = open('setup.py', 'w') -setup_file.write(setup_content) -setup_file.close() - -print('Executing setup.py') -os.system('python setup.py sdist') - -print('Moving zip file') -zips = glob.glob('dist\\*.zip') -for zip_filename in zips: - new_zip = os.path.basename(zip_filename) - new_zip = os.path.abspath(new_zip) - shutil.move(zip_filename, new_zip) - -print('Deleting temp') -shutil.rmtree('dist') -shutil.rmtree(PACKAGE) -shutil.rmtree(glob.glob('*.egg-info')[0]) -os.remove('setup.py') -os.rename(glob.glob('*.zip')[0], 'voussoirkit.zip') diff --git a/_voussoirkit/voussoirkit.zip b/_voussoirkit/voussoirkit.zip deleted file mode 100644 index 635ae9a..0000000 Binary files a/_voussoirkit/voussoirkit.zip and /dev/null differ