diff --git a/Downloady/downloady.py b/Downloady/downloady.py index f3d526f..5b729b3 100644 --- a/Downloady/downloady.py +++ b/Downloady/downloady.py @@ -9,8 +9,8 @@ import warnings try: sys.path.append('C:\\git\\else\\Bytestring') - sys.path.append('C:\\git\\else\\clipext'); - sys.path.append('C:\\git\\else\\ratelimiter'); + sys.path.append('C:\\git\\else\\clipext') + sys.path.append('C:\\git\\else\\ratelimiter') import bytestring import ratelimiter import clipext diff --git a/Instathief/instathief.py b/Instathief/instathief.py index 39cdeca..1d4f1d7 100644 --- a/Instathief/instathief.py +++ b/Instathief/instathief.py @@ -7,8 +7,8 @@ import requests import sys try: - sys.path.append('C:\\git\\else\\Clipext'); - sys.path.append('C:\\git\\else\\Downloady'); + sys.path.append('C:\\git\\else\\Clipext') + sys.path.append('C:\\git\\else\\Downloady') import clipext import downloady except ImportError: diff --git a/OpenDirDL/opendirdl.py b/OpenDirDL/opendirdl.py index 078e7be..a5e974e 100644 --- a/OpenDirDL/opendirdl.py +++ b/OpenDirDL/opendirdl.py @@ -113,6 +113,7 @@ tree: import argparse ## import bs4 import collections +import concurrent.futures ## import hashlib import os ## import re @@ -124,8 +125,8 @@ import sys import urllib.parse try: - sys.path.append('C:\\git\\else\\Bytestring'); - sys.path.append('C:\\git\\else\\Downloady'); + sys.path.append('C:\\git\\else\\Bytestring') + sys.path.append('C:\\git\\else\\Downloady') import bytestring import downloady except ImportError: @@ -299,6 +300,7 @@ class Walker: self.fullscan = bool(fullscan) self.queue = collections.deque() self.seen_directories = set() + self.threadpool = concurrent.futures.ThreadPoolExecutor(4) def smart_insert(self, url=None, head=None, commit=True): ''' diff --git a/OpenDirDL/requirements.txt b/OpenDirDL/requirements.txt new file mode 100644 index 0000000..a751733 --- /dev/null +++ b/OpenDirDL/requirements.txt @@ -0,0 +1,3 @@ +https://github.com/voussoir/else/raw/master/_voussoirkit/voussoirkit.zip +bs4 +requests \ No newline at end of file diff --git a/Pathclass/pathclass.py b/Pathclass/pathclass.py index f285d1b..d2da2b0 100644 --- a/Pathclass/pathclass.py +++ b/Pathclass/pathclass.py @@ -10,13 +10,19 @@ class Path: self.absolute_path = path def __contains__(self, other): - return other.absolute_path.startswith(self.absolute_path) + this = os.path.normcase(self.absolute_path) + that = os.path.normcase(other.absolute_path) + return that.startswith(this) def __eq__(self, other): - return hasattr(other, 'absolute_path') and self.absolute_path == other.absolute_path + if not hasattr(other, 'absolute_path'): + return False + this = os.path.normcase(self.absolute_path) + that = os.path.normcase(other.absolute_path) + return this == that def __hash__(self): - return hash(self.absolute_path) + return hash(os.path.normcase(self.absolute_path)) def __repr__(self): return '{c}({path})'.format(c=self.__class__.__name__, path=repr(self.absolute_path)) diff --git a/ServerReference/simpleserver.py b/ServerReference/simpleserver.py index 6bb1eff..5442b1d 100644 --- a/ServerReference/simpleserver.py +++ b/ServerReference/simpleserver.py @@ -9,9 +9,9 @@ import sys import types try: - sys.path.append('C:\\git\\else\\Bytestring'); - sys.path.append('C:\\git\\else\\Pathclass'); - sys.path.append('C:\\git\\else\\Ratelimiter'); + sys.path.append('C:\\git\\else\\Bytestring') + sys.path.append('C:\\git\\else\\Pathclass') + sys.path.append('C:\\git\\else\\Ratelimiter') import bytestring import pathclass import ratelimiter diff --git a/SpinalTap/spinal.py b/SpinalTap/spinal.py index 90fc53b..8b95fad 100644 --- a/SpinalTap/spinal.py +++ b/SpinalTap/spinal.py @@ -11,9 +11,9 @@ import sys import time try: - sys.path.append('C:\\git\\else\\Bytestring'); - sys.path.append('C:\\git\\else\\Pathclass'); - sys.path.append('C:\\git\\else\\Ratelimiter'); + sys.path.append('C:\\git\\else\\Bytestring') + sys.path.append('C:\\git\\else\\Pathclass') + sys.path.append('C:\\git\\else\\Ratelimiter') import bytestring import pathclass import ratelimiter @@ -24,7 +24,7 @@ except ImportError: from voussoirkit import pathclass from voussoirkit import ratelimiter -logging.basicConfig(level=logging.DEBUG) +logging.basicConfig(level=logging.CRITICAL) log = logging.getLogger(__name__) CHUNK_SIZE = 128 * bytestring.KIBIBYTE diff --git a/ThreadedDL/threaded_dl.py b/ThreadedDL/threaded_dl.py index 8c807c4..496b0be 100644 --- a/ThreadedDL/threaded_dl.py +++ b/ThreadedDL/threaded_dl.py @@ -4,8 +4,8 @@ import threading import time try: - sys.path.append('C:\\git\\else\\Clipext'); - sys.path.append('C:\\git\\else\\Downloady'); + sys.path.append('C:\\git\\else\\Clipext') + sys.path.append('C:\\git\\else\\Downloady') import clipext import downloady except ImportError: diff --git a/WebstreamZip/test.zip b/WebstreamZip/test.zip new file mode 100644 index 0000000..a2f3957 Binary files /dev/null and b/WebstreamZip/test.zip differ diff --git a/WebstreamZip/tester.py b/WebstreamZip/tester.py new file mode 100644 index 0000000..f158cfb --- /dev/null +++ b/WebstreamZip/tester.py @@ -0,0 +1,7 @@ +import webstreamzip + +g = webstreamzip.stream_zip({'C:\\git\\else\\readme.md':'michael.md'}) + +x = open('test.zip', 'wb') +for chunk in g: + x.write(chunk) \ No newline at end of file diff --git a/WebstreamZip/webstreamzip.py b/WebstreamZip/webstreamzip.py new file mode 100644 index 0000000..9f92781 --- /dev/null +++ b/WebstreamZip/webstreamzip.py @@ -0,0 +1,81 @@ +import threading +import zipfile +import tarfile + + +class Stream: + def __init__(self): + self.active_chunk = None + self.write_ready = threading.Event() + self.read_ready = threading.Event() + self.write_ready.set() + self.all_done = False + + def write(self, chunk): + if chunk == b'': + self.all_done = True + + if self.all_done: + return 0 + + self.write_ready.wait() + #print('writing', chunk) + self.active_chunk = chunk + self.write_ready.clear() + self.read_ready.set() + return 0 + + def tell(self): + return 0 + + def read(self, size=None): + if self.all_done: + return b'' + + self.read_ready.wait() + chunk = self.active_chunk + self.active_chunk = None + self.read_ready.clear() + self.write_ready.set() + return chunk + + def close(self): + #print('Internal close') + self.all_done = True + while not self.write_ready.is_set(): + #print('writeset') + self.write_ready.set() + while not self.read_ready.is_set(): + #print('readset') + self.read_ready.set() + + def flush(self): + #print('flushy') + pass + +def _stream_tar(memstream, filenames): + z = tarfile.TarFile(fileobj=memstream, mode='w') + if isinstance(filenames, dict): + for (realpath, fakepath) in filenames.items(): + z.add(realpath, arcname=fakepath) + else: + for filename in filenames: + z.add(filename) + #print('zclosing') + z.close() + memstream.close() + #print('zclosed') + return + +def stream_tar(filenames): + memstream = Stream() + zip_thread = threading.Thread(target=_stream_tar, args=(memstream, filenames)) + zip_thread.start() + while True: + #print('reading') + chunk = memstream.read() + yield chunk + if not chunk: + break + memstream.close() + #print('ending') diff --git a/_voussoirkit/voussoirkit.zip b/_voussoirkit/voussoirkit.zip index c1c6834..78d4fe9 100644 Binary files a/_voussoirkit/voussoirkit.zip and b/_voussoirkit/voussoirkit.zip differ