else

Downloady/downloady.py

@@ -9,8 +9,8 @@ import warnings
 
 try:
     sys.path.append('C:\\git\\else\\Bytestring')
-    sys.path.append('C:\\git\\else\\clipext');
+    sys.path.append('C:\\git\\else\\clipext')
-    sys.path.append('C:\\git\\else\\ratelimiter');
+    sys.path.append('C:\\git\\else\\ratelimiter')
     import bytestring
     import ratelimiter
     import clipext

Instathief/instathief.py

@@ -7,8 +7,8 @@ import requests
 import sys
 
 try:
-    sys.path.append('C:\\git\\else\\Clipext');
+    sys.path.append('C:\\git\\else\\Clipext')
-    sys.path.append('C:\\git\\else\\Downloady');
+    sys.path.append('C:\\git\\else\\Downloady')
     import clipext
     import downloady
 except ImportError:

OpenDirDL/opendirdl.py

@@ -113,6 +113,7 @@ tree:
 import argparse
 ## import bs4
 import collections
+import concurrent.futures
 ## import hashlib
 import os
 ## import re
@@ -124,8 +125,8 @@ import sys
 import urllib.parse
 
 try:
-    sys.path.append('C:\\git\\else\\Bytestring');
+    sys.path.append('C:\\git\\else\\Bytestring')
-    sys.path.append('C:\\git\\else\\Downloady');
+    sys.path.append('C:\\git\\else\\Downloady')
     import bytestring
     import downloady
 except ImportError:
@@ -299,6 +300,7 @@ class Walker:
         self.fullscan = bool(fullscan)
         self.queue = collections.deque()
         self.seen_directories = set()
+        self.threadpool = concurrent.futures.ThreadPoolExecutor(4)
 
     def smart_insert(self, url=None, head=None, commit=True):
         '''

OpenDirDL/requirements.txt

@@ -0,0 +1,3 @@
+https://github.com/voussoir/else/raw/master/_voussoirkit/voussoirkit.zip
+bs4
+requests

Pathclass/pathclass.py

@@ -10,13 +10,19 @@ class Path:
         self.absolute_path = path
 
     def __contains__(self, other):
-        return other.absolute_path.startswith(self.absolute_path)
+        this = os.path.normcase(self.absolute_path)
+        that = os.path.normcase(other.absolute_path)
+        return that.startswith(this)
 
     def __eq__(self, other):
-        return hasattr(other, 'absolute_path') and self.absolute_path == other.absolute_path
+        if not hasattr(other, 'absolute_path'):
+            return False
+        this = os.path.normcase(self.absolute_path)
+        that = os.path.normcase(other.absolute_path)
+        return this == that
 
     def __hash__(self):
-        return hash(self.absolute_path)
+        return hash(os.path.normcase(self.absolute_path))
 
     def __repr__(self):
         return '{c}({path})'.format(c=self.__class__.__name__, path=repr(self.absolute_path))

ServerReference/simpleserver.py

@@ -9,9 +9,9 @@ import sys
 import types
 
 try:
-    sys.path.append('C:\\git\\else\\Bytestring');
+    sys.path.append('C:\\git\\else\\Bytestring')
-    sys.path.append('C:\\git\\else\\Pathclass');
+    sys.path.append('C:\\git\\else\\Pathclass')
-    sys.path.append('C:\\git\\else\\Ratelimiter');
+    sys.path.append('C:\\git\\else\\Ratelimiter')
     import bytestring
     import pathclass
     import ratelimiter

SpinalTap/spinal.py

@@ -11,9 +11,9 @@ import sys
 import time
 
 try:
-    sys.path.append('C:\\git\\else\\Bytestring');
+    sys.path.append('C:\\git\\else\\Bytestring')
-    sys.path.append('C:\\git\\else\\Pathclass');
+    sys.path.append('C:\\git\\else\\Pathclass')
-    sys.path.append('C:\\git\\else\\Ratelimiter');
+    sys.path.append('C:\\git\\else\\Ratelimiter')
     import bytestring
     import pathclass
     import ratelimiter
@@ -24,7 +24,7 @@ except ImportError:
     from voussoirkit import pathclass
     from voussoirkit import ratelimiter
 
-logging.basicConfig(level=logging.DEBUG)
+logging.basicConfig(level=logging.CRITICAL)
 log = logging.getLogger(__name__)
 
 CHUNK_SIZE = 128 * bytestring.KIBIBYTE

ThreadedDL/threaded_dl.py

@@ -4,8 +4,8 @@ import threading
 import time
 
 try:
-    sys.path.append('C:\\git\\else\\Clipext');
+    sys.path.append('C:\\git\\else\\Clipext')
-    sys.path.append('C:\\git\\else\\Downloady');
+    sys.path.append('C:\\git\\else\\Downloady')
     import clipext
     import downloady
 except ImportError:

WebstreamZip/tester.py

@@ -0,0 +1,7 @@
+import webstreamzip
+
+g = webstreamzip.stream_zip({'C:\\git\\else\\readme.md':'michael.md'})
+
+x = open('test.zip', 'wb')
+for chunk in g:
+    x.write(chunk)

WebstreamZip/webstreamzip.py

@@ -0,0 +1,81 @@
+import threading
+import zipfile
+import tarfile
+
+
+class Stream:
+    def __init__(self):
+        self.active_chunk = None
+        self.write_ready = threading.Event()
+        self.read_ready = threading.Event()
+        self.write_ready.set()
+        self.all_done = False
+
+    def write(self, chunk):
+        if chunk == b'':
+            self.all_done = True
+
+        if self.all_done:
+            return 0
+
+        self.write_ready.wait()
+        #print('writing', chunk)
+        self.active_chunk = chunk
+        self.write_ready.clear()
+        self.read_ready.set()
+        return 0
+
+    def tell(self):
+        return 0
+
+    def read(self, size=None):
+        if self.all_done:
+            return b''
+
+        self.read_ready.wait()
+        chunk = self.active_chunk
+        self.active_chunk = None
+        self.read_ready.clear()
+        self.write_ready.set()
+        return chunk
+
+    def close(self):
+        #print('Internal close')
+        self.all_done = True
+        while not self.write_ready.is_set():
+            #print('writeset')
+            self.write_ready.set()
+        while not self.read_ready.is_set():
+            #print('readset')
+            self.read_ready.set()
+
+    def flush(self):
+        #print('flushy')
+        pass
+
+def _stream_tar(memstream, filenames):
+    z = tarfile.TarFile(fileobj=memstream, mode='w')
+    if isinstance(filenames, dict):
+        for (realpath, fakepath) in filenames.items():
+            z.add(realpath, arcname=fakepath)
+    else:
+        for filename in filenames:
+            z.add(filename)
+    #print('zclosing')
+    z.close()
+    memstream.close()
+    #print('zclosed')
+    return
+
+def stream_tar(filenames):
+    memstream = Stream()
+    zip_thread = threading.Thread(target=_stream_tar, args=(memstream, filenames))
+    zip_thread.start()
+    while True:
+        #print('reading')
+        chunk = memstream.read()
+        yield chunk
+        if not chunk:
+            break
+    memstream.close()
+    #print('ending')