Slight hack: allow pre-given filenames in threaded_dl.

This commit is contained in:
voussoir 2020-10-25 20:15:11 -07:00
parent 0c481f454f
commit 9e65971829

View file

@ -12,6 +12,10 @@ from voussoirkit import downloady
def clean_url_list(urls): def clean_url_list(urls):
for url in urls: for url in urls:
if isinstance(url, (tuple, list)):
(url, filename) = url
else:
filename = None
url = url.strip() url = url.strip()
if not url: if not url:
@ -20,7 +24,10 @@ def clean_url_list(urls):
if url.startswith('#'): if url.startswith('#'):
continue continue
yield url if filename:
yield (url, filename)
else:
yield url
def download_thread(url, filename, *, bytespersecond=None, headers=None, timeout=None): def download_thread(url, filename, *, bytespersecond=None, headers=None, timeout=None):
print(f' Starting "{filename}"') print(f' Starting "{filename}"')
@ -61,15 +68,18 @@ def threaded_dl(
threads = remove_finished(threads) threads = remove_finished(threads)
time.sleep(0.1) time.sleep(0.1)
basename = downloady.basename_from_url(url) if isinstance(url, (tuple, list)):
extension = os.path.splitext(basename)[1] (url, filename) = url
filename = filename_format.format( else:
basename=basename, basename = downloady.basename_from_url(url)
ext=extension, extension = os.path.splitext(basename)[1]
extension=extension, filename = filename_format.format(
index=index, basename=basename,
now=now, ext=extension,
) extension=extension,
index=index,
now=now,
)
if os.path.exists(filename): if os.path.exists(filename):
print(f'Skipping existing file "{filename}"') print(f'Skipping existing file "{filename}"')
@ -100,6 +110,8 @@ def threaded_dl_argparse(args):
urls = clipext.resolve(args.url_file) urls = clipext.resolve(args.url_file)
urls = urls.replace('\r', '').split('\n') urls = urls.replace('\r', '').split('\n')
urls = [u.split(' ', 1) if ' ' in u else u for u in urls]
headers = args.headers headers = args.headers
if headers is not None: if headers is not None:
if len(headers) == 1 and headers[0].startswith('{'): if len(headers) == 1 and headers[0].startswith('{'):