Add more ways of normalizing given request headers.
This commit is contained in:
parent
06dcd31019
commit
4211e85afc
1 changed files with 32 additions and 8 deletions
|
@ -61,6 +61,37 @@ def download_job(
|
|||
)
|
||||
log.info(f'Finished "{filename}"')
|
||||
|
||||
def normalize_headers(headers):
|
||||
if headers is None:
|
||||
return {}
|
||||
|
||||
if not headers:
|
||||
return {}
|
||||
|
||||
if isinstance(headers, dict):
|
||||
return headers
|
||||
|
||||
if isinstance(headers, list) and len(headers) == 1:
|
||||
headers = headers[0]
|
||||
|
||||
if isinstance(headers, (list, tuple)):
|
||||
keys = headers[::2]
|
||||
vals = headers[1::2]
|
||||
return {key: val for (key, val) in zip(keys, vals)}
|
||||
|
||||
if isinstance(headers, str) and os.path.isfile(headers):
|
||||
headers = pathclass.Path(headers).readlines('r', encoding='utf-8')
|
||||
|
||||
if isinstance(headers, str):
|
||||
if headers.startswith('{'):
|
||||
return ast.literal_eval(headers)
|
||||
else:
|
||||
lines = [line for line in headers.splitlines() if line.strip()]
|
||||
pairs = [line.strip().split(': ', 1) for line in lines]
|
||||
return {key: value for (key, value) in pairs}
|
||||
|
||||
return headers
|
||||
|
||||
def prepare_urls_filenames(urls, filename_format):
|
||||
now = int(time.time())
|
||||
|
||||
|
@ -173,14 +204,7 @@ def threaded_dl_argparse(args):
|
|||
urls = pipeable.input(args.url_file, read_files=True, skip_blank=True, strip=True)
|
||||
urls = [u.split(' ', 1) if ' ' in u else u for u in urls]
|
||||
|
||||
headers = args.headers
|
||||
if headers is not None:
|
||||
if len(headers) == 1 and headers[0].startswith('{'):
|
||||
headers = ast.literal_eval(headers[0])
|
||||
else:
|
||||
keys = headers[::2]
|
||||
vals = headers[1::2]
|
||||
headers = {key: val for (key, val) in zip(keys, vals)}
|
||||
headers = normalize_headers(args.headers)
|
||||
|
||||
bytespersecond = args.bytespersecond
|
||||
if bytespersecond is not None:
|
||||
|
|
Loading…
Reference in a new issue