From 8e5f50323dcf1f16ccf6b89176cad638d52d06e9 Mon Sep 17 00:00:00 2001 From: Ethan Dalool Date: Tue, 27 Oct 2020 22:28:42 -0700 Subject: [PATCH] Add 2s timeout to get requests. The hn api usually returns in less than 0.25 seconds, but occasionally I'm getting huge latency spikes. A quick timeout and retry solves that for me. If you are using this tool with a very high baseline ping, 2s may be too aggressive. --- hnarchive.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hnarchive.py b/hnarchive.py index da41851..c80aa09 100644 --- a/hnarchive.py +++ b/hnarchive.py @@ -69,7 +69,7 @@ def get(url, retries=1): while retries > 0: log.loud(url) try: - response = requests.get(url, headers=HEADERS) + response = requests.get(url, headers=HEADERS, timeout=2) response.raise_for_status() break except requests.exceptions.HTTPError as exc: @@ -80,7 +80,8 @@ def get(url, retries=1): retries -= 1 log.loud('Request failed, %d tries remain.', retries) time.sleep(bo.next()) - except requests.exceptions.ConnectionError: + except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout): + log.loud('Request failed, %d tries remain.', retries) time.sleep(bo.next()) end_time = time.time()