voussoir/hnarchive
1
0
Fork 0
Code Issues Activity

Use voussoirkit/httperrors to smooth out this function.

Browse source
This commit is contained in:
voussoir 2021-10-18 18:23:43 -07:00
parent 64a299c479
commit 45a9d2eefa
No known key found for this signature in database
GPG key ID: 5F7554F8C26DACCB
1 changed files with 12 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
hnarchive.py
View file

@ -7,6 +7,7 @@ import time
from voussoirkit import backoff from voussoirkit import backoff
from voussoirkit import betterhelp from voussoirkit import betterhelp
from voussoirkit import httperrors
from voussoirkit import mutables from voussoirkit import mutables
from voussoirkit import operatornotify from voussoirkit import operatornotify
from voussoirkit import ratelimiter from voussoirkit import ratelimiter
@ -76,24 +77,23 @@ def int_or_none(x):
def get(url, retries=1): def get(url, retries=1):
bo = backoff.Quadratic(a=0.2, b=0, c=1, max=10) bo = backoff.Quadratic(a=0.2, b=0, c=1, max=10)
while retries > 0: while retries > 0:
log.loud(url)
try: try:
log.loud(url)
response = session.get(url, timeout=2) response = session.get(url, timeout=2)
response.raise_for_status() httperrors.raise_for_status(response)
break return response
except requests.exceptions.HTTPError as exc: except (
if exc.response.status_code == 429: httperrors.HTTP429,
pass httperrors.HTTP5XX,
elif 400 <= exc.response.status_code <= 499: requests.exceptions.ConnectionError,
raise requests.exceptions.ReadTimeout,
):
# Any other 4XX should raise.
retries -= 1 retries -= 1
log.loud('Request failed, %d tries remain.', retries) log.loud('Request failed, %d tries remain.', retries)
time.sleep(bo.next()) time.sleep(bo.next())
except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout):
log.loud('Request failed, %d tries remain.', retries)
time.sleep(bo.next())
return response raise RuntimeError(f'Ran out of retries on {url}.')
def get_item(id): def get_item(id):
url = f'https://hacker-news.firebaseio.com/v0/item/{id}.json' url = f'https://hacker-news.firebaseio.com/v0/item/{id}.json'