Give a proper error message for 429 errors

These occur when too many requests are coming from a Tor exit node.
Before, there would be an error page with an exception instructing users to report the issue.
But this is an expected and persistent issue.
This commit is contained in:
James Taylor
2020-01-31 20:06:15 -08:00
parent cd4a2fb0eb
commit f787e4e202
6 changed files with 49 additions and 3 deletions

View File

@@ -97,6 +97,12 @@ class HTTPAsymmetricCookieProcessor(urllib.request.BaseHandler):
https_request = http_request
https_response = http_response
class FetchError(Exception):
def __init__(self, code, reason='', ip=None):
Exception.__init__(self, 'HTTP error during request: ' + code + ' ' + reason)
self.code = code
self.reason = reason
self.ip = ip
def decode_content(content, encoding_header):
encodings = encoding_header.replace(' ', '').split(',')
@@ -161,6 +167,17 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookieja
content = response.read()
response.release_conn()
if (response.status == 429
and content.startswith(b'<!DOCTYPE')
and b'Our systems have detected unusual traffic' in content):
ip = re.search(br'IP address: ((?:[\da-f]*:)+[\da-f]+|(?:\d+\.)+\d+)',
content)
ip = ip.group(1).decode('ascii') if ip else None
raise FetchError('429', reason=response.reason, ip=ip)
elif response.status >= 400:
raise FetchError(str(response.status), reason=response.reason, ip=None)
read_finish = time.time()
if report_text:
print(report_text, ' Latency:', round(response_time - start_time,3), ' Read time:', round(read_finish - response_time,3))
@@ -359,3 +376,9 @@ def parse_info_prepare_for_html(renderer, additional_info={}):
add_extra_html_info(item)
return item
def check_gevent_exceptions(*tasks):
for task in tasks:
if task.exception:
raise task.exception