feat: improve 429 handling with Tor support and clean CI
- Retry with new Tor identity on 429 - Improve error logging - Remove .build.yml and .drone.yml
This commit is contained in:
@@ -372,10 +372,16 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None,
|
||||
if not use_tor or not settings.route_tor:
|
||||
logger.warning(f'YouTube returned 429 but Tor is not enabled. Consider enabling Tor routing.')
|
||||
raise FetchError('429', reason=response.reason, ip=ip)
|
||||
else:
|
||||
# Tor is enabled but we've exhausted retries
|
||||
logger.error(f'YouTube blocked request - Tor exit node overutilized after {max_retries} retries. Exit IP: {ip}')
|
||||
raise FetchError('429', reason=response.reason, ip=ip,
|
||||
error_message='Tor exit node overutilized after multiple retries')
|
||||
|
||||
logger.error(f'YouTube blocked request - Tor exit node overutilized. Exit IP: {ip}')
|
||||
# For Tor: get new identity immediately on 429
|
||||
if use_tor and settings.route_tor:
|
||||
logger.info(f'YouTube blocked request - Tor exit node overutilized. Exit IP: {ip}. Getting new identity...')
|
||||
|
||||
# get new identity
|
||||
error = tor_manager.new_identity(start_time)
|
||||
if error:
|
||||
raise FetchError(
|
||||
@@ -384,7 +390,7 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None,
|
||||
else:
|
||||
continue # retry with new identity
|
||||
|
||||
# Calculate delay with exponential backoff and jitter
|
||||
# For non-Tor: exponential backoff
|
||||
delay = (base_delay * (2 ** attempt)) + random.uniform(0, 1)
|
||||
logger.info(f'Rate limited (429). Waiting {delay:.1f}s before retry {attempt + 1}/{max_retries}...')
|
||||
time.sleep(delay)
|
||||
|
||||
Reference in New Issue
Block a user