remove yt-dlp, fix captions PO Token issue, fix 429 retry logic
- Remove yt-dlp entirely (modules, routes, settings, dependency) Was blocking page loads by running synchronously in gevent - Fix captions: use Android client caption URLs (no PO Token needed) instead of web timedtext URLs that YouTube now blocks - Fix 429 retry: fail immediately without Tor (same IP = pointless retry) Was causing ~27s delays with exponential backoff - Accept ytdlp_enabled as legacy setting to avoid warning on startup
This commit is contained in:
@@ -367,34 +367,25 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None,
|
||||
response.getheader('Set-Cookie') or '')
|
||||
ip = ip.group(1) if ip else None
|
||||
|
||||
# If this is the last attempt, raise error
|
||||
# Without Tor, no point retrying with same IP
|
||||
if not use_tor or not settings.route_tor:
|
||||
logger.warning('Rate limited (429). Enable Tor routing to retry with new IP.')
|
||||
raise FetchError('429', reason=response.reason, ip=ip)
|
||||
|
||||
# Tor: exhausted retries
|
||||
if attempt >= max_retries - 1:
|
||||
if not use_tor or not settings.route_tor:
|
||||
logger.warning(f'YouTube returned 429 but Tor is not enabled. Consider enabling Tor routing.')
|
||||
raise FetchError('429', reason=response.reason, ip=ip)
|
||||
else:
|
||||
# Tor is enabled but we've exhausted retries
|
||||
logger.error(f'YouTube blocked request - Tor exit node overutilized after {max_retries} retries. Exit IP: {ip}')
|
||||
raise FetchError('429', reason=response.reason, ip=ip,
|
||||
error_message='Tor exit node overutilized after multiple retries')
|
||||
logger.error(f'Rate limited after {max_retries} retries. Exit IP: {ip}')
|
||||
raise FetchError('429', reason=response.reason, ip=ip,
|
||||
error_message='Tor exit node overutilized after multiple retries')
|
||||
|
||||
# For Tor: get new identity immediately on 429
|
||||
if use_tor and settings.route_tor:
|
||||
logger.info(f'YouTube blocked request - Tor exit node overutilized. Exit IP: {ip}. Getting new identity...')
|
||||
|
||||
error = tor_manager.new_identity(start_time)
|
||||
if error:
|
||||
raise FetchError(
|
||||
'429', reason=response.reason, ip=ip,
|
||||
error_message='Automatic circuit change: ' + error)
|
||||
else:
|
||||
continue # retry with new identity
|
||||
|
||||
# For non-Tor: exponential backoff
|
||||
delay = (base_delay * (2 ** attempt)) + random.uniform(0, 1)
|
||||
logger.info(f'Rate limited (429). Waiting {delay:.1f}s before retry {attempt + 1}/{max_retries}...')
|
||||
time.sleep(delay)
|
||||
continue # retry
|
||||
# Tor: get new identity and retry
|
||||
logger.info(f'Rate limited. Getting new Tor identity... (IP: {ip})')
|
||||
error = tor_manager.new_identity(start_time)
|
||||
if error:
|
||||
raise FetchError(
|
||||
'429', reason=response.reason, ip=ip,
|
||||
error_message='Automatic circuit change: ' + error)
|
||||
continue # retry with new identity
|
||||
|
||||
# Check for client errors (400, 404) - don't retry these
|
||||
if response.status == 400:
|
||||
|
||||
Reference in New Issue
Block a user