feat: improve 429 handling with Tor support and clean CI
- Retry with new Tor identity on 429 - Improve error logging - Remove .build.yml and .drone.yml
This commit is contained in:
12
.build.yml
12
.build.yml
@@ -1,12 +0,0 @@
|
|||||||
image: debian/buster
|
|
||||||
packages:
|
|
||||||
- python3-pip
|
|
||||||
- virtualenv
|
|
||||||
tasks:
|
|
||||||
- test: |
|
|
||||||
cd yt-local
|
|
||||||
virtualenv -p python3 venv
|
|
||||||
source venv/bin/activate
|
|
||||||
python --version
|
|
||||||
pip install -r requirements-dev.txt
|
|
||||||
pytest
|
|
||||||
10
.drone.yml
10
.drone.yml
@@ -1,10 +0,0 @@
|
|||||||
kind: pipeline
|
|
||||||
name: default
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: test
|
|
||||||
image: python:3.7.3
|
|
||||||
commands:
|
|
||||||
- pip install --upgrade pip
|
|
||||||
- pip install -r requirements-dev.txt
|
|
||||||
- pytest
|
|
||||||
@@ -372,10 +372,16 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None,
|
|||||||
if not use_tor or not settings.route_tor:
|
if not use_tor or not settings.route_tor:
|
||||||
logger.warning(f'YouTube returned 429 but Tor is not enabled. Consider enabling Tor routing.')
|
logger.warning(f'YouTube returned 429 but Tor is not enabled. Consider enabling Tor routing.')
|
||||||
raise FetchError('429', reason=response.reason, ip=ip)
|
raise FetchError('429', reason=response.reason, ip=ip)
|
||||||
|
else:
|
||||||
|
# Tor is enabled but we've exhausted retries
|
||||||
|
logger.error(f'YouTube blocked request - Tor exit node overutilized after {max_retries} retries. Exit IP: {ip}')
|
||||||
|
raise FetchError('429', reason=response.reason, ip=ip,
|
||||||
|
error_message='Tor exit node overutilized after multiple retries')
|
||||||
|
|
||||||
logger.error(f'YouTube blocked request - Tor exit node overutilized. Exit IP: {ip}')
|
# For Tor: get new identity immediately on 429
|
||||||
|
if use_tor and settings.route_tor:
|
||||||
|
logger.info(f'YouTube blocked request - Tor exit node overutilized. Exit IP: {ip}. Getting new identity...')
|
||||||
|
|
||||||
# get new identity
|
|
||||||
error = tor_manager.new_identity(start_time)
|
error = tor_manager.new_identity(start_time)
|
||||||
if error:
|
if error:
|
||||||
raise FetchError(
|
raise FetchError(
|
||||||
@@ -384,7 +390,7 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None,
|
|||||||
else:
|
else:
|
||||||
continue # retry with new identity
|
continue # retry with new identity
|
||||||
|
|
||||||
# Calculate delay with exponential backoff and jitter
|
# For non-Tor: exponential backoff
|
||||||
delay = (base_delay * (2 ** attempt)) + random.uniform(0, 1)
|
delay = (base_delay * (2 ** attempt)) + random.uniform(0, 1)
|
||||||
logger.info(f'Rate limited (429). Waiting {delay:.1f}s before retry {attempt + 1}/{max_retries}...')
|
logger.info(f'Rate limited (429). Waiting {delay:.1f}s before retry {attempt + 1}/{max_retries}...')
|
||||||
time.sleep(delay)
|
time.sleep(delay)
|
||||||
|
|||||||
Reference in New Issue
Block a user