heckyel/yt-local
0
0
Fork 0
Code Issues Pull Requests Releases Activity

Add tor video routing

Browse Source 
Includes non-tor video routing by default, so no more chances
of the browser leaking headers or user agent to googlevideo
Adjust settings upgrade system to facilitate change to route_tor
setting.
Add some more space on settings page for dropdown settings so does
not overflow due to options with long names.
Closes #7
This commit is contained in:
James Taylor
2020-09-18 14:37:24 -07:00
parent 1ff97bfde1
commit e9989af03a
5 changed files with 99 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
youtube/templates/settings.html
View File

@@ -4,7 +4,7 @@
{% block style %}
.settings-form {
margin: auto;
width: 500px;
width: 600px;
margin-top:10px;
padding: 10px;
display: block;

37
youtube/util.py
View File

@@ -119,8 +119,11 @@ def decode_content(content, encoding_header):
content = gzip.decompress(content)
return content
def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookiejar_send=None, cookiejar_receive=None, use_tor=True, return_response=False, debug_name=None):
def fetch_url_response(url, headers=(), timeout=15, data=None,
cookiejar_send=None, cookiejar_receive=None,
use_tor=True):
'''
returns response, cleanup_function
When cookiejar_send is set to a CookieJar object,
those cookies will be sent in the request (but cookies in response will not be merged into it)
When cookiejar_receive is set to a CookieJar object,
@@ -147,8 +150,6 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookieja
elif not isinstance(data, bytes):
data = urllib.parse.urlencode(data).encode('ascii')
start_time = time.time()
if cookiejar_send is not None or cookiejar_receive is not None: # Use urllib
req = urllib.request.Request(url, data=data, headers=headers)
@@ -160,19 +161,30 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookieja
opener = urllib.request.build_opener(cookie_processor)
response = opener.open(req, timeout=timeout)
response_time = time.time()
content = response.read()
cleanup_func = (lambda r: None)
else: # Use a urllib3 pool. Cookies can't be used since urllib3 doesn't have easy support for them.
pool = get_pool(use_tor and settings.route_tor)
response = pool.request(method, url, headers=headers, timeout=timeout, preload_content=False, decode_content=False)
response_time = time.time()
cleanup_func = (lambda r: r.release_conn())
content = response.read()
response.release_conn()
return response, cleanup_func
def fetch_url(url, headers=(), timeout=15, report_text=None, data=None,
cookiejar_send=None, cookiejar_receive=None, use_tor=True,
debug_name=None):
start_time = time.time()
response, cleanup_func = fetch_url_response(
url, headers, timeout=timeout,
cookiejar_send=cookiejar_send, cookiejar_receive=cookiejar_receive,
use_tor=use_tor)
response_time = time.time()
content = response.read()
read_finish = time.time()
cleanup_func(response) # release_connection for urllib3
if (response.status == 429
and content.startswith(b'<!DOCTYPE')
@@ -185,7 +197,6 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookieja
elif response.status >= 400:
raise FetchError(str(response.status), reason=response.reason, ip=None)
read_finish = time.time()
if report_text:
print(report_text, ' Latency:', round(response_time - start_time,3), ' Read time:', round(read_finish - response_time,3))
content = decode_content(content, response.getheader('Content-Encoding', default='identity'))
@@ -198,8 +209,6 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookieja
with open(os.path.join(save_dir, debug_name), 'wb') as f:
f.write(content)
if return_response:
return content, response
return content
def head(url, use_tor=False, report_text=None, max_redirects=10):

14
youtube/watch.py
View File

@@ -24,7 +24,7 @@ except FileNotFoundError:
def get_video_sources(info):
video_sources = []
if not settings.theater_mode:
if (not settings.theater_mode) or settings.route_tor == 2:
max_resolution = 360
else:
max_resolution = settings.default_resolution
@@ -270,10 +270,11 @@ def extract_info(video_id, use_invidious, playlist_id=None, index=None):
else:
info['hls_formats'] = []
# check for 403
# check for 403. Unnecessary for tor video routing b/c ip address is same
info['invidious_used'] = False
info['invidious_reload_button'] = False
if settings.route_tor and info['formats'] and info['formats'][0]['url']:
if (settings.route_tor == 1
and info['formats'] and info['formats'][0]['url']):
try:
response = util.head(info['formats'][0]['url'],
report_text='Checked for URL access')
@@ -408,10 +409,10 @@ def get_watch_page(video_id=None):
"author": info['author'],
}
# prefix urls, and other post-processing not handled by yt_data_extract
for item in info['related_videos']:
util.prefix_urls(item)
util.add_extra_html_info(item)
if info['playlist']:
playlist_id = info['playlist']['id']
for item in info['playlist']['items']:
@@ -423,6 +424,11 @@ def get_watch_page(video_id=None):
item['url'] += '&index=' + str(item['index'])
info['playlist']['author_url'] = util.prefix_url(
info['playlist']['author_url'])
# Don't prefix hls_formats for now because the urls inside the manifest
# would need to be prefixed as well.
for fmt in info['formats']:
fmt['url'] = util.prefix_url(fmt['url'])
if settings.gather_googlevideo_domains:
with open(os.path.join(settings.data_dir, 'googlevideo-domains.txt'), 'a+', encoding='utf-8') as f: