Refactoring code and reuse INNERTUBE_CLIENTS
This commit is contained in:
parent
8f00cbcdd6
commit
8af98968dd
GPG Key ID:
97504AF0027B1A56
@ -431,34 +431,29 @@ class RateLimitedQueue(gevent.queue.Queue):
|
|||||||
gevent.queue.Queue.__init__(self)
|
gevent.queue.Queue.__init__(self)
|
||||||
|
|
||||||
def get(self):
|
def get(self):
|
||||||
self.lock.acquire() # blocks if another greenlet currently has the lock
|
with self.lock: # blocks if another greenlet currently has the lock
|
||||||
if self.count_since_last_wait >= self.subsequent_bursts and self.surpassed_initial:
|
if ((self.count_since_last_wait >= self.subsequent_bursts and self.surpassed_initial) or
|
||||||
gevent.sleep(self.waiting_period)
|
(self.count_since_last_wait >= self.initial_burst and not self.surpassed_initial)):
|
||||||
self.count_since_last_wait = 0
|
self.surpassed_initial = True
|
||||||
|
gevent.sleep(self.waiting_period)
|
||||||
elif self.count_since_last_wait >= self.initial_burst and not self.surpassed_initial:
|
|
||||||
self.surpassed_initial = True
|
|
||||||
gevent.sleep(self.waiting_period)
|
|
||||||
self.count_since_last_wait = 0
|
|
||||||
|
|
||||||
self.count_since_last_wait += 1
|
|
||||||
|
|
||||||
if not self.currently_empty and self.empty():
|
|
||||||
self.currently_empty = True
|
|
||||||
self.empty_start = time.monotonic()
|
|
||||||
|
|
||||||
item = gevent.queue.Queue.get(self) # blocks when nothing left
|
|
||||||
|
|
||||||
if self.currently_empty:
|
|
||||||
if time.monotonic() - self.empty_start >= self.waiting_period:
|
|
||||||
self.count_since_last_wait = 0
|
self.count_since_last_wait = 0
|
||||||
self.surpassed_initial = False
|
|
||||||
|
|
||||||
self.currently_empty = False
|
self.count_since_last_wait += 1
|
||||||
|
|
||||||
self.lock.release()
|
if not self.currently_empty and self.empty():
|
||||||
|
self.currently_empty = True
|
||||||
|
self.empty_start = time.monotonic()
|
||||||
|
|
||||||
return item
|
item = gevent.queue.Queue.get(self) # blocks when nothing left
|
||||||
|
|
||||||
|
if self.currently_empty:
|
||||||
|
if time.monotonic() - self.empty_start >= self.waiting_period:
|
||||||
|
self.count_since_last_wait = 0
|
||||||
|
self.surpassed_initial = False
|
||||||
|
|
||||||
|
self.currently_empty = False
|
||||||
|
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
def download_thumbnail(save_directory, video_id):
|
def download_thumbnail(save_directory, video_id):
|
||||||
@ -667,25 +662,6 @@ def to_valid_filename(name):
|
|||||||
|
|
||||||
# https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/extractor/youtube.py#L72
|
# https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/extractor/youtube.py#L72
|
||||||
INNERTUBE_CLIENTS = {
|
INNERTUBE_CLIENTS = {
|
||||||
'android_music': {
|
|
||||||
'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
|
|
||||||
'INNERTUBE_CONTEXT': {
|
|
||||||
'client': {
|
|
||||||
'hl': 'en',
|
|
||||||
'gl': 'US',
|
|
||||||
'clientName': 'ANDROID_MUSIC',
|
|
||||||
'clientVersion': '6.48.51',
|
|
||||||
'osName': 'Android',
|
|
||||||
'osVersion': '14',
|
|
||||||
'androidSdkVersion': 34,
|
|
||||||
'platform': 'MOBILE',
|
|
||||||
'userAgent': 'com.google.android.apps.youtube.music/6.48.51 (Linux; U; Android 14; US) gzip'
|
|
||||||
}
|
|
||||||
},
|
|
||||||
'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
|
|
||||||
'REQUIRE_JS_PLAYER': False
|
|
||||||
},
|
|
||||||
|
|
||||||
'android': {
|
'android': {
|
||||||
'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
|
'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
|
||||||
'INNERTUBE_CONTEXT': {
|
'INNERTUBE_CONTEXT': {
|
||||||
@ -721,6 +697,25 @@ INNERTUBE_CLIENTS = {
|
|||||||
'REQUIRE_JS_PLAYER': False
|
'REQUIRE_JS_PLAYER': False
|
||||||
},
|
},
|
||||||
|
|
||||||
|
'android_music': {
|
||||||
|
'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
|
||||||
|
'INNERTUBE_CONTEXT': {
|
||||||
|
'client': {
|
||||||
|
'hl': 'en',
|
||||||
|
'gl': 'US',
|
||||||
|
'clientName': 'ANDROID_MUSIC',
|
||||||
|
'clientVersion': '6.48.51',
|
||||||
|
'osName': 'Android',
|
||||||
|
'osVersion': '14',
|
||||||
|
'androidSdkVersion': 34,
|
||||||
|
'platform': 'MOBILE',
|
||||||
|
'userAgent': 'com.google.android.apps.youtube.music/6.48.51 (Linux; U; Android 14; US) gzip'
|
||||||
|
}
|
||||||
|
},
|
||||||
|
'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
|
||||||
|
'REQUIRE_JS_PLAYER': False
|
||||||
|
},
|
||||||
|
|
||||||
# This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
|
# This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
|
||||||
# See: https://github.com/zerodytrash/YouTube-Internal-Clients
|
# See: https://github.com/zerodytrash/YouTube-Internal-Clients
|
||||||
'tv_embedded': {
|
'tv_embedded': {
|
||||||
|
|||||||
143
youtube/watch.py
143
youtube/watch.py
@ -2,6 +2,7 @@ import youtube
|
|||||||
from youtube import yt_app
|
from youtube import yt_app
|
||||||
from youtube import util, comments, local_playlist, yt_data_extract
|
from youtube import util, comments, local_playlist, yt_data_extract
|
||||||
from youtube.util import time_utc_isoformat
|
from youtube.util import time_utc_isoformat
|
||||||
|
from youtube.util import INNERTUBE_CLIENTS
|
||||||
import settings
|
import settings
|
||||||
|
|
||||||
from flask import request
|
from flask import request
|
||||||
@ -369,83 +370,83 @@ def fetch_watch_page_info(video_id, playlist_id, index):
|
|||||||
return yt_data_extract.extract_watch_info_from_html(watch_page)
|
return yt_data_extract.extract_watch_info_from_html(watch_page)
|
||||||
|
|
||||||
def extract_info(video_id, use_invidious, playlist_id=None, index=None):
|
def extract_info(video_id, use_invidious, playlist_id=None, index=None):
|
||||||
tasks = (
|
for client in INNERTUBE_CLIENTS:
|
||||||
# Get video metadata from here
|
tasks = (
|
||||||
gevent.spawn(fetch_watch_page_info, video_id, playlist_id, index),
|
gevent.spawn(fetch_watch_page_info, video_id, playlist_id, index),
|
||||||
gevent.spawn(fetch_player_response, 'android', video_id)
|
gevent.spawn(fetch_player_response, client, video_id) # Use client from INNERTUBE_CLIENTS
|
||||||
)
|
)
|
||||||
gevent.joinall(tasks)
|
gevent.joinall(tasks)
|
||||||
util.check_gevent_exceptions(*tasks)
|
util.check_gevent_exceptions(*tasks)
|
||||||
info, player_response = tasks[0].value, tasks[1].value
|
info, player_response = tasks[0].value, tasks[1].value
|
||||||
|
|
||||||
yt_data_extract.update_with_new_urls(info, player_response)
|
|
||||||
|
|
||||||
# Age restricted video, retry
|
|
||||||
if info['age_restricted'] or info['player_urls_missing']:
|
|
||||||
if info['age_restricted']:
|
|
||||||
print('Age restricted video, retrying')
|
|
||||||
else:
|
|
||||||
print('Player urls missing, retrying')
|
|
||||||
player_response = fetch_player_response('tv_embedded', video_id)
|
|
||||||
yt_data_extract.update_with_new_urls(info, player_response)
|
yt_data_extract.update_with_new_urls(info, player_response)
|
||||||
|
|
||||||
# signature decryption
|
# Age restricted video, retry
|
||||||
decryption_error = decrypt_signatures(info, video_id)
|
if info['age_restricted'] or info['player_urls_missing']:
|
||||||
if decryption_error:
|
if info['age_restricted']:
|
||||||
decryption_error = 'Error decrypting url signatures: ' + decryption_error
|
print('Age restricted video, retrying')
|
||||||
info['playability_error'] = decryption_error
|
else:
|
||||||
|
print('Player urls missing, retrying')
|
||||||
|
player_response = fetch_player_response('tv_embedded', video_id)
|
||||||
|
yt_data_extract.update_with_new_urls(info, player_response)
|
||||||
|
|
||||||
# check if urls ready (non-live format) in former livestream
|
# signature decryption
|
||||||
# urls not ready if all of them have no filesize
|
decryption_error = decrypt_signatures(info, video_id)
|
||||||
if info['was_live']:
|
if decryption_error:
|
||||||
info['urls_ready'] = False
|
decryption_error = 'Error decrypting url signatures: ' + decryption_error
|
||||||
for fmt in info['formats']:
|
info['playability_error'] = decryption_error
|
||||||
if fmt['file_size'] is not None:
|
|
||||||
info['urls_ready'] = True
|
|
||||||
else:
|
|
||||||
info['urls_ready'] = True
|
|
||||||
|
|
||||||
# livestream urls
|
# check if urls ready (non-live format) in former livestream
|
||||||
# sometimes only the livestream urls work soon after the livestream is over
|
# urls not ready if all of them have no filesize
|
||||||
if (info['hls_manifest_url']
|
if info['was_live']:
|
||||||
and (info['live'] or not info['formats'] or not info['urls_ready'])
|
info['urls_ready'] = False
|
||||||
):
|
|
||||||
manifest = util.fetch_url(info['hls_manifest_url'],
|
|
||||||
debug_name='hls_manifest.m3u8',
|
|
||||||
report_text='Fetched hls manifest'
|
|
||||||
).decode('utf-8')
|
|
||||||
|
|
||||||
info['hls_formats'], err = yt_data_extract.extract_hls_formats(manifest)
|
|
||||||
if not err:
|
|
||||||
info['playability_error'] = None
|
|
||||||
for fmt in info['hls_formats']:
|
|
||||||
fmt['video_quality'] = video_quality_string(fmt)
|
|
||||||
else:
|
|
||||||
info['hls_formats'] = []
|
|
||||||
|
|
||||||
# check for 403. Unnecessary for tor video routing b/c ip address is same
|
|
||||||
info['invidious_used'] = False
|
|
||||||
info['invidious_reload_button'] = False
|
|
||||||
info['tor_bypass_used'] = False
|
|
||||||
if (settings.route_tor == 1
|
|
||||||
and info['formats'] and info['formats'][0]['url']):
|
|
||||||
try:
|
|
||||||
response = util.head(info['formats'][0]['url'],
|
|
||||||
report_text='Checked for URL access')
|
|
||||||
except urllib3.exceptions.HTTPError:
|
|
||||||
print('Error while checking for URL access:\n')
|
|
||||||
traceback.print_exc()
|
|
||||||
return info
|
|
||||||
|
|
||||||
if response.status == 403:
|
|
||||||
print('Access denied (403) for video urls.')
|
|
||||||
print('Routing video through Tor')
|
|
||||||
info['tor_bypass_used'] = True
|
|
||||||
for fmt in info['formats']:
|
for fmt in info['formats']:
|
||||||
fmt['url'] += '&use_tor=1'
|
if fmt['file_size'] is not None:
|
||||||
elif 300 <= response.status < 400:
|
info['urls_ready'] = True
|
||||||
print('Error: exceeded max redirects while checking video URL')
|
else:
|
||||||
return info
|
info['urls_ready'] = True
|
||||||
|
|
||||||
|
# livestream urls
|
||||||
|
# sometimes only the livestream urls work soon after the livestream is over
|
||||||
|
if (info['hls_manifest_url']
|
||||||
|
and (info['live'] or not info['formats'] or not info['urls_ready'])
|
||||||
|
):
|
||||||
|
manifest = util.fetch_url(info['hls_manifest_url'],
|
||||||
|
debug_name='hls_manifest.m3u8',
|
||||||
|
report_text='Fetched hls manifest'
|
||||||
|
).decode('utf-8')
|
||||||
|
|
||||||
|
info['hls_formats'], err = yt_data_extract.extract_hls_formats(manifest)
|
||||||
|
if not err:
|
||||||
|
info['playability_error'] = None
|
||||||
|
for fmt in info['hls_formats']:
|
||||||
|
fmt['video_quality'] = video_quality_string(fmt)
|
||||||
|
else:
|
||||||
|
info['hls_formats'] = []
|
||||||
|
|
||||||
|
# check for 403. Unnecessary for tor video routing b/c ip address is same
|
||||||
|
info['invidious_used'] = False
|
||||||
|
info['invidious_reload_button'] = False
|
||||||
|
info['tor_bypass_used'] = False
|
||||||
|
if (settings.route_tor == 1
|
||||||
|
and info['formats'] and info['formats'][0]['url']):
|
||||||
|
try:
|
||||||
|
response = util.head(info['formats'][0]['url'],
|
||||||
|
report_text='Checked for URL access')
|
||||||
|
except urllib3.exceptions.HTTPError:
|
||||||
|
print('Error while checking for URL access:\n')
|
||||||
|
traceback.print_exc()
|
||||||
|
return info
|
||||||
|
|
||||||
|
if response.status == 403:
|
||||||
|
print('Access denied (403) for video urls.')
|
||||||
|
print('Routing video through Tor')
|
||||||
|
info['tor_bypass_used'] = True
|
||||||
|
for fmt in info['formats']:
|
||||||
|
fmt['url'] += '&use_tor=1'
|
||||||
|
elif 300 <= response.status < 400:
|
||||||
|
print('Error: exceeded max redirects while checking video URL')
|
||||||
|
return info
|
||||||
|
|
||||||
|
|
||||||
def video_quality_string(format):
|
def video_quality_string(format):
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user