Revert "Refactoring code and reuse INNERTUBE_CLIENTS"

This reverts commit 8af98968dd4325d5686bfed109aa4ed18b17edbc.

youtube/util.py

@@ -431,29 +431,34 @@ class RateLimitedQueue(gevent.queue.Queue):
         gevent.queue.Queue.__init__(self)
 
     def get(self):
-        with self.lock:     # blocks if another greenlet currently has the lock
+        self.lock.acquire()     # blocks if another greenlet currently has the lock
-            if ((self.count_since_last_wait >= self.subsequent_bursts and self.surpassed_initial) or
+        if self.count_since_last_wait >= self.subsequent_bursts and self.surpassed_initial:
-                (self.count_since_last_wait >= self.initial_burst and not self.surpassed_initial)):
+            gevent.sleep(self.waiting_period)
-                self.surpassed_initial = True
+            self.count_since_last_wait = 0
-                gevent.sleep(self.waiting_period)
+
+        elif self.count_since_last_wait >= self.initial_burst and not self.surpassed_initial:
+            self.surpassed_initial = True
+            gevent.sleep(self.waiting_period)
+            self.count_since_last_wait = 0
+
+        self.count_since_last_wait += 1
+
+        if not self.currently_empty and self.empty():
+            self.currently_empty = True
+            self.empty_start = time.monotonic()
+
+        item = gevent.queue.Queue.get(self)     # blocks when nothing left
+
+        if self.currently_empty:
+            if time.monotonic() - self.empty_start >= self.waiting_period:
                 self.count_since_last_wait = 0
+                self.surpassed_initial = False
 
-            self.count_since_last_wait += 1
+            self.currently_empty = False
 
-            if not self.currently_empty and self.empty():
+        self.lock.release()
-                self.currently_empty = True
-                self.empty_start = time.monotonic()
 
-            item = gevent.queue.Queue.get(self)     # blocks when nothing left
+        return item
-
-            if self.currently_empty:
-                if time.monotonic() - self.empty_start >= self.waiting_period:
-                    self.count_since_last_wait = 0
-                    self.surpassed_initial = False
-
-                self.currently_empty = False
-
-            return item
 
 
 def download_thumbnail(save_directory, video_id):
@@ -662,45 +667,16 @@ def to_valid_filename(name):
 
 # https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/extractor/youtube.py#L72
 INNERTUBE_CLIENTS = {
-    'android-test-suite': {
+    'web_creator': {
-        'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
+        'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
         'INNERTUBE_CONTEXT': {
             'client': {
-                'hl': 'en',
+                'clientName': 'WEB_CREATOR',
-                'gl': 'US',
+                'clientVersion': '1.20240723.03.00',
-                'clientName': 'ANDROID_TESTSUITE',
-                'clientVersion': '1.9',
-                'osName': 'Android',
-                'osVersion': '12',
-                'androidSdkVersion': 31,
-                'platform': 'MOBILE',
-                'userAgent': 'com.google.android.youtube/1.9 (Linux; U; Android 12; US) gzip'
             },
-            # https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-887739287
-            #'thirdParty': {
-            #    'embedUrl': 'https://google.com',  # Can be any valid URL
-            #}
         },
-        'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
-        'REQUIRE_JS_PLAYER': False,
     },
-
-    'ios': {
-        'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
-        'INNERTUBE_CONTEXT': {
-            'client': {
-                'hl': 'en',
-                'gl': 'US',
-                'clientName': 'IOS',
-                'clientVersion': '19.12.3',
-                'deviceModel': 'iPhone14,3',
-                'userAgent': 'com.google.ios.youtube/19.12.3 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
-            }
-        },
-        'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
-        'REQUIRE_JS_PLAYER': False
-    },
-
     'android': {
         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
         'INNERTUBE_CONTEXT': {
@@ -720,22 +696,22 @@ INNERTUBE_CLIENTS = {
         'REQUIRE_JS_PLAYER': False,
     },
 
-    'android_music': {
+    'ios': {
-        'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
+        'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
         'INNERTUBE_CONTEXT': {
             'client': {
                 'hl': 'en',
                 'gl': 'US',
-                'clientName': 'ANDROID_MUSIC',
+                'clientName': 'IOS',
-                'clientVersion': '6.48.51',
+                'clientVersion': '19.29.1',
-                'osName': 'Android',
+                'deviceMake': 'Apple',
-                'osVersion': '14',
+                'deviceModel': 'iPhone16,2',
-                'androidSdkVersion': 34,
+                'userAgent': 'com.google.ios.youtube/19.29.1 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X;)',
-                'platform': 'MOBILE',
+                'osName': 'iPhone',
-                'userAgent': 'com.google.android.apps.youtube.music/6.48.51 (Linux; U; Android 14; US) gzip'
+                'osVersion': '17.5.1.21F90',
             }
         },
-        'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
         'REQUIRE_JS_PLAYER': False
     },
 

youtube/watch.py

@@ -2,7 +2,6 @@ import youtube
 from youtube import yt_app
 from youtube import util, comments, local_playlist, yt_data_extract
 from youtube.util import time_utc_isoformat
-from youtube.util import INNERTUBE_CLIENTS
 import settings
 
 from flask import request
@@ -370,83 +369,83 @@ def fetch_watch_page_info(video_id, playlist_id, index):
     return yt_data_extract.extract_watch_info_from_html(watch_page)
 
 def extract_info(video_id, use_invidious, playlist_id=None, index=None):
-    for client in INNERTUBE_CLIENTS:
+    tasks = (
-        tasks = (
+        # Get video metadata from here
-            gevent.spawn(fetch_watch_page_info, video_id, playlist_id, index),
+        gevent.spawn(fetch_watch_page_info, video_id, playlist_id, index),
-            gevent.spawn(fetch_player_response, client, video_id)  # Use client from INNERTUBE_CLIENTS
+        gevent.spawn(fetch_player_response, 'ios', video_id)
-        )
+    )
-        gevent.joinall(tasks)
+    gevent.joinall(tasks)
-        util.check_gevent_exceptions(*tasks)
+    util.check_gevent_exceptions(*tasks)
-        info, player_response = tasks[0].value, tasks[1].value
+    info, player_response = tasks[0].value, tasks[1].value
 
+    yt_data_extract.update_with_new_urls(info, player_response)
+
+    # Age restricted video, retry
+    if info['age_restricted'] or info['player_urls_missing']:
+        if info['age_restricted']:
+            print('Age restricted video, retrying')
+        else:
+            print('Player urls missing, retrying')
+        player_response = fetch_player_response('tv_embedded', video_id)
         yt_data_extract.update_with_new_urls(info, player_response)
 
-        # Age restricted video, retry
+    # signature decryption
-        if info['age_restricted'] or info['player_urls_missing']:
+    decryption_error = decrypt_signatures(info, video_id)
-            if info['age_restricted']:
+    if decryption_error:
-                print('Age restricted video, retrying')
+        decryption_error = 'Error decrypting url signatures: ' + decryption_error
-            else:
+        info['playability_error'] = decryption_error
-                print('Player urls missing, retrying')
-            player_response = fetch_player_response('tv_embedded', video_id)
-            yt_data_extract.update_with_new_urls(info, player_response)
 
-        # signature decryption
+    # check if urls ready (non-live format) in former livestream
-        decryption_error = decrypt_signatures(info, video_id)
+    # urls not ready if all of them have no filesize
-        if decryption_error:
+    if info['was_live']:
-            decryption_error = 'Error decrypting url signatures: ' + decryption_error
+        info['urls_ready'] = False
-            info['playability_error'] = decryption_error
+        for fmt in info['formats']:
+            if fmt['file_size'] is not None:
+                info['urls_ready'] = True
+    else:
+        info['urls_ready'] = True
 
-        # check if urls ready (non-live format) in former livestream
+    # livestream urls
-        # urls not ready if all of them have no filesize
+    # sometimes only the livestream urls work soon after the livestream is over
-        if info['was_live']:
+    if (info['hls_manifest_url']
-            info['urls_ready'] = False
+        and (info['live'] or not info['formats'] or not info['urls_ready'])
+    ):
+        manifest = util.fetch_url(info['hls_manifest_url'],
+            debug_name='hls_manifest.m3u8',
+            report_text='Fetched hls manifest'
+        ).decode('utf-8')
+
+        info['hls_formats'], err = yt_data_extract.extract_hls_formats(manifest)
+        if not err:
+            info['playability_error'] = None
+        for fmt in info['hls_formats']:
+            fmt['video_quality'] = video_quality_string(fmt)
+    else:
+        info['hls_formats'] = []
+
+    # check for 403. Unnecessary for tor video routing b/c ip address is same
+    info['invidious_used'] = False
+    info['invidious_reload_button'] = False
+    info['tor_bypass_used'] = False
+    if (settings.route_tor == 1
+            and info['formats'] and info['formats'][0]['url']):
+        try:
+            response = util.head(info['formats'][0]['url'],
+                                 report_text='Checked for URL access')
+        except urllib3.exceptions.HTTPError:
+            print('Error while checking for URL access:\n')
+            traceback.print_exc()
+            return info
+
+        if response.status == 403:
+            print('Access denied (403) for video urls.')
+            print('Routing video through Tor')
+            info['tor_bypass_used'] = True
             for fmt in info['formats']:
-                if fmt['file_size'] is not None:
+                fmt['url'] += '&use_tor=1'
-                    info['urls_ready'] = True
+        elif 300 <= response.status < 400:
-        else:
+            print('Error: exceeded max redirects while checking video URL')
-            info['urls_ready'] = True
+    return info
-
-        # livestream urls
-        # sometimes only the livestream urls work soon after the livestream is over
-        if (info['hls_manifest_url']
-            and (info['live'] or not info['formats'] or not info['urls_ready'])
-        ):
-            manifest = util.fetch_url(info['hls_manifest_url'],
-                debug_name='hls_manifest.m3u8',
-                report_text='Fetched hls manifest'
-            ).decode('utf-8')
-
-            info['hls_formats'], err = yt_data_extract.extract_hls_formats(manifest)
-            if not err:
-                info['playability_error'] = None
-            for fmt in info['hls_formats']:
-                fmt['video_quality'] = video_quality_string(fmt)
-        else:
-            info['hls_formats'] = []
-
-        # check for 403. Unnecessary for tor video routing b/c ip address is same
-        info['invidious_used'] = False
-        info['invidious_reload_button'] = False
-        info['tor_bypass_used'] = False
-        if (settings.route_tor == 1
-                and info['formats'] and info['formats'][0]['url']):
-            try:
-                response = util.head(info['formats'][0]['url'],
-                                     report_text='Checked for URL access')
-            except urllib3.exceptions.HTTPError:
-                print('Error while checking for URL access:\n')
-                traceback.print_exc()
-                return info
-
-            if response.status == 403:
-                print('Access denied (403) for video urls.')
-                print('Routing video through Tor')
-                info['tor_bypass_used'] = True
-                for fmt in info['formats']:
-                    fmt['url'] += '&use_tor=1'
-            elif 300 <= response.status < 400:
-                print('Error: exceeded max redirects while checking video URL')
-        return info
 
 
 def video_quality_string(format):