fix: support YouTube 2024+ data formats for playlists, podcasts and channels

- Add PODCAST content type support in lockupViewModel extraction
- Extract thumbnails and episode count from thumbnail overlay badges
- Migrate playlist page fetching from pbj=1 to innertube API (youtubei/v1/browse)
- Support new pageHeaderRenderer format in playlist metadata extraction
- Fix subscriber count extraction when YouTube returns handle instead of count
- Hide "None subscribers" in template when data is unavailable

youtube/playlist.py

@@ -30,42 +30,58 @@ def playlist_ctoken(playlist_id, offset, include_shorts=True):
 
 def playlist_first_page(playlist_id, report_text="Retrieved playlist",
                         use_mobile=False):
-    if use_mobile:
-        url = 'https://m.youtube.com/playlist?list=' + playlist_id + '&pbj=1'
-        content = util.fetch_url(
-            url, util.mobile_xhr_headers,
-            report_text=report_text, debug_name='playlist_first_page'
-        )
-        content = json.loads(content.decode('utf-8'))
-    else:
-        url = 'https://www.youtube.com/playlist?list=' + playlist_id + '&pbj=1'
-        content = util.fetch_url(
-            url, util.desktop_xhr_headers,
-            report_text=report_text, debug_name='playlist_first_page'
-        )
-        content = json.loads(content.decode('utf-8'))
+    # Use innertube API (pbj=1 no longer works for many playlists)
+    key = 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'
+    url = 'https://www.youtube.com/youtubei/v1/browse?key=' + key
 
-    return content
+    data = {
+        'context': {
+            'client': {
+                'hl': 'en',
+                'gl': 'US',
+                'clientName': 'WEB',
+                'clientVersion': '2.20240327.00.00',
+            },
+        },
+        'browseId': 'VL' + playlist_id,
+    }
+
+    content_type_header = (('Content-Type', 'application/json'),)
+    content = util.fetch_url(
+        url, util.desktop_xhr_headers + content_type_header,
+        data=json.dumps(data),
+        report_text=report_text, debug_name='playlist_first_page'
+    )
+    return json.loads(content.decode('utf-8'))
 
 
 def get_videos(playlist_id, page, include_shorts=True, use_mobile=False,
                report_text='Retrieved playlist'):
-    # mobile requests return 20 videos per page
-    if use_mobile:
-        page_size = 20
-        headers = util.mobile_xhr_headers
-    # desktop requests return 100 videos per page
-    else:
-        page_size = 100
-        headers = util.desktop_xhr_headers
+    page_size = 100
 
-    url = "https://m.youtube.com/playlist?ctoken="
-    url += playlist_ctoken(playlist_id, (int(page)-1)*page_size,
-                           include_shorts=include_shorts)
-    url += "&pbj=1"
+    key = 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'
+    url = 'https://www.youtube.com/youtubei/v1/browse?key=' + key
+
+    ctoken = playlist_ctoken(playlist_id, (int(page)-1)*page_size,
+                             include_shorts=include_shorts)
+
+    data = {
+        'context': {
+            'client': {
+                'hl': 'en',
+                'gl': 'US',
+                'clientName': 'WEB',
+                'clientVersion': '2.20240327.00.00',
+            },
+        },
+        'continuation': ctoken,
+    }
+
+    content_type_header = (('Content-Type', 'application/json'),)
     content = util.fetch_url(
-        url, headers, report_text=report_text,
-        debug_name='playlist_videos'
+        url, util.desktop_xhr_headers + content_type_header,
+        data=json.dumps(data),
+        report_text=report_text, debug_name='playlist_videos'
     )
 
     info = json.loads(content.decode('utf-8'))
@@ -96,7 +112,7 @@ def get_playlist_page():
         tasks = (
             gevent.spawn(
                 playlist_first_page, playlist_id,
-                report_text="Retrieved playlist info", use_mobile=True
+                report_text="Retrieved playlist info"
             ),
             gevent.spawn(get_videos, playlist_id, page)
         )