fix(channel): fix shorts/streams pagination using continuation tokens
- Add continuation_token_cache to store ctokens between page requests - Use cached ctoken for page 2+ instead of generating fresh tokens - Switch shorts/streams to Next/Previous buttons (no page numbers) - Show "N+ videos" indicator when more pages are available - Fix UnboundLocalError when page_call was undefined for shorts/streams The issue was that YouTube's InnerTube API requires continuation tokens for pagination on shorts/streams tabs, but the code was generating a new ctoken each time, always returning the same 30 videos.
This commit is contained in:
@@ -274,6 +274,8 @@ def get_channel_tab(channel_id, page="1", sort=3, tab='videos', view=1,
|
|||||||
|
|
||||||
# cache entries expire after 30 minutes
|
# cache entries expire after 30 minutes
|
||||||
number_of_videos_cache = cachetools.TTLCache(128, 30*60)
|
number_of_videos_cache = cachetools.TTLCache(128, 30*60)
|
||||||
|
# Cache for continuation tokens (shorts/streams pagination)
|
||||||
|
continuation_token_cache = cachetools.TTLCache(512, 15*60)
|
||||||
@cachetools.cached(number_of_videos_cache)
|
@cachetools.cached(number_of_videos_cache)
|
||||||
def get_number_of_videos_channel(channel_id):
|
def get_number_of_videos_channel(channel_id):
|
||||||
if channel_id is None:
|
if channel_id is None:
|
||||||
@@ -487,7 +489,43 @@ def get_channel_page_general_url(base_url, tab, request, channel_id=None):
|
|||||||
if not channel_id:
|
if not channel_id:
|
||||||
channel_id = get_channel_id(base_url)
|
channel_id = get_channel_id(base_url)
|
||||||
|
|
||||||
# Use youtubei browse API with continuation token for all pages
|
# For shorts/streams, use continuation token from cache or request
|
||||||
|
if tab in ('shorts', 'streams'):
|
||||||
|
if ctoken:
|
||||||
|
# Use ctoken directly from request (passed via pagination)
|
||||||
|
polymer_json = util.call_youtube_api('web', 'browse', {
|
||||||
|
'continuation': ctoken,
|
||||||
|
})
|
||||||
|
continuation = True
|
||||||
|
elif page_number > 1:
|
||||||
|
# For page 2+, get ctoken from cache
|
||||||
|
cache_key = (channel_id, tab, sort, page_number - 1)
|
||||||
|
cached_ctoken = continuation_token_cache.get(cache_key)
|
||||||
|
if cached_ctoken:
|
||||||
|
polymer_json = util.call_youtube_api('web', 'browse', {
|
||||||
|
'continuation': cached_ctoken,
|
||||||
|
})
|
||||||
|
continuation = True
|
||||||
|
else:
|
||||||
|
# Fallback: generate fresh ctoken
|
||||||
|
page_call = (get_channel_tab, channel_id, str(page_number), sort, tab, int(view))
|
||||||
|
continuation = True
|
||||||
|
polymer_json = gevent.spawn(*page_call)
|
||||||
|
polymer_json.join()
|
||||||
|
if polymer_json.exception:
|
||||||
|
raise polymer_json.exception
|
||||||
|
polymer_json = polymer_json.value
|
||||||
|
else:
|
||||||
|
# Page 1: generate fresh ctoken
|
||||||
|
page_call = (get_channel_tab, channel_id, str(page_number), sort, tab, int(view))
|
||||||
|
continuation = True
|
||||||
|
polymer_json = gevent.spawn(*page_call)
|
||||||
|
polymer_json.join()
|
||||||
|
if polymer_json.exception:
|
||||||
|
raise polymer_json.exception
|
||||||
|
polymer_json = polymer_json.value
|
||||||
|
else:
|
||||||
|
# videos tab - original logic
|
||||||
page_call = (get_channel_tab, channel_id, str(page_number), sort,
|
page_call = (get_channel_tab, channel_id, str(page_number), sort,
|
||||||
tab, int(view))
|
tab, int(view))
|
||||||
continuation = True
|
continuation = True
|
||||||
@@ -505,14 +543,7 @@ def get_channel_page_general_url(base_url, tab, request, channel_id=None):
|
|||||||
gevent.joinall(tasks)
|
gevent.joinall(tasks)
|
||||||
util.check_gevent_exceptions(*tasks)
|
util.check_gevent_exceptions(*tasks)
|
||||||
number_of_videos, polymer_json = tasks[0].value, tasks[1].value
|
number_of_videos, polymer_json = tasks[0].value, tasks[1].value
|
||||||
else:
|
# For shorts/streams, polymer_json is already set above, nothing to do here
|
||||||
# For shorts/streams, item count is used instead
|
|
||||||
polymer_json = gevent.spawn(*page_call)
|
|
||||||
polymer_json.join()
|
|
||||||
if polymer_json.exception:
|
|
||||||
raise polymer_json.exception
|
|
||||||
polymer_json = polymer_json.value
|
|
||||||
number_of_videos = 0 # will be replaced by actual item count later
|
|
||||||
|
|
||||||
elif tab == 'about':
|
elif tab == 'about':
|
||||||
# polymer_json = util.fetch_url(base_url + '/about?pbj=1', headers_desktop, debug_name='gen_channel_about')
|
# polymer_json = util.fetch_url(base_url + '/about?pbj=1', headers_desktop, debug_name='gen_channel_about')
|
||||||
@@ -580,9 +611,13 @@ def get_channel_page_general_url(base_url, tab, request, channel_id=None):
|
|||||||
|
|
||||||
if tab in ('videos', 'shorts', 'streams'):
|
if tab in ('videos', 'shorts', 'streams'):
|
||||||
if tab in ('shorts', 'streams'):
|
if tab in ('shorts', 'streams'):
|
||||||
# For shorts/streams, use the actual item count since
|
# For shorts/streams, use ctoken to determine pagination
|
||||||
# get_number_of_videos_channel counts regular uploads only
|
info['is_last_page'] = (info.get('ctoken') is None)
|
||||||
number_of_videos = len(info.get('items', []))
|
number_of_videos = len(info.get('items', []))
|
||||||
|
# Cache the ctoken for next page
|
||||||
|
if info.get('ctoken'):
|
||||||
|
cache_key = (channel_id, tab, sort, page_number)
|
||||||
|
continuation_token_cache[cache_key] = info['ctoken']
|
||||||
info['number_of_videos'] = number_of_videos
|
info['number_of_videos'] = number_of_videos
|
||||||
info['number_of_pages'] = math.ceil(number_of_videos/page_size) if number_of_videos else 1
|
info['number_of_pages'] = math.ceil(number_of_videos/page_size) if number_of_videos else 1
|
||||||
info['header_playlist_names'] = local_playlist.get_playlist_names()
|
info['header_playlist_names'] = local_playlist.get_playlist_names()
|
||||||
|
|||||||
@@ -82,7 +82,11 @@
|
|||||||
<div id="links-metadata">
|
<div id="links-metadata">
|
||||||
{% if current_tab in ('videos', 'shorts', 'streams') %}
|
{% if current_tab in ('videos', 'shorts', 'streams') %}
|
||||||
{% set sorts = [('3', 'newest'), ('4', 'newest - no shorts')] %}
|
{% set sorts = [('3', 'newest'), ('4', 'newest - no shorts')] %}
|
||||||
|
{% if current_tab in ('shorts', 'streams') and not is_last_page %}
|
||||||
|
<div id="number-of-results">{{ number_of_videos }}+ videos</div>
|
||||||
|
{% else %}
|
||||||
<div id="number-of-results">{{ number_of_videos }} videos</div>
|
<div id="number-of-results">{{ number_of_videos }} videos</div>
|
||||||
|
{% endif %}
|
||||||
{% elif current_tab == 'playlists' %}
|
{% elif current_tab == 'playlists' %}
|
||||||
{% set sorts = [('3', 'newest'), ('4', 'last video added')] %}
|
{% set sorts = [('3', 'newest'), ('4', 'last video added')] %}
|
||||||
{% if items %}
|
{% if items %}
|
||||||
@@ -117,7 +121,11 @@
|
|||||||
<hr/>
|
<hr/>
|
||||||
|
|
||||||
<footer class="pagination-container">
|
<footer class="pagination-container">
|
||||||
{% if current_tab in ('videos', 'shorts', 'streams') %}
|
{% if current_tab in ('shorts', 'streams') %}
|
||||||
|
<nav class="next-previous-button-row">
|
||||||
|
{{ common_elements.next_previous_buttons(is_last_page, channel_url + '/' + current_tab, parameters_dictionary) }}
|
||||||
|
</nav>
|
||||||
|
{% elif current_tab == 'videos' %}
|
||||||
<nav class="pagination-list">
|
<nav class="pagination-list">
|
||||||
{{ common_elements.page_buttons(number_of_pages, channel_url + '/' + current_tab, parameters_dictionary, include_ends=(current_sort.__str__() in '34')) }}
|
{{ common_elements.page_buttons(number_of_pages, channel_url + '/' + current_tab, parameters_dictionary, include_ends=(current_sort.__str__() in '34')) }}
|
||||||
</nav>
|
</nav>
|
||||||
|
|||||||
Reference in New Issue
Block a user