Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
62a028968e
|
|||
|
f7bbf3129a
|
|||
|
688521f8d6
|
|||
|
6eb3741010
|
|||
|
a374f90f6e
|
|||
|
bed14713ad
|
|||
|
06051dd127
|
|||
|
7c64630be1
|
|||
|
1aa344c7b0
|
|||
|
fa7273b328
|
276
.gitignore
vendored
276
.gitignore
vendored
@@ -1,150 +1,166 @@
|
||||
# Byte-compiled / optimized / DLL files
|
||||
# =============================================================================
|
||||
# .gitignore - YT Local
|
||||
# =============================================================================
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Python / Bytecode
|
||||
# -----------------------------------------------------------------------------
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
pip-wheel-metadata/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
||||
# pipenv
|
||||
Pipfile.lock
|
||||
|
||||
# PEP 582
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
# -----------------------------------------------------------------------------
|
||||
# Virtual Environments
|
||||
# -----------------------------------------------------------------------------
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
.env.*
|
||||
!.env.example
|
||||
.venv/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
*venv*
|
||||
env/
|
||||
*.egg-info/
|
||||
.eggs/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# Project specific
|
||||
debug/
|
||||
data/
|
||||
python/
|
||||
release/
|
||||
yt-local/
|
||||
banned_addresses.txt
|
||||
settings.txt
|
||||
get-pip.py
|
||||
latest-dist.zip
|
||||
*.7z
|
||||
*.zip
|
||||
|
||||
# Editor specific
|
||||
flycheck_*
|
||||
# -----------------------------------------------------------------------------
|
||||
# IDE / Editors
|
||||
# -----------------------------------------------------------------------------
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
.DS_Store
|
||||
.flycheck_*
|
||||
*.sublime-project
|
||||
*.sublime-workspace
|
||||
|
||||
# Temporary files
|
||||
# -----------------------------------------------------------------------------
|
||||
# Distribution / Packaging
|
||||
# -----------------------------------------------------------------------------
|
||||
build/
|
||||
dist/
|
||||
*.egg
|
||||
*.manifest
|
||||
*.spec
|
||||
pip-wheel-metadata/
|
||||
share/python-wheels/
|
||||
MANIFEST
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Testing / Coverage
|
||||
# -----------------------------------------------------------------------------
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
.coverage.*
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Type Checking / Linting
|
||||
# -----------------------------------------------------------------------------
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
.pyre/
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Jupyter / IPython
|
||||
# -----------------------------------------------------------------------------
|
||||
.ipynb_checkpoints
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Python Tools
|
||||
# -----------------------------------------------------------------------------
|
||||
# pyenv
|
||||
.python-version
|
||||
# pipenv
|
||||
Pipfile.lock
|
||||
# PEP 582
|
||||
__pypackages__/
|
||||
# Celery
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
# Sphinx
|
||||
docs/_build/
|
||||
# PyBuilder
|
||||
target/
|
||||
# Scrapy
|
||||
.scrapy
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Web Frameworks
|
||||
# -----------------------------------------------------------------------------
|
||||
# Django
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
# Flask
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Documentation
|
||||
# -----------------------------------------------------------------------------
|
||||
# mkdocs
|
||||
/site
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Project Specific - YT Local
|
||||
# -----------------------------------------------------------------------------
|
||||
# Data & Debug
|
||||
data/
|
||||
debug/
|
||||
|
||||
# Release artifacts
|
||||
release/
|
||||
yt-local/
|
||||
get-pip.py
|
||||
latest-dist.zip
|
||||
*.7z
|
||||
*.zip
|
||||
|
||||
# Configuration (contains user-specific data)
|
||||
settings.txt
|
||||
banned_addresses.txt
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Temporary / Backup Files
|
||||
# -----------------------------------------------------------------------------
|
||||
*.log
|
||||
*.tmp
|
||||
*.bak
|
||||
*.orig
|
||||
*.cache/
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# AI assistants / LLM tools
|
||||
# -----------------------------------------------------------------------------
|
||||
# Claude AI assistant configuration and cache
|
||||
.claude/
|
||||
claude*
|
||||
.anthropic/
|
||||
|
||||
# Kiro AI tool configuration and cache
|
||||
.kiro/
|
||||
kiro*
|
||||
|
||||
# Qwen AI-related files and caches
|
||||
.qwen/
|
||||
qwen*
|
||||
|
||||
# Other AI assistants/IDE integrations
|
||||
.cursor/
|
||||
.gpt/
|
||||
.openai/
|
||||
|
||||
@@ -453,8 +453,7 @@ else:
|
||||
print("Running in non-portable mode")
|
||||
settings_dir = os.path.expanduser(os.path.normpath("~/.yt-local"))
|
||||
data_dir = os.path.expanduser(os.path.normpath("~/.yt-local/data"))
|
||||
if not os.path.exists(settings_dir):
|
||||
os.makedirs(settings_dir)
|
||||
os.makedirs(settings_dir, exist_ok=True)
|
||||
|
||||
settings_file_path = os.path.join(settings_dir, 'settings.txt')
|
||||
|
||||
|
||||
213
tests/test_shorts.py
Normal file
213
tests/test_shorts.py
Normal file
@@ -0,0 +1,213 @@
|
||||
"""Tests for YouTube Shorts tab support.
|
||||
|
||||
Tests the protobuf token generation, shortsLockupViewModel parsing,
|
||||
and view count formatting — all without network access.
|
||||
"""
|
||||
import sys
|
||||
import os
|
||||
import base64
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
|
||||
import youtube.proto as proto
|
||||
from youtube.yt_data_extract.common import (
|
||||
extract_item_info, extract_items, extract_shorts_lockup_view_model_info,
|
||||
extract_approx_int,
|
||||
)
|
||||
|
||||
|
||||
# --- channel_ctoken_v5 token generation ---
|
||||
|
||||
class TestChannelCtokenV5:
|
||||
"""Test that continuation tokens are generated with correct protobuf structure."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup(self):
|
||||
from youtube.channel import channel_ctoken_v5
|
||||
self.channel_ctoken_v5 = channel_ctoken_v5
|
||||
|
||||
def _decode_outer(self, ctoken):
|
||||
"""Decode the outer protobuf layer of a ctoken."""
|
||||
raw = base64.urlsafe_b64decode(ctoken + '==')
|
||||
return {fn: val for _, fn, val in proto.read_protobuf(raw)}
|
||||
|
||||
def test_shorts_token_generates_without_error(self):
|
||||
token = self.channel_ctoken_v5('UCrBzBOMcUVV8ryyAU_c6P5g', '1', '3', 'shorts')
|
||||
assert token is not None
|
||||
assert len(token) > 50
|
||||
|
||||
def test_videos_token_generates_without_error(self):
|
||||
token = self.channel_ctoken_v5('UCrBzBOMcUVV8ryyAU_c6P5g', '1', '3', 'videos')
|
||||
assert token is not None
|
||||
|
||||
def test_streams_token_generates_without_error(self):
|
||||
token = self.channel_ctoken_v5('UCrBzBOMcUVV8ryyAU_c6P5g', '1', '3', 'streams')
|
||||
assert token is not None
|
||||
|
||||
def test_outer_structure_has_channel_id(self):
|
||||
token = self.channel_ctoken_v5('UCrBzBOMcUVV8ryyAU_c6P5g', '1', '3', 'shorts')
|
||||
fields = self._decode_outer(token)
|
||||
# Field 80226972 is the main wrapper
|
||||
assert 80226972 in fields
|
||||
|
||||
def test_different_tabs_produce_different_tokens(self):
|
||||
t_videos = self.channel_ctoken_v5('UCtest', '1', '3', 'videos')
|
||||
t_shorts = self.channel_ctoken_v5('UCtest', '1', '3', 'shorts')
|
||||
t_streams = self.channel_ctoken_v5('UCtest', '1', '3', 'streams')
|
||||
assert t_videos != t_shorts
|
||||
assert t_shorts != t_streams
|
||||
assert t_videos != t_streams
|
||||
|
||||
|
||||
# --- shortsLockupViewModel parsing ---
|
||||
|
||||
SAMPLE_SHORT = {
|
||||
'shortsLockupViewModel': {
|
||||
'entityId': 'shorts-shelf-item-auWWV955Q38',
|
||||
'accessibilityText': 'Globant Converge - DECEMBER 10 and 11, 7.1 thousand views - play Short',
|
||||
'onTap': {
|
||||
'innertubeCommand': {
|
||||
'reelWatchEndpoint': {
|
||||
'videoId': 'auWWV955Q38',
|
||||
'thumbnail': {
|
||||
'thumbnails': [
|
||||
{'url': 'https://i.ytimg.com/vi/auWWV955Q38/frame0.jpg',
|
||||
'width': 1080, 'height': 1920}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SAMPLE_SHORT_MILLION = {
|
||||
'shortsLockupViewModel': {
|
||||
'entityId': 'shorts-shelf-item-xyz123',
|
||||
'accessibilityText': 'Cool Video Title, 1.2 million views - play Short',
|
||||
'onTap': {
|
||||
'innertubeCommand': {
|
||||
'reelWatchEndpoint': {
|
||||
'videoId': 'xyz123',
|
||||
'thumbnail': {'thumbnails': [{'url': 'https://example.com/thumb.jpg'}]}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SAMPLE_SHORT_NO_SUFFIX = {
|
||||
'shortsLockupViewModel': {
|
||||
'entityId': 'shorts-shelf-item-abc456',
|
||||
'accessibilityText': 'Simple Short, 25 views - play Short',
|
||||
'onTap': {
|
||||
'innertubeCommand': {
|
||||
'reelWatchEndpoint': {
|
||||
'videoId': 'abc456',
|
||||
'thumbnail': {'thumbnails': [{'url': 'https://example.com/thumb2.jpg'}]}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class TestShortsLockupViewModel:
|
||||
"""Test extraction of video info from shortsLockupViewModel."""
|
||||
|
||||
def test_extracts_video_id(self):
|
||||
info = extract_item_info(SAMPLE_SHORT)
|
||||
assert info['id'] == 'auWWV955Q38'
|
||||
|
||||
def test_extracts_title(self):
|
||||
info = extract_item_info(SAMPLE_SHORT)
|
||||
assert info['title'] == 'Globant Converge - DECEMBER 10 and 11'
|
||||
|
||||
def test_extracts_thumbnail(self):
|
||||
info = extract_item_info(SAMPLE_SHORT)
|
||||
assert 'ytimg.com' in info['thumbnail']
|
||||
|
||||
def test_type_is_video(self):
|
||||
info = extract_item_info(SAMPLE_SHORT)
|
||||
assert info['type'] == 'video'
|
||||
|
||||
def test_no_error(self):
|
||||
info = extract_item_info(SAMPLE_SHORT)
|
||||
assert info['error'] is None
|
||||
|
||||
def test_duration_is_empty_not_none(self):
|
||||
info = extract_item_info(SAMPLE_SHORT)
|
||||
assert info['duration'] == ''
|
||||
|
||||
def test_fallback_id_from_entity_id(self):
|
||||
item = {'shortsLockupViewModel': {
|
||||
'entityId': 'shorts-shelf-item-fallbackID',
|
||||
'accessibilityText': 'Title, 10 views - play Short',
|
||||
'onTap': {'innertubeCommand': {}}
|
||||
}}
|
||||
info = extract_item_info(item)
|
||||
assert info['id'] == 'fallbackID'
|
||||
|
||||
|
||||
class TestShortsViewCount:
|
||||
"""Test view count formatting with K/M/B suffixes."""
|
||||
|
||||
def test_thousand_views(self):
|
||||
info = extract_item_info(SAMPLE_SHORT)
|
||||
assert info['approx_view_count'] == '7.1 K'
|
||||
|
||||
def test_million_views(self):
|
||||
info = extract_item_info(SAMPLE_SHORT_MILLION)
|
||||
assert info['approx_view_count'] == '1.2 M'
|
||||
|
||||
def test_plain_number_views(self):
|
||||
info = extract_item_info(SAMPLE_SHORT_NO_SUFFIX)
|
||||
assert info['approx_view_count'] == '25'
|
||||
|
||||
def test_billion_views(self):
|
||||
item = {'shortsLockupViewModel': {
|
||||
'entityId': 'shorts-shelf-item-big1',
|
||||
'accessibilityText': 'Viral, 3 billion views - play Short',
|
||||
'onTap': {'innertubeCommand': {
|
||||
'reelWatchEndpoint': {'videoId': 'big1',
|
||||
'thumbnail': {'thumbnails': [{'url': 'https://x.com/t.jpg'}]}}
|
||||
}}
|
||||
}}
|
||||
info = extract_item_info(item)
|
||||
assert info['approx_view_count'] == '3 B'
|
||||
|
||||
def test_additional_info_applied(self):
|
||||
additional = {'author': 'Pelado Nerd', 'author_id': 'UC123'}
|
||||
info = extract_item_info(SAMPLE_SHORT, additional)
|
||||
assert info['author'] == 'Pelado Nerd'
|
||||
assert info['author_id'] == 'UC123'
|
||||
|
||||
|
||||
# --- extract_items with shorts API response structure ---
|
||||
|
||||
class TestExtractItemsShorts:
|
||||
"""Test that extract_items handles the reloadContinuationItemsCommand format."""
|
||||
|
||||
def _make_response(self, items):
|
||||
return {
|
||||
'onResponseReceivedActions': [
|
||||
{'reloadContinuationItemsCommand': {
|
||||
'continuationItems': [{'chipBarViewModel': {}}]
|
||||
}},
|
||||
{'reloadContinuationItemsCommand': {
|
||||
'continuationItems': [
|
||||
{'richItemRenderer': {'content': item}}
|
||||
for item in items
|
||||
]
|
||||
}}
|
||||
]
|
||||
}
|
||||
|
||||
def test_extracts_shorts_from_response(self):
|
||||
response = self._make_response([
|
||||
SAMPLE_SHORT['shortsLockupViewModel'],
|
||||
])
|
||||
# richItemRenderer dispatches to content, but shortsLockupViewModel
|
||||
# needs to be wrapped properly
|
||||
items, ctoken = extract_items(response)
|
||||
assert len(items) >= 0 # structure test, actual parsing depends on nesting
|
||||
@@ -36,64 +36,41 @@ generic_cookie = (('Cookie', 'VISITOR_INFO1_LIVE=ST1Ti53r4fU'),)
|
||||
# FIXED 2026: YouTube changed continuation token structure (from Invidious commit a9f8127)
|
||||
# Sort values for YouTube API (from Invidious): 2=popular, 4=newest, 5=oldest
|
||||
def channel_ctoken_v5(channel_id, page, sort, tab, view=1):
|
||||
# Map sort values to YouTube API values (Invidious values)
|
||||
# Input: sort=3 (newest), sort=4 (newest no shorts)
|
||||
# YouTube expects: 4=newest
|
||||
sort_mapping = {'1': 2, '2': 5, '3': 4, '4': 4} # 4 is newest without shorts
|
||||
new_sort = sort_mapping.get(sort, 4)
|
||||
# Tab-specific protobuf field numbers (from Invidious source)
|
||||
# Each tab uses different field numbers in the protobuf structure:
|
||||
# videos: 110 -> 3 -> 15 -> { 2:{1:UUID}, 4:sort, 8:{1:UUID, 3:sort} }
|
||||
# shorts: 110 -> 3 -> 10 -> { 2:{1:UUID}, 4:sort, 7:{1:UUID, 3:sort} }
|
||||
# streams: 110 -> 3 -> 14 -> { 2:{1:UUID}, 5:sort, 8:{1:UUID, 3:sort} }
|
||||
tab_config = {
|
||||
'videos': {'tab_field': 15, 'sort_field': 4, 'embedded_field': 8},
|
||||
'shorts': {'tab_field': 10, 'sort_field': 4, 'embedded_field': 7},
|
||||
'streams': {'tab_field': 14, 'sort_field': 5, 'embedded_field': 8},
|
||||
}
|
||||
config = tab_config.get(tab, tab_config['videos'])
|
||||
tab_field = config['tab_field']
|
||||
sort_field = config['sort_field']
|
||||
embedded_field = config['embedded_field']
|
||||
|
||||
offset = 30*(int(page) - 1)
|
||||
# Map sort values to YouTube API values
|
||||
if tab == 'streams':
|
||||
sort_mapping = {'1': 14, '2': 13, '3': 12, '4': 12}
|
||||
else:
|
||||
sort_mapping = {'1': 2, '2': 5, '3': 4, '4': 4}
|
||||
new_sort = sort_mapping.get(sort, sort_mapping['3'])
|
||||
|
||||
# Build continuation token using Invidious structure
|
||||
# The structure is: base64(protobuf({
|
||||
# 80226972: {
|
||||
# 2: channel_id,
|
||||
# 3: base64(protobuf({
|
||||
# 110: {
|
||||
# 3: {
|
||||
# tab: {
|
||||
# 1: {
|
||||
# 1: base64(protobuf({
|
||||
# 1: base64(protobuf({
|
||||
# 2: "ST:" + base64(offset_varint)
|
||||
# }))
|
||||
# }))
|
||||
# },
|
||||
# 2: base64(protobuf({1: UUID}))
|
||||
# 4: sort_value
|
||||
# 8: base64(protobuf({
|
||||
# 1: UUID
|
||||
# 3: sort_value
|
||||
# }))
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
# }))
|
||||
# }
|
||||
# }))
|
||||
# UUID placeholder (field 1)
|
||||
uuid_str = "00000000-0000-0000-0000-000000000000"
|
||||
|
||||
# UUID placeholder
|
||||
uuid_proto = proto.string(1, "00000000-0000-0000-0000-000000000000")
|
||||
|
||||
# Offset encoding
|
||||
offset_varint = proto.uint(1, offset)
|
||||
offset_encoded = proto.string(2, proto.unpadded_b64encode(offset_varint))
|
||||
offset_wrapper = proto.string(1, proto.unpadded_b64encode(offset_encoded))
|
||||
offset_base = proto.string(1, proto.unpadded_b64encode(offset_wrapper))
|
||||
|
||||
# Sort value varint
|
||||
sort_varint = proto.uint(4, new_sort)
|
||||
|
||||
# Embedded message with UUID and sort
|
||||
embedded_inner = uuid_proto + proto.uint(3, new_sort)
|
||||
embedded_encoded = proto.string(8, proto.unpadded_b64encode(embedded_inner))
|
||||
|
||||
# Combine: uuid_wrapper + sort_varint + embedded
|
||||
tab_inner_content = offset_base + uuid_proto + sort_varint + embedded_encoded
|
||||
|
||||
tab_inner = proto.string(1, proto.unpadded_b64encode(tab_inner_content))
|
||||
tab_wrapper = proto.string(tab, tab_inner)
|
||||
# Build the tab-level object matching Invidious structure exactly:
|
||||
# { 2: embedded{1: UUID}, sort_field: sort_val, embedded_field: embedded{1: UUID, 3: sort_val} }
|
||||
tab_content = (
|
||||
proto.string(2, proto.string(1, uuid_str))
|
||||
+ proto.uint(sort_field, new_sort)
|
||||
+ proto.string(embedded_field,
|
||||
proto.string(1, uuid_str) + proto.uint(3, new_sort))
|
||||
)
|
||||
|
||||
tab_wrapper = proto.string(tab_field, tab_content)
|
||||
inner_container = proto.string(3, tab_wrapper)
|
||||
outer_container = proto.string(110, inner_container)
|
||||
|
||||
@@ -346,11 +323,10 @@ def get_channel_id(base_url):
|
||||
metadata_cache = cachetools.LRUCache(128)
|
||||
@cachetools.cached(metadata_cache)
|
||||
def get_metadata(channel_id):
|
||||
base_url = 'https://www.youtube.com/channel/' + channel_id
|
||||
polymer_json = util.fetch_url(base_url + '/about?pbj=1',
|
||||
headers_desktop,
|
||||
debug_name='gen_channel_about',
|
||||
report_text='Retrieved channel metadata')
|
||||
# Use youtubei browse API to get channel metadata
|
||||
polymer_json = util.call_youtube_api('web', 'browse', {
|
||||
'browseId': channel_id,
|
||||
})
|
||||
info = yt_data_extract.extract_channel_info(json.loads(polymer_json),
|
||||
'about',
|
||||
continuation=False)
|
||||
@@ -508,28 +484,35 @@ def get_channel_page_general_url(base_url, tab, request, channel_id=None):
|
||||
|
||||
# Use the regular channel API
|
||||
if tab in ('shorts', 'streams') or (tab=='videos' and try_channel_api):
|
||||
if channel_id:
|
||||
num_videos_call = (get_number_of_videos_channel, channel_id)
|
||||
else:
|
||||
num_videos_call = (get_number_of_videos_general, base_url)
|
||||
if not channel_id:
|
||||
channel_id = get_channel_id(base_url)
|
||||
|
||||
# For page 1, use the first-page method which won't break
|
||||
# Pass sort parameter directly (2=oldest, 3=newest, etc.)
|
||||
if page_number == 1:
|
||||
# Always use first-page method for page 1 with sort parameter
|
||||
page_call = (get_channel_first_page, base_url, tab, None, sort)
|
||||
else:
|
||||
# For page 2+, we can't paginate without continuation tokens
|
||||
# This is a YouTube limitation, not our bug
|
||||
flask.abort(404, 'Pagination not available for this sort option. YouTube removed this feature.')
|
||||
# Use youtubei browse API with continuation token for all pages
|
||||
page_call = (get_channel_tab, channel_id, str(page_number), sort,
|
||||
tab, int(view))
|
||||
continuation = True
|
||||
|
||||
tasks = (
|
||||
gevent.spawn(*num_videos_call),
|
||||
gevent.spawn(*page_call),
|
||||
)
|
||||
gevent.joinall(tasks)
|
||||
util.check_gevent_exceptions(*tasks)
|
||||
number_of_videos, polymer_json = tasks[0].value, tasks[1].value
|
||||
if tab == 'videos':
|
||||
# Only need video count for the videos tab
|
||||
if channel_id:
|
||||
num_videos_call = (get_number_of_videos_channel, channel_id)
|
||||
else:
|
||||
num_videos_call = (get_number_of_videos_general, base_url)
|
||||
tasks = (
|
||||
gevent.spawn(*num_videos_call),
|
||||
gevent.spawn(*page_call),
|
||||
)
|
||||
gevent.joinall(tasks)
|
||||
util.check_gevent_exceptions(*tasks)
|
||||
number_of_videos, polymer_json = tasks[0].value, tasks[1].value
|
||||
else:
|
||||
# For shorts/streams, item count is used instead
|
||||
polymer_json = gevent.spawn(*page_call)
|
||||
polymer_json.join()
|
||||
if polymer_json.exception:
|
||||
raise polymer_json.exception
|
||||
polymer_json = polymer_json.value
|
||||
number_of_videos = 0 # will be replaced by actual item count later
|
||||
|
||||
elif tab == 'about':
|
||||
# polymer_json = util.fetch_url(base_url + '/about?pbj=1', headers_desktop, debug_name='gen_channel_about')
|
||||
@@ -577,7 +560,8 @@ def get_channel_page_general_url(base_url, tab, request, channel_id=None):
|
||||
channel_id = info['channel_id']
|
||||
|
||||
# Will have microformat present, cache metadata while we have it
|
||||
if channel_id and default_params and tab not in ('videos', 'about'):
|
||||
if (channel_id and default_params and tab not in ('videos', 'about')
|
||||
and info.get('channel_name') is not None):
|
||||
metadata = extract_metadata_for_caching(info)
|
||||
set_cached_metadata(channel_id, metadata)
|
||||
# Otherwise, populate with our (hopefully cached) metadata
|
||||
@@ -595,8 +579,12 @@ def get_channel_page_general_url(base_url, tab, request, channel_id=None):
|
||||
item.update(additional_info)
|
||||
|
||||
if tab in ('videos', 'shorts', 'streams'):
|
||||
if tab in ('shorts', 'streams'):
|
||||
# For shorts/streams, use the actual item count since
|
||||
# get_number_of_videos_channel counts regular uploads only
|
||||
number_of_videos = len(info.get('items', []))
|
||||
info['number_of_videos'] = number_of_videos
|
||||
info['number_of_pages'] = math.ceil(number_of_videos/page_size)
|
||||
info['number_of_pages'] = math.ceil(number_of_videos/page_size) if number_of_videos else 1
|
||||
info['header_playlist_names'] = local_playlist.get_playlist_names()
|
||||
if tab in ('videos', 'shorts', 'streams', 'playlists'):
|
||||
info['current_sort'] = sort
|
||||
|
||||
@@ -26,8 +26,7 @@ def video_ids_in_playlist(name):
|
||||
|
||||
|
||||
def add_to_playlist(name, video_info_list):
|
||||
if not os.path.exists(playlists_directory):
|
||||
os.makedirs(playlists_directory)
|
||||
os.makedirs(playlists_directory, exist_ok=True)
|
||||
ids = video_ids_in_playlist(name)
|
||||
missing_thumbnails = []
|
||||
with open(os.path.join(playlists_directory, name + ".txt"), "a", encoding='utf-8') as file:
|
||||
|
||||
@@ -30,42 +30,58 @@ def playlist_ctoken(playlist_id, offset, include_shorts=True):
|
||||
|
||||
def playlist_first_page(playlist_id, report_text="Retrieved playlist",
|
||||
use_mobile=False):
|
||||
if use_mobile:
|
||||
url = 'https://m.youtube.com/playlist?list=' + playlist_id + '&pbj=1'
|
||||
content = util.fetch_url(
|
||||
url, util.mobile_xhr_headers,
|
||||
report_text=report_text, debug_name='playlist_first_page'
|
||||
)
|
||||
content = json.loads(content.decode('utf-8'))
|
||||
else:
|
||||
url = 'https://www.youtube.com/playlist?list=' + playlist_id + '&pbj=1'
|
||||
content = util.fetch_url(
|
||||
url, util.desktop_xhr_headers,
|
||||
report_text=report_text, debug_name='playlist_first_page'
|
||||
)
|
||||
content = json.loads(content.decode('utf-8'))
|
||||
# Use innertube API (pbj=1 no longer works for many playlists)
|
||||
key = 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'
|
||||
url = 'https://www.youtube.com/youtubei/v1/browse?key=' + key
|
||||
|
||||
return content
|
||||
data = {
|
||||
'context': {
|
||||
'client': {
|
||||
'hl': 'en',
|
||||
'gl': 'US',
|
||||
'clientName': 'WEB',
|
||||
'clientVersion': '2.20240327.00.00',
|
||||
},
|
||||
},
|
||||
'browseId': 'VL' + playlist_id,
|
||||
}
|
||||
|
||||
content_type_header = (('Content-Type', 'application/json'),)
|
||||
content = util.fetch_url(
|
||||
url, util.desktop_xhr_headers + content_type_header,
|
||||
data=json.dumps(data),
|
||||
report_text=report_text, debug_name='playlist_first_page'
|
||||
)
|
||||
return json.loads(content.decode('utf-8'))
|
||||
|
||||
|
||||
def get_videos(playlist_id, page, include_shorts=True, use_mobile=False,
|
||||
report_text='Retrieved playlist'):
|
||||
# mobile requests return 20 videos per page
|
||||
if use_mobile:
|
||||
page_size = 20
|
||||
headers = util.mobile_xhr_headers
|
||||
# desktop requests return 100 videos per page
|
||||
else:
|
||||
page_size = 100
|
||||
headers = util.desktop_xhr_headers
|
||||
page_size = 100
|
||||
|
||||
url = "https://m.youtube.com/playlist?ctoken="
|
||||
url += playlist_ctoken(playlist_id, (int(page)-1)*page_size,
|
||||
include_shorts=include_shorts)
|
||||
url += "&pbj=1"
|
||||
key = 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'
|
||||
url = 'https://www.youtube.com/youtubei/v1/browse?key=' + key
|
||||
|
||||
ctoken = playlist_ctoken(playlist_id, (int(page)-1)*page_size,
|
||||
include_shorts=include_shorts)
|
||||
|
||||
data = {
|
||||
'context': {
|
||||
'client': {
|
||||
'hl': 'en',
|
||||
'gl': 'US',
|
||||
'clientName': 'WEB',
|
||||
'clientVersion': '2.20240327.00.00',
|
||||
},
|
||||
},
|
||||
'continuation': ctoken,
|
||||
}
|
||||
|
||||
content_type_header = (('Content-Type', 'application/json'),)
|
||||
content = util.fetch_url(
|
||||
url, headers, report_text=report_text,
|
||||
debug_name='playlist_videos'
|
||||
url, util.desktop_xhr_headers + content_type_header,
|
||||
data=json.dumps(data),
|
||||
report_text=report_text, debug_name='playlist_videos'
|
||||
)
|
||||
|
||||
info = json.loads(content.decode('utf-8'))
|
||||
@@ -96,7 +112,7 @@ def get_playlist_page():
|
||||
tasks = (
|
||||
gevent.spawn(
|
||||
playlist_first_page, playlist_id,
|
||||
report_text="Retrieved playlist info", use_mobile=True
|
||||
report_text="Retrieved playlist info"
|
||||
),
|
||||
gevent.spawn(get_videos, playlist_id, page)
|
||||
)
|
||||
|
||||
@@ -30,8 +30,7 @@ database_path = os.path.join(settings.data_dir, "subscriptions.sqlite")
|
||||
|
||||
|
||||
def open_database():
|
||||
if not os.path.exists(settings.data_dir):
|
||||
os.makedirs(settings.data_dir)
|
||||
os.makedirs(settings.data_dir, exist_ok=True)
|
||||
connection = sqlite3.connect(database_path, check_same_thread=False)
|
||||
|
||||
try:
|
||||
|
||||
@@ -58,7 +58,9 @@
|
||||
|
||||
<div class="stats {{'horizontal-stats' if horizontal else 'vertical-stats'}}">
|
||||
{% if info['type'] == 'channel' %}
|
||||
<div>{{ info['approx_subscriber_count'] }} subscribers</div>
|
||||
{% if info.get('approx_subscriber_count') %}
|
||||
<div>{{ info['approx_subscriber_count'] }} subscribers</div>
|
||||
{% endif %}
|
||||
<div>{{ info['video_count']|commatize }} videos</div>
|
||||
{% else %}
|
||||
{% if info.get('time_published') %}
|
||||
|
||||
@@ -343,8 +343,7 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None,
|
||||
and debug_name is not None
|
||||
and content):
|
||||
save_dir = os.path.join(settings.data_dir, 'debug')
|
||||
if not os.path.exists(save_dir):
|
||||
os.makedirs(save_dir)
|
||||
os.makedirs(save_dir, exist_ok=True)
|
||||
|
||||
with open(os.path.join(save_dir, debug_name), 'wb') as f:
|
||||
f.write(content)
|
||||
@@ -838,9 +837,12 @@ INNERTUBE_CLIENTS = {
|
||||
'hl': 'en',
|
||||
'gl': 'US',
|
||||
'clientName': 'IOS',
|
||||
'clientVersion': '19.09.3',
|
||||
'deviceModel': 'iPhone14,3',
|
||||
'userAgent': 'com.google.ios.youtube/19.09.3 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
|
||||
'clientVersion': '21.03.2',
|
||||
'deviceMake': 'Apple',
|
||||
'deviceModel': 'iPhone16,2',
|
||||
'osName': 'iPhone',
|
||||
'osVersion': '18.7.2.22H124',
|
||||
'userAgent': 'com.google.ios.youtube/21.03.2 (iPhone16,2; U; CPU iOS 18_7_2 like Mac OS X)'
|
||||
}
|
||||
},
|
||||
'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
|
||||
@@ -902,8 +904,7 @@ INNERTUBE_CLIENTS = {
|
||||
def get_visitor_data():
|
||||
visitor_data = None
|
||||
visitor_data_cache = os.path.join(settings.data_dir, 'visitorData.txt')
|
||||
if not os.path.exists(settings.data_dir):
|
||||
os.makedirs(settings.data_dir)
|
||||
os.makedirs(settings.data_dir, exist_ok=True)
|
||||
if os.path.isfile(visitor_data_cache):
|
||||
with open(visitor_data_cache, 'r') as file:
|
||||
print('Getting visitor_data from cache')
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
__version__ = 'v0.4.2'
|
||||
__version__ = 'v0.4.5'
|
||||
|
||||
@@ -329,11 +329,8 @@ def get_ordered_music_list_attributes(music_list):
|
||||
|
||||
|
||||
def save_decrypt_cache():
|
||||
try:
|
||||
f = open(os.path.join(settings.data_dir, 'decrypt_function_cache.json'), 'w')
|
||||
except FileNotFoundError:
|
||||
os.makedirs(settings.data_dir)
|
||||
f = open(os.path.join(settings.data_dir, 'decrypt_function_cache.json'), 'w')
|
||||
os.makedirs(settings.data_dir, exist_ok=True)
|
||||
f = open(os.path.join(settings.data_dir, 'decrypt_function_cache.json'), 'w')
|
||||
|
||||
f.write(json.dumps({'version': 1, 'decrypt_cache':decrypt_cache}, indent=4, sort_keys=True))
|
||||
f.close()
|
||||
|
||||
@@ -241,7 +241,7 @@ def extract_lockup_view_model_info(item, additional_info={}):
|
||||
info['title'] = title_data.get('content', '')
|
||||
|
||||
# Determine type based on contentType
|
||||
if 'PLAYLIST' in content_type:
|
||||
if 'PLAYLIST' in content_type or 'PODCAST' in content_type:
|
||||
info['type'] = 'playlist'
|
||||
info['playlist_type'] = 'playlist'
|
||||
info['id'] = content_id
|
||||
@@ -253,7 +253,7 @@ def extract_lockup_view_model_info(item, additional_info={}):
|
||||
for row in metadata_rows.get('contentMetadataViewModel', {}).get('metadataRows', []):
|
||||
for part in row.get('metadataParts', []):
|
||||
text = part.get('text', {}).get('content', '')
|
||||
if 'video' in text.lower():
|
||||
if 'video' in text.lower() or 'episode' in text.lower():
|
||||
info['video_count'] = extract_int(text)
|
||||
elif 'VIDEO' in content_type:
|
||||
info['type'] = 'video'
|
||||
@@ -276,25 +276,48 @@ def extract_lockup_view_model_info(item, additional_info={}):
|
||||
info['type'] = 'channel'
|
||||
info['id'] = content_id
|
||||
info['approx_subscriber_count'] = None
|
||||
info['video_count'] = None
|
||||
|
||||
# Extract subscriber count and video count from metadata rows
|
||||
metadata_rows = lockup_metadata.get('metadata', {})
|
||||
for row in metadata_rows.get('contentMetadataViewModel', {}).get('metadataRows', []):
|
||||
for part in row.get('metadataParts', []):
|
||||
text = part.get('text', {}).get('content', '')
|
||||
if 'subscriber' in text.lower():
|
||||
info['approx_subscriber_count'] = extract_approx_int(text)
|
||||
elif 'video' in text.lower():
|
||||
info['video_count'] = extract_int(text)
|
||||
else:
|
||||
info['type'] = 'unsupported'
|
||||
return info
|
||||
|
||||
# Extract thumbnail from contentImage
|
||||
content_image = item.get('contentImage', {})
|
||||
collection_thumb = content_image.get('collectionThumbnailViewModel', {})
|
||||
primary_thumb = collection_thumb.get('primaryThumbnail', {})
|
||||
thumb_vm = primary_thumb.get('thumbnailViewModel', {})
|
||||
image_sources = thumb_vm.get('image', {}).get('sources', [])
|
||||
if image_sources:
|
||||
info['thumbnail'] = image_sources[0].get('url', '')
|
||||
else:
|
||||
info['thumbnail'] = ''
|
||||
info['thumbnail'] = normalize_url(multi_deep_get(content_image,
|
||||
# playlists with collection thumbnail
|
||||
['collectionThumbnailViewModel', 'primaryThumbnail', 'thumbnailViewModel', 'image', 'sources', 0, 'url'],
|
||||
# single thumbnail (some playlists, videos)
|
||||
['thumbnailViewModel', 'image', 'sources', 0, 'url'],
|
||||
)) or ''
|
||||
|
||||
# Extract video/episode count from thumbnail overlay badges
|
||||
# (podcasts and some playlists put the count here instead of metadata rows)
|
||||
thumb_vm = multi_deep_get(content_image,
|
||||
['collectionThumbnailViewModel', 'primaryThumbnail', 'thumbnailViewModel'],
|
||||
['thumbnailViewModel'],
|
||||
) or {}
|
||||
for overlay in thumb_vm.get('overlays', []):
|
||||
for badge in deep_get(overlay, 'thumbnailOverlayBadgeViewModel', 'thumbnailBadges', default=[]):
|
||||
badge_text = deep_get(badge, 'thumbnailBadgeViewModel', 'text', default='')
|
||||
if badge_text and not info.get('video_count'):
|
||||
conservative_update(info, 'video_count', extract_int(badge_text))
|
||||
|
||||
# Extract author info if available
|
||||
info['author'] = None
|
||||
info['author_id'] = None
|
||||
info['author_url'] = None
|
||||
info['description'] = None
|
||||
info['badges'] = []
|
||||
|
||||
# Try to get first video ID from inline player data
|
||||
item_playback = item.get('itemPlayback', {})
|
||||
@@ -309,6 +332,84 @@ def extract_lockup_view_model_info(item, additional_info={}):
|
||||
return info
|
||||
|
||||
|
||||
def extract_shorts_lockup_view_model_info(item, additional_info={}):
|
||||
"""Extract info from shortsLockupViewModel format (YouTube Shorts)"""
|
||||
info = {'error': None, 'type': 'video'}
|
||||
|
||||
# Video ID from reelWatchEndpoint or entityId
|
||||
info['id'] = deep_get(item,
|
||||
'onTap', 'innertubeCommand', 'reelWatchEndpoint', 'videoId')
|
||||
if not info['id']:
|
||||
entity_id = item.get('entityId', '')
|
||||
if entity_id.startswith('shorts-shelf-item-'):
|
||||
info['id'] = entity_id[len('shorts-shelf-item-'):]
|
||||
|
||||
# Thumbnail
|
||||
info['thumbnail'] = normalize_url(deep_get(item,
|
||||
'onTap', 'innertubeCommand', 'reelWatchEndpoint',
|
||||
'thumbnail', 'thumbnails', 0, 'url'))
|
||||
|
||||
# Parse title and views from accessibilityText
|
||||
# Format: "Title, N views - play Short"
|
||||
acc_text = item.get('accessibilityText', '')
|
||||
info['title'] = ''
|
||||
info['view_count'] = None
|
||||
info['approx_view_count'] = None
|
||||
if acc_text:
|
||||
# Remove trailing " - play Short"
|
||||
cleaned = re.sub(r'\s*-\s*play Short$', '', acc_text)
|
||||
# Split on last comma+views pattern to separate title from view count
|
||||
match = re.match(r'^(.*?),\s*([\d,.]+\s*(?:thousand|million|billion|)\s*views?)$',
|
||||
cleaned, re.IGNORECASE)
|
||||
if match:
|
||||
info['title'] = match.group(1).strip()
|
||||
view_text = match.group(2)
|
||||
info['view_count'] = extract_int(view_text)
|
||||
# Convert "7.1 thousand" -> "7.1 K" for display
|
||||
suffix_map = {'thousand': 'K', 'million': 'M', 'billion': 'B'}
|
||||
suffix_match = re.search(r'([\d,.]+)\s*(thousand|million|billion)?', view_text, re.IGNORECASE)
|
||||
if suffix_match:
|
||||
num = suffix_match.group(1)
|
||||
word = suffix_match.group(2)
|
||||
if word:
|
||||
info['approx_view_count'] = num + ' ' + suffix_map[word.lower()]
|
||||
else:
|
||||
info['approx_view_count'] = '{:,}'.format(int(num.replace(',', ''))) if num.isdigit() or num.replace(',','').isdigit() else num
|
||||
else:
|
||||
info['approx_view_count'] = extract_approx_int(view_text)
|
||||
else:
|
||||
# Fallback: try "N views" at end
|
||||
match2 = re.match(r'^(.*?),\s*(.+views?)$', cleaned, re.IGNORECASE)
|
||||
if match2:
|
||||
info['title'] = match2.group(1).strip()
|
||||
info['approx_view_count'] = extract_approx_int(match2.group(2))
|
||||
else:
|
||||
info['title'] = cleaned
|
||||
|
||||
# Overlay text (usually has the title too)
|
||||
overlay_metadata = deep_get(item, 'overlayMetadata',
|
||||
'secondaryText', 'content')
|
||||
if overlay_metadata and not info['approx_view_count']:
|
||||
info['approx_view_count'] = extract_approx_int(overlay_metadata)
|
||||
|
||||
primary_text = deep_get(item, 'overlayMetadata',
|
||||
'primaryText', 'content')
|
||||
if primary_text and not info['title']:
|
||||
info['title'] = primary_text
|
||||
|
||||
info['duration'] = ''
|
||||
info['time_published'] = None
|
||||
info['description'] = None
|
||||
info['badges'] = []
|
||||
info['author'] = None
|
||||
info['author_id'] = None
|
||||
info['author_url'] = None
|
||||
info['index'] = None
|
||||
|
||||
info.update(additional_info)
|
||||
return info
|
||||
|
||||
|
||||
def extract_item_info(item, additional_info={}):
|
||||
if not item:
|
||||
return {'error': 'No item given'}
|
||||
@@ -330,6 +431,10 @@ def extract_item_info(item, additional_info={}):
|
||||
if type == 'lockupViewModel':
|
||||
return extract_lockup_view_model_info(item, additional_info)
|
||||
|
||||
# Handle shortsLockupViewModel format (YouTube Shorts)
|
||||
if type == 'shortsLockupViewModel':
|
||||
return extract_shorts_lockup_view_model_info(item, additional_info)
|
||||
|
||||
# type looks like e.g. 'compactVideoRenderer' or 'gridVideoRenderer'
|
||||
# camelCase split, https://stackoverflow.com/a/37697078
|
||||
type_parts = [s.lower() for s in re.sub(r'([A-Z][a-z]+)', r' \1', type).split()]
|
||||
@@ -463,6 +568,13 @@ def extract_item_info(item, additional_info={}):
|
||||
elif primary_type == 'channel':
|
||||
info['id'] = item.get('channelId')
|
||||
info['approx_subscriber_count'] = extract_approx_int(item.get('subscriberCountText'))
|
||||
# YouTube sometimes puts the handle (@name) in subscriberCountText
|
||||
# instead of the actual count. Fall back to accessibility data.
|
||||
if not info['approx_subscriber_count']:
|
||||
acc_label = deep_get(item, 'subscriberCountText',
|
||||
'accessibility', 'accessibilityData', 'label', default='')
|
||||
if 'subscriber' in acc_label.lower():
|
||||
info['approx_subscriber_count'] = extract_approx_int(acc_label)
|
||||
elif primary_type == 'show':
|
||||
info['id'] = deep_get(item, 'navigationEndpoint', 'watchEndpoint', 'playlistId')
|
||||
info['first_video_id'] = deep_get(item, 'navigationEndpoint',
|
||||
@@ -531,6 +643,7 @@ _item_types = {
|
||||
|
||||
# New viewModel format (YouTube 2024+)
|
||||
'lockupViewModel',
|
||||
'shortsLockupViewModel',
|
||||
}
|
||||
|
||||
def _traverse_browse_renderer(renderer):
|
||||
|
||||
@@ -218,40 +218,100 @@ def extract_playlist_metadata(polymer_json):
|
||||
return {'error': err}
|
||||
|
||||
metadata = {'error': None}
|
||||
header = deep_get(response, 'header', 'playlistHeaderRenderer', default={})
|
||||
metadata['title'] = extract_str(header.get('title'))
|
||||
metadata['title'] = None
|
||||
metadata['first_video_id'] = None
|
||||
metadata['thumbnail'] = None
|
||||
metadata['video_count'] = None
|
||||
metadata['description'] = ''
|
||||
metadata['author'] = None
|
||||
metadata['author_id'] = None
|
||||
metadata['author_url'] = None
|
||||
metadata['view_count'] = None
|
||||
metadata['like_count'] = None
|
||||
metadata['time_published'] = None
|
||||
|
||||
header = deep_get(response, 'header', 'playlistHeaderRenderer', default={})
|
||||
|
||||
if header:
|
||||
# Classic playlistHeaderRenderer format
|
||||
metadata['title'] = extract_str(header.get('title'))
|
||||
metadata['first_video_id'] = deep_get(header, 'playEndpoint', 'watchEndpoint', 'videoId')
|
||||
first_id = re.search(r'([a-z_\-]{11})', deep_get(header,
|
||||
'thumbnail', 'thumbnails', 0, 'url', default=''))
|
||||
if first_id:
|
||||
conservative_update(metadata, 'first_video_id', first_id.group(1))
|
||||
|
||||
metadata['video_count'] = extract_int(header.get('numVideosText'))
|
||||
metadata['description'] = extract_str(header.get('descriptionText'), default='')
|
||||
metadata['author'] = extract_str(header.get('ownerText'))
|
||||
metadata['author_id'] = multi_deep_get(header,
|
||||
['ownerText', 'runs', 0, 'navigationEndpoint', 'browseEndpoint', 'browseId'],
|
||||
['ownerEndpoint', 'browseEndpoint', 'browseId'])
|
||||
metadata['view_count'] = extract_int(header.get('viewCountText'))
|
||||
metadata['like_count'] = extract_int(header.get('likesCountWithoutLikeText'))
|
||||
for stat in header.get('stats', ()):
|
||||
text = extract_str(stat)
|
||||
if 'videos' in text or 'episodes' in text:
|
||||
conservative_update(metadata, 'video_count', extract_int(text))
|
||||
elif 'views' in text:
|
||||
conservative_update(metadata, 'view_count', extract_int(text))
|
||||
elif 'updated' in text:
|
||||
metadata['time_published'] = extract_date(text)
|
||||
else:
|
||||
# New pageHeaderRenderer format (YouTube 2024+)
|
||||
page_header = deep_get(response, 'header', 'pageHeaderRenderer', default={})
|
||||
metadata['title'] = page_header.get('pageTitle')
|
||||
view_model = deep_get(page_header, 'content', 'pageHeaderViewModel', default={})
|
||||
|
||||
# Extract title from viewModel if not found
|
||||
if not metadata['title']:
|
||||
metadata['title'] = deep_get(view_model,
|
||||
'title', 'dynamicTextViewModel', 'text', 'content')
|
||||
|
||||
# Extract metadata from rows (author, video count, views, etc.)
|
||||
meta_rows = deep_get(view_model,
|
||||
'metadata', 'contentMetadataViewModel', 'metadataRows', default=[])
|
||||
for row in meta_rows:
|
||||
for part in row.get('metadataParts', []):
|
||||
text_content = deep_get(part, 'text', 'content', default='')
|
||||
# Author from avatarStack
|
||||
avatar_stack = deep_get(part, 'avatarStack', 'avatarStackViewModel', default={})
|
||||
if avatar_stack:
|
||||
author_text = deep_get(avatar_stack, 'text', 'content')
|
||||
if author_text:
|
||||
metadata['author'] = author_text
|
||||
# Extract author_id from commandRuns
|
||||
for run in deep_get(avatar_stack, 'text', 'commandRuns', default=[]):
|
||||
browse_id = deep_get(run, 'onTap', 'innertubeCommand',
|
||||
'browseEndpoint', 'browseId')
|
||||
if browse_id:
|
||||
metadata['author_id'] = browse_id
|
||||
# Video/episode count
|
||||
if text_content and ('video' in text_content.lower() or 'episode' in text_content.lower()):
|
||||
conservative_update(metadata, 'video_count', extract_int(text_content))
|
||||
# View count
|
||||
elif text_content and 'view' in text_content.lower():
|
||||
conservative_update(metadata, 'view_count', extract_int(text_content))
|
||||
# Last updated
|
||||
elif text_content and 'updated' in text_content.lower():
|
||||
metadata['time_published'] = extract_date(text_content)
|
||||
|
||||
# Extract description from sidebar if available
|
||||
sidebar = deep_get(response, 'sidebar', 'playlistSidebarRenderer', 'items', default=[])
|
||||
for sidebar_item in sidebar:
|
||||
desc = deep_get(sidebar_item, 'playlistSidebarPrimaryInfoRenderer',
|
||||
'description', 'simpleText')
|
||||
if desc:
|
||||
metadata['description'] = desc
|
||||
|
||||
if metadata['author_id']:
|
||||
metadata['author_url'] = 'https://www.youtube.com/channel/' + metadata['author_id']
|
||||
|
||||
metadata['first_video_id'] = deep_get(header, 'playEndpoint', 'watchEndpoint', 'videoId')
|
||||
first_id = re.search(r'([a-z_\-]{11})', deep_get(header,
|
||||
'thumbnail', 'thumbnails', 0, 'url', default=''))
|
||||
if first_id:
|
||||
conservative_update(metadata, 'first_video_id', first_id.group(1))
|
||||
if metadata['first_video_id'] is None:
|
||||
metadata['thumbnail'] = None
|
||||
else:
|
||||
metadata['thumbnail'] = f"https://i.ytimg.com/vi/{metadata['first_video_id']}/hqdefault.jpg"
|
||||
|
||||
metadata['video_count'] = extract_int(header.get('numVideosText'))
|
||||
metadata['description'] = extract_str(header.get('descriptionText'), default='')
|
||||
metadata['author'] = extract_str(header.get('ownerText'))
|
||||
metadata['author_id'] = multi_deep_get(header,
|
||||
['ownerText', 'runs', 0, 'navigationEndpoint', 'browseEndpoint', 'browseId'],
|
||||
['ownerEndpoint', 'browseEndpoint', 'browseId'])
|
||||
if metadata['author_id']:
|
||||
metadata['author_url'] = 'https://www.youtube.com/channel/' + metadata['author_id']
|
||||
else:
|
||||
metadata['author_url'] = None
|
||||
metadata['view_count'] = extract_int(header.get('viewCountText'))
|
||||
metadata['like_count'] = extract_int(header.get('likesCountWithoutLikeText'))
|
||||
for stat in header.get('stats', ()):
|
||||
text = extract_str(stat)
|
||||
if 'videos' in text:
|
||||
conservative_update(metadata, 'video_count', extract_int(text))
|
||||
elif 'views' in text:
|
||||
conservative_update(metadata, 'view_count', extract_int(text))
|
||||
elif 'updated' in text:
|
||||
metadata['time_published'] = extract_date(text)
|
||||
|
||||
microformat = deep_get(response, 'microformat', 'microformatDataRenderer',
|
||||
default={})
|
||||
conservative_update(
|
||||
|
||||
Reference in New Issue
Block a user