version 0.2.20

This was an old experiment to collect googlevideo domains to see
if there was a pattern that could correlate to IP address to
look for workarounds for 403 errors

Can bug out if enabled and if failed to get any vidoe urls,
so remove since it is obsolete and some people are enabling it

See #218

.gitea/workflows/ci.yaml

@@ -0,0 +1,23 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install -r requirements-dev.txt
+
+      - name: Run tests
+        run: pytest

.gitea/workflows/git-sync.yaml

@@ -0,0 +1,40 @@
+name: git-sync-with-mirror
+
+on:
+  push:
+    branches: [ master ]
+  workflow_dispatch:
+
+jobs:
+  git-sync:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: git-sync
+        env:
+          git_sync_source_repo: git@git.fridu.us:heckyel/yt-local.git
+          git_sync_destination_repo: ssh://git@c.fridu.us/software/yt-local.git
+        if: env.git_sync_source_repo && env.git_sync_destination_repo
+        uses: astounds/git-sync@v1
+        with:
+          source_repo: git@git.fridu.us:heckyel/yt-local.git
+          source_branch: "master"
+          destination_repo: ssh://git@c.fridu.us/software/yt-local.git
+          destination_branch: "master"
+          source_ssh_private_key: ${{ secrets.GIT_SYNC_SOURCE_SSH_PRIVATE_KEY }}
+          destination_ssh_private_key:  ${{ secrets.GIT_SYNC_DESTINATION_SSH_PRIVATE_KEY }}
+
+      - name: git-sync-sourcehut
+        env:
+          git_sync_source_repo: git@git.fridu.us:heckyel/yt-local.git
+          git_sync_destination_repo: git@git.sr.ht:~heckyel/yt-local
+        if: env.git_sync_source_repo && env.git_sync_destination_repo
+        uses: astounds/git-sync@v1
+        with:
+          source_repo: git@git.fridu.us:heckyel/yt-local.git
+          source_branch: "master"
+          destination_repo: git@git.sr.ht:~heckyel/yt-local
+          destination_branch: "master"
+          source_ssh_private_key: ${{ secrets.GIT_SYNC_SOURCE_SSH_PRIVATE_KEY }}
+          destination_ssh_private_key:  ${{ secrets.GIT_SYNC_DESTINATION_SSH_PRIVATE_KEY }}
+        continue-on-error: true

README.md

@@ -151,7 +151,7 @@ For coding guidelines and an overview of the software architecture, see the [HAC
 
 yt-local is not made to work in public mode, however there is an instance of yt-local in public mode but with less features
 
-- <https://1cd1-93-95-230-133.ngrok-free.app/https://youtube.com>
+- <https://m.fridu.us/https://youtube.com>
 
 ## License
 

requirements-dev.txt

@@ -8,14 +8,14 @@ gevent==24.2.1
 greenlet==3.0.3
 iniconfig==2.0.0
 itsdangerous==2.1.2
-Jinja2==3.1.3
+Jinja2==3.1.4
 MarkupSafe==2.1.5
 packaging==24.0
 pluggy==1.4.0
 PySocks==1.7.1
 pytest==8.1.1
 stem==1.8.2
-urllib3==2.2.1
-Werkzeug==3.0.1
+urllib3==2.2.2
+Werkzeug==3.0.3
 zope.event==5.0
 zope.interface==6.2

requirements.txt

@@ -7,11 +7,11 @@ Flask==3.0.2
 gevent==24.2.1
 greenlet==3.0.3
 itsdangerous==2.1.2
-Jinja2==3.1.3
+Jinja2==3.1.4
 MarkupSafe==2.1.5
 PySocks==1.7.1
 stem==1.8.2
-urllib3==2.2.1
-Werkzeug==3.0.1
+urllib3==2.2.2
+Werkzeug==3.0.3
 zope.event==5.0
 zope.interface==6.2

settings.py

@@ -322,13 +322,6 @@ Archive: https://archive.ph/OZQbN''',
         'comment': '',
     }),
 
-    ('gather_googlevideo_domains', {
-        'type': bool,
-        'default': False,
-        'comment': '''Developer use to debug 403s''',
-        'hidden': True,
-    }),
-
     ('debugging_save_responses', {
         'type': bool,
         'default': False,
@@ -338,7 +331,7 @@ Archive: https://archive.ph/OZQbN''',
 
     ('settings_version', {
         'type': int,
-        'default': 5,
+        'default': 6,
         'comment': '''Do not change, remove, or comment out this value, or else your settings may be lost or corrupted''',
         'hidden': True,
     }),
@@ -419,11 +412,20 @@ def upgrade_to_5(settings_dict):
     return new_settings
 
 
+def upgrade_to_6(settings_dict):
+    new_settings = settings_dict.copy()
+    if 'gather_googlevideo_domains' in new_settings:
+        del new_settings['gather_googlevideo_domains']
+    new_settings['settings_version'] = 6
+    return new_settings
+
+
 upgrade_functions = {
     1: upgrade_to_2,
     2: upgrade_to_3,
     3: upgrade_to_4,
     4: upgrade_to_5,
+    5: upgrade_to_6,
 }
 
 

youtube/channel.py

@@ -292,7 +292,7 @@ def get_number_of_videos_channel(channel_id):
     try:
         response = util.fetch_url(url, headers_mobile,
             debug_name='number_of_videos', report_text='Got number of videos')
-    except urllib.error.HTTPError as e:
+    except (urllib.error.HTTPError, util.FetchError) as e:
         traceback.print_exc()
         print("Couldn't retrieve number of videos")
         return 1000

youtube/playlist.py

@@ -115,7 +115,7 @@ def get_playlist_page():
 
     video_count = yt_data_extract.deep_get(info, 'metadata', 'video_count')
     if video_count is None:
-        video_count = 40
+        video_count = 1000
 
     return flask.render_template(
         'playlist.html',

youtube/static/channel.css

@@ -256,7 +256,8 @@ hr {
     padding-top: 6px;
     text-align: center;
     white-space: nowrap;
-    border: none;
+    border: 1px solid;
+    border-color: var(--button-border);
     border-radius: 0.2rem;
 }
 

youtube/static/dark_theme.css

@@ -1,20 +1,22 @@
 :root {
-    --background:           #212121;
+    --background:           #121113;
     --text:                 #FFFFFF;
-    --secondary-hover:      #73828c;
-    --secondary-focus:      #303030;
-    --secondary-inverse:    #FFF;
+    --secondary-hover:      #222222;
+    --secondary-focus:      #121113;
+    --secondary-inverse:    #FFFFFF;
     --primary-background:   #242424;
-    --secondary-background: #424242;
-    --thumb-background:     #757575;
+    --secondary-background: #222222;
+    --thumb-background:     #222222;
     --link:                 #00B0FF;
     --link-visited:         #40C4FF;
-    --border-bg:            #FFFFFF;
-    --buttom:               #dcdcdb;
-    --buttom-text:          #415462;
-    --button-border:        #91918c;
-    --buttom-hover:         #BBB;
-    --search-text:          #FFF;
-    --time-background:      #212121;
-    --time-text:            #FFF;
+    --border-bg:            #222222;
+    --border-bg-settings:   #000000;
+    --border-bg-license:    #000000;
+    --buttom:               #121113;
+    --buttom-text:          #FFFFFF;
+    --button-border:        #222222;
+    --buttom-hover:         #222222;
+    --search-text:          #FFFFFF;
+    --time-background:      #121113;
+    --time-text:            #FFFFFF;
 }

youtube/static/gray_theme.css

@@ -1,19 +1,21 @@
 :root {
-    --background:            #2d3743;
+    --background:            #2D3743;
     --text:                  #FFFFFF;
-    --secondary-hover:       #73828c;
+    --secondary-hover:       #73828C;
     --secondary-focus:       rgba(115, 130, 140, 0.125);
     --secondary-inverse:     #FFFFFF;
-    --primary-background:    #2d3743;
+    --primary-background:    #2D3743;
     --secondary-background:  #102027;
     --thumb-background:      #35404D;
-    --link:                  #22aaff;
-    --link-visited:          #7755ff;
+    --link:                  #22AAFF;
+    --link-visited:          #7755FF;
     --border-bg:             #FFFFFF;
-    --buttom:                #DCDCDC;
-    --buttom-text:           #415462;
-    --button-border:         #91918c;
-    --buttom-hover:          #BBBBBB;
+    --border-bg-settings:    #FFFFFF;
+    --border-bg-license:     #FFFFFF;
+    --buttom:                #2D3743;
+    --buttom-text:           #FFFFFF;
+    --button-border:         #102027;
+    --buttom-hover:          #102027;
     --search-text:           #FFFFFF;
     --time-background:       #212121;
     --time-text:             #FFFFFF;

youtube/static/license.css

@@ -181,7 +181,7 @@ label[for=options-toggle-cbox] {
 
 .table td,.table th {
     padding: 10px 10px;
-    border: 1px solid var(--secondary-background);
+    border: 1px solid var(--border-bg-license);
     text-align: center;
 }
 

youtube/static/light_theme.css

@@ -10,9 +10,11 @@
     --link:                 #212121;
     --link-visited:         #808080;
     --border-bg:            #212121;
-    --buttom:               #DCDCDC;
+    --border-bg-settings:   #91918C;
+    --border-bg-license:    #91918C;
+    --buttom:               #FFFFFF;
     --buttom-text:          #212121;
-    --button-border:        #91918c;
+    --button-border:        #91918C;
     --buttom-hover:         #BBBBBB;
     --search-text:          #212121;
     --time-background:      #212121;

youtube/static/settings.css

@@ -155,7 +155,7 @@ label[for=options-toggle-cbox] {
 }
 
 .settings-form > h2 {
-    border-bottom: 2px solid var(--border-bg);
+    border-bottom: 2px solid var(--border-bg-settings);
     padding-bottom: 0.5rem;
 }
 

youtube/util.py

@@ -321,7 +321,8 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None,
             response.getheader('Content-Encoding', default='identity'))
 
         if (settings.debugging_save_responses
-                and debug_name is not None and content):
+                and debug_name is not None
+                and content):
             save_dir = os.path.join(settings.data_dir, 'debug')
             if not os.path.exists(save_dir):
                 os.makedirs(save_dir)
@@ -394,23 +395,22 @@ def head(url, use_tor=False, report_text=None, max_redirects=10):
             round(time.monotonic() - start_time, 3))
     return response
 
-
-mobile_user_agent = 'Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.80 Mobile Safari/537.36'
+mobile_user_agent = 'Mozilla/5.0 (Linux; Android 7.0; Redmi Note 4 Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Mobile Safari/537.36'
 mobile_ua = (('User-Agent', mobile_user_agent),)
-desktop_user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:124.0) Gecko/20100101 Firefox/124.0'
+desktop_user_agent = 'Mozilla/5.0 (Windows NT 6.1; rv:52.0) Gecko/20100101 Firefox/52.0'
 desktop_ua = (('User-Agent', desktop_user_agent),)
 json_header = (('Content-Type', 'application/json'),)
 desktop_xhr_headers = (
     ('Accept', '*/*'),
     ('Accept-Language', 'en-US,en;q=0.5'),
     ('X-YouTube-Client-Name', '1'),
-    ('X-YouTube-Client-Version', '2.20240327.00.00'),
+    ('X-YouTube-Client-Version', '2.20240304.00.00'),
 ) + desktop_ua
 mobile_xhr_headers = (
     ('Accept', '*/*'),
     ('Accept-Language', 'en-US,en;q=0.5'),
-    ('X-YouTube-Client-Name', '1'),
-    ('X-YouTube-Client-Version', '2.20240328.08.00'),
+    ('X-YouTube-Client-Name', '2'),
+    ('X-YouTube-Client-Version', '2.20240304.08.00'),
 ) + mobile_ua
 
 
@@ -674,12 +674,12 @@ INNERTUBE_CLIENTS = {
                 'hl': 'en',
                 'gl': 'US',
                 'clientName': 'ANDROID',
-                'clientVersion': '19.12.36',
+                'clientVersion': '19.09.36',
                 'osName': 'Android',
-                'osVersion': '14',
-                'androidSdkVersion': 34,
+                'osVersion': '12',
+                'androidSdkVersion': 31,
                 'platform': 'MOBILE',
-                'userAgent': 'com.google.android.youtube/19.12.36 (Linux; U; Android 14; US) gzip'
+                'userAgent': 'com.google.android.youtube/19.09.36 (Linux; U; Android 12; US) gzip'
             },
             # https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-887739287
             #'thirdParty': {
@@ -690,6 +690,45 @@ INNERTUBE_CLIENTS = {
         'REQUIRE_JS_PLAYER': False,
     },
 
+    'android-test-suite': {
+        'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'hl': 'en',
+                'gl': 'US',
+                'clientName': 'ANDROID_TESTSUITE',
+                'clientVersion': '1.9',
+                'osName': 'Android',
+                'osVersion': '12',
+                'androidSdkVersion': 31,
+                'platform': 'MOBILE',
+                'userAgent': 'com.google.android.youtube/1.9 (Linux; U; Android 12; US) gzip'
+            },
+            # https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-887739287
+            #'thirdParty': {
+            #    'embedUrl': 'https://google.com',  # Can be any valid URL
+            #}
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
+        'REQUIRE_JS_PLAYER': False,
+    },
+
+    'ios': {
+        'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'hl': 'en',
+                'gl': 'US',
+                'clientName': 'IOS',
+                'clientVersion': '19.09.3',
+                'deviceModel': 'iPhone14,3',
+                'userAgent': 'com.google.ios.youtube/19.09.3 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
+            }
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
+        'REQUIRE_JS_PLAYER': False
+    },
+
     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
     'tv_embedded': {
@@ -717,14 +756,62 @@ INNERTUBE_CLIENTS = {
         'INNERTUBE_CONTEXT': {
             'client': {
                 'clientName': 'WEB',
-                'clientVersion': '2.20240327.00.00',
+                'clientVersion': '2.20220801.00.00',
                 'userAgent': desktop_user_agent,
             }
         },
         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
     },
+    'android_vr': {
+        'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'clientName': 'ANDROID_VR',
+                'clientVersion': '1.60.19',
+                'deviceMake': 'Oculus',
+                'deviceModel': 'Quest 3',
+                'androidSdkVersion': 32,
+                'userAgent': 'com.google.android.apps.youtube.vr.oculus/1.60.19 (Linux; U; Android 12L; eureka-user Build/SQ3A.220605.009.A1) gzip',
+                'osName': 'Android',
+                'osVersion': '12L',
+            },
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 28,
+        'REQUIRE_JS_PLAYER': False,
+    },
 }
 
+def get_visitor_data():
+    visitor_data = None
+    visitor_data_cache = os.path.join(settings.data_dir, 'visitorData.txt')
+    if not os.path.exists(settings.data_dir):
+        os.makedirs(settings.data_dir)
+    if os.path.isfile(visitor_data_cache):
+        with open(visitor_data_cache, 'r') as file:
+            print('Getting visitor_data from cache')
+            visitor_data = file.read()
+        max_age = 12*3600
+        file_age = time.time() - os.path.getmtime(visitor_data_cache)
+        if file_age > max_age:
+            print('visitor_data cache is too old. Removing file...')
+            os.remove(visitor_data_cache)
+        return visitor_data
+
+    print('Fetching youtube homepage to get visitor_data')
+    yt_homepage = 'https://www.youtube.com'
+    yt_resp = fetch_url(yt_homepage, headers={'User-Agent': mobile_user_agent}, report_text='Getting youtube homepage')
+    visitor_data_re = r'''"visitorData":\s*?"(.+?)"'''
+    visitor_data_match = re.search(visitor_data_re, yt_resp.decode())
+    if visitor_data_match:
+        visitor_data = visitor_data_match.group(1)
+        print(f'Got visitor_data: {len(visitor_data)}')
+        with open(visitor_data_cache, 'w') as file:
+            print('Saving visitor_data cache...')
+            file.write(visitor_data)
+        return visitor_data
+    else:
+        print('Unable to get visitor_data value')
+    return visitor_data
 
 def call_youtube_api(client, api, data):
     client_params = INNERTUBE_CLIENTS[client]
@@ -732,12 +819,17 @@ def call_youtube_api(client, api, data):
     key = client_params['INNERTUBE_API_KEY']
     host = client_params.get('INNERTUBE_HOST') or 'www.youtube.com'
     user_agent = context['client'].get('userAgent') or mobile_user_agent
+    visitor_data = get_visitor_data()
 
     url = 'https://' + host + '/youtubei/v1/' + api + '?key=' + key
+    if visitor_data:
+        context['client'].update({'visitorData': visitor_data})
     data['context'] = context
 
     data = json.dumps(data)
     headers = (('Content-Type', 'application/json'),('User-Agent', user_agent))
+    if visitor_data:
+        headers = ( *headers, ('X-Goog-Visitor-Id', visitor_data ))
     response = fetch_url(
         url, data=data, headers=headers,
         debug_name='youtubei_' + api + '_' + client,

youtube/version.py

@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '0.2.11'
+__version__ = '0.2.20'

youtube/watch.py

@@ -343,7 +343,6 @@ def _add_to_error(info, key, additional_message):
 def fetch_player_response(client, video_id):
     return util.call_youtube_api(client, 'player', {
         'videoId': video_id,
-        'params': 'CgIQBg',
     })
 
 
@@ -372,17 +371,7 @@ def extract_info(video_id, use_invidious, playlist_id=None, index=None):
     tasks = (
         # Get video metadata from here
         gevent.spawn(fetch_watch_page_info, video_id, playlist_id, index),
-
-        # Get video URLs by spoofing as android client because its urls don't
-        # require decryption
-        # The URLs returned with WEB for videos requiring decryption
-        # couldn't be decrypted with the base.js from the web page for some
-        # reason
-        # https://github.com/yt-dlp/yt-dlp/issues/574#issuecomment-887171136
-
-        # Update 4/26/23, these URLs will randomly start returning 403
-        # mid-playback and I'm not sure why
-        gevent.spawn(fetch_player_response, 'android', video_id)
+        gevent.spawn(fetch_player_response, 'android_vr', video_id)
     )
     gevent.joinall(tasks)
     util.check_gevent_exceptions(*tasks)
@@ -660,12 +649,6 @@ def get_watch_page(video_id=None):
             '/videoplayback',
             '/videoplayback/name/' + filename)
 
-    if settings.gather_googlevideo_domains:
-        with open(os.path.join(settings.data_dir, 'googlevideo-domains.txt'), 'a+', encoding='utf-8') as f:
-            url = info['formats'][0]['url']
-            subdomain = url[0:url.find(".googlevideo.com")]
-            f.write(subdomain + "\n")
-
     download_formats = []
 
     for format in (info['formats'] + info['hls_formats']):