version 0.2.19

This reverts commit 8af98968dd.

.gitea/workflows/ci.yaml

@@ -0,0 +1,23 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install -r requirements-dev.txt
+
+      - name: Run tests
+        run: pytest

.gitea/workflows/git-sync.yaml

@@ -0,0 +1,40 @@
+name: git-sync-with-mirror
+
+on:
+  push:
+    branches: [ master ]
+  workflow_dispatch:
+
+jobs:
+  git-sync:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: git-sync
+        env:
+          git_sync_source_repo: git@git.fridu.us:heckyel/yt-local.git
+          git_sync_destination_repo: ssh://git@c.fridu.us/software/yt-local.git
+        if: env.git_sync_source_repo && env.git_sync_destination_repo
+        uses: astounds/git-sync@v1
+        with:
+          source_repo: git@git.fridu.us:heckyel/yt-local.git
+          source_branch: "master"
+          destination_repo: ssh://git@c.fridu.us/software/yt-local.git
+          destination_branch: "master"
+          source_ssh_private_key: ${{ secrets.GIT_SYNC_SOURCE_SSH_PRIVATE_KEY }}
+          destination_ssh_private_key:  ${{ secrets.GIT_SYNC_DESTINATION_SSH_PRIVATE_KEY }}
+
+      - name: git-sync-sourcehut
+        env:
+          git_sync_source_repo: git@git.fridu.us:heckyel/yt-local.git
+          git_sync_destination_repo: git@git.sr.ht:~heckyel/yt-local
+        if: env.git_sync_source_repo && env.git_sync_destination_repo
+        uses: astounds/git-sync@v1
+        with:
+          source_repo: git@git.fridu.us:heckyel/yt-local.git
+          source_branch: "master"
+          destination_repo: git@git.sr.ht:~heckyel/yt-local
+          destination_branch: "master"
+          source_ssh_private_key: ${{ secrets.GIT_SYNC_SOURCE_SSH_PRIVATE_KEY }}
+          destination_ssh_private_key:  ${{ secrets.GIT_SYNC_DESTINATION_SSH_PRIVATE_KEY }}
+        continue-on-error: true

youtube/util.py

@@ -431,29 +431,34 @@ class RateLimitedQueue(gevent.queue.Queue):
         gevent.queue.Queue.__init__(self)
 
     def get(self):
-        with self.lock:     # blocks if another greenlet currently has the lock
-            if ((self.count_since_last_wait >= self.subsequent_bursts and self.surpassed_initial) or
-                (self.count_since_last_wait >= self.initial_burst and not self.surpassed_initial)):
-                self.surpassed_initial = True
-                gevent.sleep(self.waiting_period)
+        self.lock.acquire()     # blocks if another greenlet currently has the lock
+        if self.count_since_last_wait >= self.subsequent_bursts and self.surpassed_initial:
+            gevent.sleep(self.waiting_period)
+            self.count_since_last_wait = 0
+
+        elif self.count_since_last_wait >= self.initial_burst and not self.surpassed_initial:
+            self.surpassed_initial = True
+            gevent.sleep(self.waiting_period)
+            self.count_since_last_wait = 0
+
+        self.count_since_last_wait += 1
+
+        if not self.currently_empty and self.empty():
+            self.currently_empty = True
+            self.empty_start = time.monotonic()
+
+        item = gevent.queue.Queue.get(self)     # blocks when nothing left
+
+        if self.currently_empty:
+            if time.monotonic() - self.empty_start >= self.waiting_period:
                 self.count_since_last_wait = 0
+                self.surpassed_initial = False
 
-            self.count_since_last_wait += 1
+            self.currently_empty = False
 
-            if not self.currently_empty and self.empty():
-                self.currently_empty = True
-                self.empty_start = time.monotonic()
+        self.lock.release()
 
-            item = gevent.queue.Queue.get(self)     # blocks when nothing left
-
-            if self.currently_empty:
-                if time.monotonic() - self.empty_start >= self.waiting_period:
-                    self.count_since_last_wait = 0
-                    self.surpassed_initial = False
-
-                self.currently_empty = False
-
-            return item
+        return item
 
 
 def download_thumbnail(save_directory, video_id):
@@ -662,45 +667,16 @@ def to_valid_filename(name):
 
 # https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/extractor/youtube.py#L72
 INNERTUBE_CLIENTS = {
-    'android-test-suite': {
-        'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
+    'web_creator': {
+        'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
         'INNERTUBE_CONTEXT': {
             'client': {
-                'hl': 'en',
-                'gl': 'US',
-                'clientName': 'ANDROID_TESTSUITE',
-                'clientVersion': '1.9',
-                'osName': 'Android',
-                'osVersion': '12',
-                'androidSdkVersion': 31,
-                'platform': 'MOBILE',
-                'userAgent': 'com.google.android.youtube/1.9 (Linux; U; Android 12; US) gzip'
+                'clientName': 'WEB_CREATOR',
+                'clientVersion': '1.20240723.03.00',
             },
-            # https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-887739287
-            #'thirdParty': {
-            #    'embedUrl': 'https://google.com',  # Can be any valid URL
-            #}
         },
-        'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
-        'REQUIRE_JS_PLAYER': False,
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
     },
-
-    'ios': {
-        'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
-        'INNERTUBE_CONTEXT': {
-            'client': {
-                'hl': 'en',
-                'gl': 'US',
-                'clientName': 'IOS',
-                'clientVersion': '19.12.3',
-                'deviceModel': 'iPhone14,3',
-                'userAgent': 'com.google.ios.youtube/19.12.3 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
-            }
-        },
-        'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
-        'REQUIRE_JS_PLAYER': False
-    },
-
     'android': {
         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
         'INNERTUBE_CONTEXT': {
@@ -720,22 +696,22 @@ INNERTUBE_CLIENTS = {
         'REQUIRE_JS_PLAYER': False,
     },
 
-    'android_music': {
-        'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
+    'ios': {
+        'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
         'INNERTUBE_CONTEXT': {
             'client': {
                 'hl': 'en',
                 'gl': 'US',
-                'clientName': 'ANDROID_MUSIC',
-                'clientVersion': '6.48.51',
-                'osName': 'Android',
-                'osVersion': '14',
-                'androidSdkVersion': 34,
-                'platform': 'MOBILE',
-                'userAgent': 'com.google.android.apps.youtube.music/6.48.51 (Linux; U; Android 14; US) gzip'
+                'clientName': 'IOS',
+                'clientVersion': '19.29.1',
+                'deviceMake': 'Apple',
+                'deviceModel': 'iPhone16,2',
+                'userAgent': 'com.google.ios.youtube/19.29.1 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X;)',
+                'osName': 'iPhone',
+                'osVersion': '17.5.1.21F90',
             }
         },
-        'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
         'REQUIRE_JS_PLAYER': False
     },
 

youtube/version.py

@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '0.2.18'
+__version__ = '0.2.19'

youtube/watch.py

@@ -2,7 +2,6 @@ import youtube
 from youtube import yt_app
 from youtube import util, comments, local_playlist, yt_data_extract
 from youtube.util import time_utc_isoformat
-from youtube.util import INNERTUBE_CLIENTS
 import settings
 
 from flask import request
@@ -370,83 +369,83 @@ def fetch_watch_page_info(video_id, playlist_id, index):
     return yt_data_extract.extract_watch_info_from_html(watch_page)
 
 def extract_info(video_id, use_invidious, playlist_id=None, index=None):
-    for client in INNERTUBE_CLIENTS:
-        tasks = (
-            gevent.spawn(fetch_watch_page_info, video_id, playlist_id, index),
-            gevent.spawn(fetch_player_response, client, video_id)  # Use client from INNERTUBE_CLIENTS
-        )
-        gevent.joinall(tasks)
-        util.check_gevent_exceptions(*tasks)
-        info, player_response = tasks[0].value, tasks[1].value
+    tasks = (
+        # Get video metadata from here
+        gevent.spawn(fetch_watch_page_info, video_id, playlist_id, index),
+        gevent.spawn(fetch_player_response, 'ios', video_id)
+    )
+    gevent.joinall(tasks)
+    util.check_gevent_exceptions(*tasks)
+    info, player_response = tasks[0].value, tasks[1].value
 
+    yt_data_extract.update_with_new_urls(info, player_response)
+
+    # Age restricted video, retry
+    if info['age_restricted'] or info['player_urls_missing']:
+        if info['age_restricted']:
+            print('Age restricted video, retrying')
+        else:
+            print('Player urls missing, retrying')
+        player_response = fetch_player_response('tv_embedded', video_id)
         yt_data_extract.update_with_new_urls(info, player_response)
 
-        # Age restricted video, retry
-        if info['age_restricted'] or info['player_urls_missing']:
-            if info['age_restricted']:
-                print('Age restricted video, retrying')
-            else:
-                print('Player urls missing, retrying')
-            player_response = fetch_player_response('tv_embedded', video_id)
-            yt_data_extract.update_with_new_urls(info, player_response)
+    # signature decryption
+    decryption_error = decrypt_signatures(info, video_id)
+    if decryption_error:
+        decryption_error = 'Error decrypting url signatures: ' + decryption_error
+        info['playability_error'] = decryption_error
 
-        # signature decryption
-        decryption_error = decrypt_signatures(info, video_id)
-        if decryption_error:
-            decryption_error = 'Error decrypting url signatures: ' + decryption_error
-            info['playability_error'] = decryption_error
+    # check if urls ready (non-live format) in former livestream
+    # urls not ready if all of them have no filesize
+    if info['was_live']:
+        info['urls_ready'] = False
+        for fmt in info['formats']:
+            if fmt['file_size'] is not None:
+                info['urls_ready'] = True
+    else:
+        info['urls_ready'] = True
 
-        # check if urls ready (non-live format) in former livestream
-        # urls not ready if all of them have no filesize
-        if info['was_live']:
-            info['urls_ready'] = False
+    # livestream urls
+    # sometimes only the livestream urls work soon after the livestream is over
+    if (info['hls_manifest_url']
+        and (info['live'] or not info['formats'] or not info['urls_ready'])
+    ):
+        manifest = util.fetch_url(info['hls_manifest_url'],
+            debug_name='hls_manifest.m3u8',
+            report_text='Fetched hls manifest'
+        ).decode('utf-8')
+
+        info['hls_formats'], err = yt_data_extract.extract_hls_formats(manifest)
+        if not err:
+            info['playability_error'] = None
+        for fmt in info['hls_formats']:
+            fmt['video_quality'] = video_quality_string(fmt)
+    else:
+        info['hls_formats'] = []
+
+    # check for 403. Unnecessary for tor video routing b/c ip address is same
+    info['invidious_used'] = False
+    info['invidious_reload_button'] = False
+    info['tor_bypass_used'] = False
+    if (settings.route_tor == 1
+            and info['formats'] and info['formats'][0]['url']):
+        try:
+            response = util.head(info['formats'][0]['url'],
+                                 report_text='Checked for URL access')
+        except urllib3.exceptions.HTTPError:
+            print('Error while checking for URL access:\n')
+            traceback.print_exc()
+            return info
+
+        if response.status == 403:
+            print('Access denied (403) for video urls.')
+            print('Routing video through Tor')
+            info['tor_bypass_used'] = True
             for fmt in info['formats']:
-                if fmt['file_size'] is not None:
-                    info['urls_ready'] = True
-        else:
-            info['urls_ready'] = True
-
-        # livestream urls
-        # sometimes only the livestream urls work soon after the livestream is over
-        if (info['hls_manifest_url']
-            and (info['live'] or not info['formats'] or not info['urls_ready'])
-        ):
-            manifest = util.fetch_url(info['hls_manifest_url'],
-                debug_name='hls_manifest.m3u8',
-                report_text='Fetched hls manifest'
-            ).decode('utf-8')
-
-            info['hls_formats'], err = yt_data_extract.extract_hls_formats(manifest)
-            if not err:
-                info['playability_error'] = None
-            for fmt in info['hls_formats']:
-                fmt['video_quality'] = video_quality_string(fmt)
-        else:
-            info['hls_formats'] = []
-
-        # check for 403. Unnecessary for tor video routing b/c ip address is same
-        info['invidious_used'] = False
-        info['invidious_reload_button'] = False
-        info['tor_bypass_used'] = False
-        if (settings.route_tor == 1
-                and info['formats'] and info['formats'][0]['url']):
-            try:
-                response = util.head(info['formats'][0]['url'],
-                                     report_text='Checked for URL access')
-            except urllib3.exceptions.HTTPError:
-                print('Error while checking for URL access:\n')
-                traceback.print_exc()
-                return info
-
-            if response.status == 403:
-                print('Access denied (403) for video urls.')
-                print('Routing video through Tor')
-                info['tor_bypass_used'] = True
-                for fmt in info['formats']:
-                    fmt['url'] += '&use_tor=1'
-            elif 300 <= response.status < 400:
-                print('Error: exceeded max redirects while checking video URL')
-        return info
+                fmt['url'] += '&use_tor=1'
+        elif 300 <= response.status < 400:
+            print('Error: exceeded max redirects while checking video URL')
+    return info
 
 
 def video_quality_string(format):