heckyel/yt-local
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Subscriptions: Order videos by the time they were added to db

Browse Source
This commit is contained in:
James Taylor 2019-09-02 19:10:38 -07:00
parent 71632a23f5
commit 0a590c3364
1 changed files with 58 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
youtube/subscriptions.py
View File

@ -39,8 +39,8 @@ def open_database():
id integer PRIMARY KEY, id integer PRIMARY KEY,
yt_channel_id text UNIQUE NOT NULL, yt_channel_id text UNIQUE NOT NULL,
channel_name text NOT NULL, channel_name text NOT NULL,
time_last_checked integer, time_last_checked integer DEFAULT 0,
next_check_time integer, next_check_time integer DEFAULT 0,
muted integer DEFAULT 0 muted integer DEFAULT 0
)''') )''')
cursor.execute('''CREATE TABLE IF NOT EXISTS videos ( cursor.execute('''CREATE TABLE IF NOT EXISTS videos (
@ -51,6 +51,7 @@ def open_database():
duration text, duration text,
time_published integer NOT NULL, time_published integer NOT NULL,
is_time_published_exact integer DEFAULT 0, is_time_published_exact integer DEFAULT 0,
time_noticed integer NOT NULL,
description text, description text,
watched integer default 0 watched integer default 0
)''') )''')
@ -96,11 +97,10 @@ def is_subscribed(channel_id):
def _subscribe(cursor, channels): def _subscribe(cursor, channels):
''' channels is a list of (channel_id, channel_name) ''' ''' channels is a list of (channel_id, channel_name) '''
# set time_last_checked to 0 on all channels being subscribed to channels = ( (channel_id, channel_name, 0, 0) for channel_id, channel_name in channels)
channels = ( (channel_id, channel_name, 0) for channel_id, channel_name in channels)
cursor.executemany('''INSERT OR IGNORE INTO subscribed_channels (yt_channel_id, channel_name, time_last_checked) cursor.executemany('''INSERT OR IGNORE INTO subscribed_channels (yt_channel_id, channel_name, time_last_checked, next_check_time)
VALUES (?, ?, ?)''', channels) VALUES (?, ?, ?, ?)''', channels)
def delete_thumbnails(to_delete): def delete_thumbnails(to_delete):
@ -146,14 +146,14 @@ def _get_videos(cursor, number_per_page, offset, tag = None):
INNER JOIN subscribed_channels on videos.sql_channel_id = subscribed_channels.id INNER JOIN subscribed_channels on videos.sql_channel_id = subscribed_channels.id
INNER JOIN tag_associations on videos.sql_channel_id = tag_associations.sql_channel_id INNER JOIN tag_associations on videos.sql_channel_id = tag_associations.sql_channel_id
WHERE tag = ? AND muted = 0 WHERE tag = ? AND muted = 0
ORDER BY time_published DESC ORDER BY time_noticed DESC, time_published DESC
LIMIT ? OFFSET ?''', (tag, number_per_page*9, offset)).fetchall() LIMIT ? OFFSET ?''', (tag, number_per_page*9, offset)).fetchall()
else: else:
db_videos = cursor.execute('''SELECT video_id, title, duration, time_published, is_time_published_exact, channel_name db_videos = cursor.execute('''SELECT video_id, title, duration, time_published, is_time_published_exact, channel_name
FROM videos FROM videos
INNER JOIN subscribed_channels on videos.sql_channel_id = subscribed_channels.id INNER JOIN subscribed_channels on videos.sql_channel_id = subscribed_channels.id
WHERE muted = 0 WHERE muted = 0
ORDER BY time_published DESC ORDER BY time_noticed DESC, time_published DESC
LIMIT ? OFFSET ?''', (number_per_page*9, offset)).fetchall() LIMIT ? OFFSET ?''', (number_per_page*9, offset)).fetchall()
pseudo_number_of_videos = offset + len(db_videos) pseudo_number_of_videos = offset + len(db_videos)
@ -328,15 +328,20 @@ if settings.autocheck_subscriptions:
with open_database() as connection: with open_database() as connection:
with connection as cursor: with connection as cursor:
now = time.time() now = time.time()
for row in cursor.execute('''SELECT yt_channel_id, channel_name, next_check_time FROM subscribed_channels WHERE next_check_time IS NOT NULL AND muted != 1''').fetchall(): for row in cursor.execute('''SELECT yt_channel_id, channel_name, next_check_time FROM subscribed_channels WHERE muted != 1''').fetchall():
# expired, check randomly within the 30 minutes if row[2] is None:
next_check_time = 0
else:
next_check_time = row[2]
# expired, check randomly within the next hour
# note: even if it isn't scheduled in the past right now, it might end up being if it's due soon and we dont start dispatching by then, see below where time_until_earliest_job is negative # note: even if it isn't scheduled in the past right now, it might end up being if it's due soon and we dont start dispatching by then, see below where time_until_earliest_job is negative
if row[2] < now: if next_check_time < now:
next_check_time = now + 3600*secrets.randbelow(60)/60 next_check_time = now + 3600*secrets.randbelow(60)/60
row = (row[0], row[1], next_check_time) row = (row[0], row[1], next_check_time)
_schedule_checking(cursor, row[0], next_check_time) _schedule_checking(cursor, row[0], next_check_time)
autocheck_jobs.append({'channel_id': row[0], 'channel_name': row[1], 'next_check_time': row[2]}) autocheck_jobs.append({'channel_id': row[0], 'channel_name': row[1], 'next_check_time': next_check_time})
@ -400,8 +405,6 @@ def _get_upstream_videos(channel_id):
print("Checking channel: " + channel_status_name) print("Checking channel: " + channel_status_name)
videos = []
tasks = ( tasks = (
gevent.spawn(channel.get_channel_tab, channel_id, print_status=False), # channel page, need for video duration gevent.spawn(channel.get_channel_tab, channel_id, print_status=False), # channel page, need for video duration
gevent.spawn(util.fetch_url, 'https://www.youtube.com/feeds/videos.xml?channel_id=' + channel_id) # atoma feed, need for exact published time gevent.spawn(util.fetch_url, 'https://www.youtube.com/feeds/videos.xml?channel_id=' + channel_id) # atoma feed, need for exact published time
@ -444,22 +447,31 @@ def _get_upstream_videos(channel_id):
print('Failed to read atoma feed for ' + channel_status_name) print('Failed to read atoma feed for ' + channel_status_name)
traceback.print_exc() traceback.print_exc()
with open_database() as connection:
with connection as cursor:
is_first_check = cursor.execute('''SELECT time_last_checked FROM subscribed_channels WHERE yt_channel_id=?''', [channel_id]).fetchone()[0] in (None, 0)
video_add_time = int(time.time())
videos = []
channel_videos = channel.extract_info(json.loads(channel_tab), 'videos')['items'] channel_videos = channel.extract_info(json.loads(channel_tab), 'videos')['items']
for i, video_item in enumerate(channel_videos): for i, video_item in enumerate(channel_videos):
if 'description' not in video_item: if 'description' not in video_item:
video_item['description'] = '' video_item['description'] = ''
if video_item['id'] in times_published: if video_item['id'] in times_published:
video_item['time_published'] = times_published[video_item['id']] time_published = times_published[video_item['id']]
video_item['is_time_published_exact'] = True is_time_published_exact = True
else: else:
video_item['is_time_published_exact'] = False is_time_published_exact = False
try: try:
video_item['time_published'] = youtube_timestamp_to_posix(video_item['published']) - i # subtract a few seconds off the videos so they will be in the right order time_published = youtube_timestamp_to_posix(video_item['published']) - i # subtract a few seconds off the videos so they will be in the right order
except KeyError: except KeyError:
print(video_item) print(video_item)
videos.append((channel_id, video_item['id'], video_item['title'], video_item['duration'], video_item['time_published'], video_item['is_time_published_exact'], video_item['description'])) if is_first_check:
time_noticed = time_published # don't want a crazy ordering on first check, since we're ordering by time_noticed
else:
time_noticed = video_add_time
videos.append((channel_id, video_item['id'], video_item['title'], video_item['duration'], time_published, is_time_published_exact, time_noticed, video_item['description']))
if len(videos) == 0: if len(videos) == 0:
@ -476,8 +488,6 @@ def _get_upstream_videos(channel_id):
next_check_delay = randomized_upload_period/10 # check at 10x the channel posting rate. might want to fine tune this number next_check_delay = randomized_upload_period/10 # check at 10x the channel posting rate. might want to fine tune this number
next_check_time = int(time.time() + next_check_delay) next_check_time = int(time.time() + next_check_delay)
with open_database() as connection:
with connection as cursor:
# calculate how many new videos there are # calculate how many new videos there are
row = cursor.execute('''SELECT video_id row = cursor.execute('''SELECT video_id
FROM videos FROM videos
@ -496,8 +506,8 @@ def _get_upstream_videos(channel_id):
index += 1 index += 1
number_of_new_videos = index number_of_new_videos = index
cursor.executemany('''INSERT OR IGNORE INTO videos (sql_channel_id, video_id, title, duration, time_published, is_time_published_exact, description) cursor.executemany('''INSERT OR IGNORE INTO videos (sql_channel_id, video_id, title, duration, time_published, is_time_published_exact, time_noticed, description)
VALUES ((SELECT id FROM subscribed_channels WHERE yt_channel_id=?), ?, ?, ?, ?, ?, ?)''', videos) VALUES ((SELECT id FROM subscribed_channels WHERE yt_channel_id=?), ?, ?, ?, ?, ?, ?, ?)''', videos)
cursor.execute('''UPDATE subscribed_channels cursor.execute('''UPDATE subscribed_channels
SET time_last_checked = ?, next_check_time = ? SET time_last_checked = ?, next_check_time = ?
WHERE yt_channel_id=?''', [int(time.time()), next_check_time, channel_id]) WHERE yt_channel_id=?''', [int(time.time()), next_check_time, channel_id])