Refactor search page

This commit is contained in:
James Taylor 2019-06-21 21:41:41 -07:00
parent e33bae2e50
commit 1156b09987
7 changed files with 364 additions and 86 deletions

View File

@ -6,7 +6,7 @@ from youtube import yt_app
from youtube import util
# these are just so the files get run - they import yt_app and add routes to it
from youtube import watch
from youtube import watch, search
import settings

View File

@ -1,16 +1,12 @@
from youtube import util, html_common, yt_data_extract, proto
from youtube import util, yt_data_extract, proto, local_playlist
from youtube import yt_app
import json
import urllib
import html
from string import Template
import base64
from math import ceil
with open("yt_search_results_template.html", "r") as file:
yt_search_results_template = file.read()
from flask import request
import flask
# Sort: 1
# Upload date: 2
@ -58,41 +54,32 @@ def get_search_json(query, page, autocorrect, sort, filters):
content = util.fetch_url(url, headers=headers, report_text="Got search results")
info = json.loads(content)
return info
showing_results_for = Template('''
<div>Showing results for <a>$corrected_query</a></div>
<div>Search instead for <a href="$original_query_url">$original_query</a></div>
''')
did_you_mean = Template('''
<div>Did you mean <a href="$corrected_query_url">$corrected_query</a></div>
''')
def get_search_page(env, start_response):
start_response('200 OK', [('Content-type','text/html'),])
parameters = env['parameters']
if len(parameters) == 0:
return html_common.yt_basic_template.substitute(
page_title = "Search",
header = html_common.get_header(),
style = '',
page = '',
).encode('utf-8')
query = parameters["query"][0]
page = parameters.get("page", "1")[0]
autocorrect = int(parameters.get("autocorrect", "1")[0])
sort = int(parameters.get("sort", "0")[0])
@yt_app.route('/search')
def get_search_page():
if len(request.args) == 0:
return flask.render_template('base.html', title="Search")
if 'query' not in request.args:
abort(400)
query = request.args.get("query")
page = request.args.get("page", "1")
autocorrect = int(request.args.get("autocorrect", "1"))
sort = int(request.args.get("sort", "0"))
filters = {}
filters['time'] = int(parameters.get("time", "0")[0])
filters['type'] = int(parameters.get("type", "0")[0])
filters['duration'] = int(parameters.get("duration", "0")[0])
filters['time'] = int(request.args.get("time", "0"))
filters['type'] = int(request.args.get("type", "0"))
filters['duration'] = int(request.args.get("duration", "0"))
info = get_search_json(query, page, autocorrect, sort, filters)
estimated_results = int(info[1]['response']['estimatedResults'])
estimated_pages = ceil(estimated_results/20)
results = info[1]['response']['contents']['twoColumnSearchResultsRenderer']['primaryContents']['sectionListRenderer']['contents'][0]['itemSectionRenderer']['contents']
corrections = ''
result_list_html = ""
parsed_results = []
corrections = {'type': None}
for renderer in results:
type = list(renderer.keys())[0]
if type == 'shelfRenderer':
@ -102,41 +89,39 @@ def get_search_page(env, start_response):
corrected_query_string = parameters.copy()
corrected_query_string['query'] = [renderer['correctedQueryEndpoint']['searchEndpoint']['query']]
corrected_query_url = util.URL_ORIGIN + '/search?' + urllib.parse.urlencode(corrected_query_string, doseq=True)
corrections = did_you_mean.substitute(
corrected_query_url = corrected_query_url,
corrected_query = yt_data_extract.format_text_runs(renderer['correctedQuery']['runs']),
)
corrections = {
'type': 'did_you_mean',
'corrected_query': yt_data_extract.format_text_runs(renderer['correctedQuery']['runs']),
'corrected_query_url': corrected_query_url,
}
continue
if type == 'showingResultsForRenderer':
renderer = renderer[type]
no_autocorrect_query_string = parameters.copy()
no_autocorrect_query_string['autocorrect'] = ['0']
no_autocorrect_query_url = util.URL_ORIGIN + '/search?' + urllib.parse.urlencode(no_autocorrect_query_string, doseq=True)
corrections = showing_results_for.substitute(
corrected_query = yt_data_extract.format_text_runs(renderer['correctedQuery']['runs']),
original_query_url = no_autocorrect_query_url,
original_query = html.escape(renderer['originalQuery']['simpleText']),
)
corrections = {
'type': 'showing_results_for',
'corrected_query': yt_data_extract.format_text_runs(renderer['correctedQuery']['runs']),
'original_query_url': no_autocorrect_query_url,
'original_query': renderer['originalQuery']['simpleText'],
}
continue
result_list_html += html_common.renderer_html(renderer, current_query_string=env['QUERY_STRING'])
page = int(page)
if page <= 5:
page_start = 1
page_end = min(9, estimated_pages)
else:
page_start = page - 4
page_end = min(page + 4, estimated_pages)
result = Template(yt_search_results_template).substitute(
header = html_common.get_header(query),
results = result_list_html,
page_title = query + " - Search",
search_box_value = html.escape(query),
number_of_results = '{:,}'.format(estimated_results),
number_of_pages = '{:,}'.format(estimated_pages),
page_buttons = html_common.page_buttons_html(page, estimated_pages, util.URL_ORIGIN + "/search", env['QUERY_STRING']),
corrections = corrections
)
return result.encode('utf-8')
info = yt_data_extract.parse_info_prepare_for_html(renderer)
if info['type'] != 'unsupported':
parsed_results.append(info)
return flask.render_template('search.html',
header_playlist_names = local_playlist.get_playlist_names(),
query = query,
estimated_results = estimated_results,
estimated_pages = estimated_pages,
corrections = corrections,
results = parsed_results,
parameters_dictionary = request.args,
)

View File

@ -219,6 +219,12 @@ address{
max-height:2.4em;
overflow:hidden;
}
.medium-item .stats > *::after{
content: " | ";
}
.medium-item .stats > *:last-child::after{
content: "";
}
.medium-item .description{
grid-column: 2 / span 2;

View File

@ -2,13 +2,14 @@
<html>
<head>
<meta charset="utf-8">
<title>{% block page_title %}{% endblock %}</title>
<title>{% block page_title %}{{ title }}{% endblock %}</title>
<link href="/youtube.com/static/shared.css" type="text/css" rel="stylesheet">
<link href="/youtube.com/static/comments.css" type="text/css" rel="stylesheet">
<link href="/youtube.com/static/favicon.ico" type="image/x-icon" rel="icon">
<link title="Youtube local" href="/youtube.com/opensearch.xml" rel="search" type="application/opensearchdescription+xml">
<style type="text/css">
{% block style %}
{{ style }}
{% endblock %}
</style>
</head>
@ -105,6 +106,7 @@
</header>
<main>
{% block main %}
{{ main }}
{% endblock %}
</main>
</body>

View File

@ -0,0 +1,152 @@
{% macro text_runs(runs) %}
{%- if runs[0] is mapping -%}
{%- for text_run in runs -%}
{%- if text_run.get("bold", false) -%}
<b>{{ text_run["text"] }}</b>
{%- elif text_run.get('italics', false) -%}
<i>{{ text_run["text"] }}</i>
{%- else -%}
{{ text_run["text"] }}
{%- endif -%}
{%- endfor -%}
{%- else -%}
{{ runs }}
{%- endif -%}
{% endmacro %}
{% macro small_item(info) %}
<div class="small-item-box">
<div class="small-item">
{% if info['type'] == 'video' %}
<a class="video-thumbnail-box" href="{{ info['url'] }}" title="{{ info['title'] }}">
<img class="video-thumbnail-img" src="{{ info['thumbnail'] }}">
<span class="video-duration">{{ info['duration'] }}</span>
</a>
<a class="title" href="{{ info['url'] }}" title="{{ info['title'] }}">{{ info['title'] }}</a>
<address>{{ info['author'] }}</address>
<span class="views">{{ info['views'] }}</span>
{% elif info['type'] == 'playlist' %}
<a class="playlist-thumbnail-box" href="{{ info['url'] }}" title="{{ info['title'] }}">
<img class="playlist-thumbnail-img" src="{{ info['thumbnail'] }}">
<div class="playlist-thumbnail-info">
<span>{{ info['size'] }}</span>
</div>
</a>
<a class="title" href="{{ info['url'] }}" title="{{ info['title'] }}">{{ info['title'] }}</a>
<address>{{ info['author'] }}</address>
{% else %}
Error: unsupported item type
{% endif %}
</div>
{% if info['type'] == 'video' %}
<input class="item-checkbox" type="checkbox" name="video_info_list" value="{{ info['video_info'] }}" form="playlist-edit">
{% endif %}
</div>
{% endmacro %}
{% macro get_stats(info) %}
{% if 'author_url' is in(info) %}
<address>By <a href="{{ info['author_url'] }}">{{ info['author'] }}</a></address>
{% else %}
<address><b>{{ info['author'] }}</b></address>
{% endif %}
{% if 'views' is in(info) %}
<span class="views">{{ info['views'] }}</span>
{% endif %}
{% if 'published' is in(info) %}
<time>{{ info['published'] }}</time>
{% endif %}
{% endmacro %}
{% macro medium_item(info) %}
<div class="medium-item-box">
<div class="medium-item">
{% if info['type'] == 'video' %}
<a class="video-thumbnail-box" href="{{ info['url'] }}" title="{{ info['title'] }}">
<img class="video-thumbnail-img" src="{{ info['thumbnail'] }}">
<span class="video-duration">{{ info['duration'] }}</span>
</a>
<a class="title" href="{{ info['url'] }}" title="{{ info['title'] }}">{{ info['title'] }}</a>
<div class="stats">
{{ get_stats(info) }}
</div>
<span class="description">{{ text_runs(info['description']) }}</span>
<span class="badges">{{ info['badges']|join(' | ') }}</span>
{% elif info['type'] == 'playlist' %}
<a class="playlist-thumbnail-box" href="{{ info['url'] }}" title="{{ info['title'] }}">
<img class="playlist-thumbnail-img" src="{{ info['thumbnail'] }}">
<div class="playlist-thumbnail-info">
<span>{{ info['size'] }}</span>
</div>
</a>
<a class="title" href="{{ info['url'] }}" title="{{ info['title'] }}">{{ info['title'] }}</a>
<div class="stats">
{{ get_stats(info) }}
</div>
{% elif info['type'] == 'channel' %}
<a class="video-thumbnail-box" href="{{ info['url'] }}" title="{{ info['title'] }}">
<img class="video-thumbnail-img" src="{{ info['thumbnail'] }}">
</a>
<a class="title" href="{{ info['url'] }}">{{ info['title'] }}</a>
<span>{{ info['subscriber_count'] }}</span>
<span>{{ info['size'] }}</span>
<span class="description">{{ text_runs(info['description']) }}</span>
{% else %}
Error: unsupported item type
{% endif %}
</div>
{% if info['type'] == 'video' %}
<input class="item-checkbox" type="checkbox" name="video_info_list" value="{{ info['video_info'] }}" form="playlist-edit">
{% endif %}
</div>
{% endmacro %}
{% macro item(info) %}
{% if info['item_size'] == 'small' %}
{{ small_item(info) }}
{% elif info['item_size'] == 'medium' %}
{{ medium_item(info) }}
{% else %}
Error: Unknown item size
{% endif %}
{% endmacro %}
{% macro page_buttons(estimated_pages, url, parameters_dictionary) %}
{% set current_page = parameters_dictionary.get('page', 1)|int %}
{% set parameters_dictionary = parameters_dictionary.to_dict() %}
{% if current_page is le(5) %}
{% set page_start = 1 %}
{% set page_end = [9, estimated_pages]|min %}
{% else %}
{% set page_start = current_page - 4 %}
{% set page_end = [current_page + 4, estimated_pages]|min %}
{% endif %}
{% for page in range(page_start, page_end+1) %}
{% if page == current_page %}
<div class="page-button">{{ page }}</div>
{% else %}
{# IMPORTANT: Jinja SUCKS #}
{# https://stackoverflow.com/questions/36886650/how-to-add-a-new-entry-into-a-dictionary-object-while-using-jinja2 #}
{% set _ = parameters_dictionary.__setitem__('page', page) %}
<a class="page-button" href="{{ url + '?' + parameters_dictionary|urlencode }}">{{ page }}</a>
{% endif %}
{% endfor %}
{% endmacro %}

View File

@ -0,0 +1,54 @@
{% set search_box_value = query %}
{% extends "base.html" %}
{% block page_title %}{{ query + ' - Search' }}{% endblock %}
{% import "common_elements.html" as common_elements %}
{% block style %}
main{
display:grid;
grid-template-columns: minmax(0px, 1fr) 800px minmax(0px,2fr);
max-width:100vw;
}
#number-of-results{
font-weight:bold;
}
#result-info{
grid-row: 1;
grid-column:2;
align-self:center;
}
.page-button-row{
grid-column: 2;
justify-self: center;
}
.item-list{
grid-row: 2;
grid-column: 2;
}
.badge{
background-color:#cccccc;
}
{% endblock style %}
{% block main %}
<div id="result-info">
<div id="number-of-results">Approximately {{ '{:,}'.format(estimated_results) }} results ({{ '{:,}'.format(estimated_pages) }} pages)</div>
{% if corrections['type'] == 'showing_results_for' %}
<div>Showing results for <a>{{ corrections['corrected_query']|safe }}</a></div>
<div>Search instead for <a href="{{ corrections['original_query_url'] }}">{{ corrections['original_query'] }}</a></div>
{% elif corrections['type'] == 'did_you_mean' %}
<div>Did you mean <a href="{{ corrections['corrected_query_url'] }}">{{ corrections['corrected_query']|safe }}</a></div>
{% endif %}
</div>
<div class="item-list">
{% for info in results %}
{{ common_elements.item(info) }}
{% endfor %}
</div>
<nav class="page-button-row">
{{ common_elements.page_buttons(estimated_pages, '/https://www.youtube.com/search', parameters_dictionary) }}
</nav>
{% endblock main %}

View File

@ -1,4 +1,7 @@
from youtube import util
import html
import json
# videos (all of type str):
@ -138,9 +141,83 @@ dispatch = {
}
def renderer_info(renderer):
def ajax_info(item_json):
try:
info = {}
for key, node in item_json.items():
try:
simple_key, function = dispatch[key]
except KeyError:
continue
info[simple_key] = function(node)
return info
except KeyError:
print(item_json)
raise
def prefix_urls(item):
try:
item['thumbnail'] = '/' + item['thumbnail'].lstrip('/')
except KeyError:
pass
try:
item['author_url'] = util.URL_ORIGIN + item['author_url']
except KeyError:
pass
def add_extra_html_info(item):
if item['type'] == 'video':
item['url'] = util.URL_ORIGIN + '/watch?v=' + item['id']
video_info = {}
for key in ('id', 'title', 'author', 'duration'):
try:
video_info[key] = item[key]
except KeyError:
video_info[key] = ''
item['video_info'] = json.dumps(video_info)
elif item['type'] == 'playlist':
item['url'] = util.URL_ORIGIN + '/playlist?list=' + item['id']
elif item['type'] == 'channel':
item['url'] = util.URL_ORIGIN + "/channel/" + item['id']
def renderer_info(renderer, additional_info={}):
type = list(renderer.keys())[0]
renderer = renderer[type]
info = {}
if type == 'itemSectionRenderer':
return renderer_info(renderer['contents'][0], additional_info)
if type in ('movieRenderer', 'clarificationRenderer'):
info['type'] = 'unsupported'
return info
info.update(additional_info)
if type.startswith('compact'):
info['item_size'] = 'small'
else:
info['item_size'] = 'medium'
if type in ('compactVideoRenderer', 'videoRenderer', 'gridVideoRenderer'):
info['type'] = 'video'
elif type in ('playlistRenderer', 'compactPlaylistRenderer', 'gridPlaylistRenderer',
'radioRenderer', 'compactRadioRenderer', 'gridRadioRenderer',
'showRenderer', 'compactShowRenderer', 'gridShowRenderer'):
info['type'] = 'playlist'
elif type == 'channelRenderer':
info['type'] = 'channel'
else:
info['type'] = 'unsupported'
return info
try:
info = {}
if 'viewCountText' in renderer: # prefer this one as it contains all the digits
info['views'] = get_text(renderer['viewCountText'])
elif 'shortViewCountText' in renderer:
@ -183,23 +260,25 @@ def renderer_info(renderer):
except KeyError:
continue
info[simple_key] = function(node)
if info['type'] == 'video' and 'duration' not in info:
info['duration'] = 'Live'
return info
except KeyError:
print(renderer)
raise
def ajax_info(item_json):
try:
info = {}
for key, node in item_json.items():
try:
simple_key, function = dispatch[key]
except KeyError:
continue
info[simple_key] = function(node)
return info
except KeyError:
print(item_json)
raise
#print(renderer)
#raise NotImplementedError('Unknown renderer type: ' + type)
return ''
def parse_info_prepare_for_html(renderer):
item = renderer_info(renderer)
prefix_urls(item)
add_extra_html_info(item)
return item