From 3cbe2e4221e0c66105ef0d72221a148f78007d8c Mon Sep 17 00:00:00 2001 From: queue Date: Sun, 14 May 2017 02:01:26 -0600 Subject: [PATCH] WIP hack in es as the provider for search results real sketch. lots of stuff is still broken. But! you can make elasticsearch q= style queries and it shows up properly. only first page works; need to adapt pager to elasticsearch's "total-hits" thing. --- import_to_es.py | 4 +++ nyaa/routes.py | 44 ++++++++++++++++++++++++++++-- nyaa/templates/search_results.html | 20 ++++++++------ 3 files changed, 56 insertions(+), 12 deletions(-) diff --git a/import_to_es.py b/import_to_es.py index 4be5e2b..e714da5 100644 --- a/import_to_es.py +++ b/import_to_es.py @@ -35,6 +35,10 @@ def mk_es(t): "_type": "torrent", "_index": "nyaav2", "_source": { + # we're also indexing the id as a number so you can + # order by it. seems like this is just equivalent to + # order by created_time, but oh well + "id": t.id, "display_name": t.display_name, "created_time": t.created_time, "updated_time": t.updated_time, diff --git a/nyaa/routes.py b/nyaa/routes.py index 4064b15..2941e5b 100644 --- a/nyaa/routes.py +++ b/nyaa/routes.py @@ -27,6 +27,11 @@ from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.utils import formatdate +from elasticsearch import Elasticsearch +from elasticsearch_dsl import Search, Q + +es_client = Elasticsearch() + DEBUG_API = False @@ -67,6 +72,16 @@ def search(term='', user=None, sort='id', order='desc', category='0_0', quality_ sort_ = sort.lower() if sort_ not in sort_keys: flask.abort(400) + + # XXX gross why are all the names subtly different + es_sort = ({ + 'id': 'id', + 'size': 'filesize', + 'name': 'display_name', + 'seeders': 'seed_count', + 'leechers': 'leech_count', + 'downloads': 'download_count' + })[sort] sort = sort_keys[sort] order_keys = { @@ -78,6 +93,10 @@ def search(term='', user=None, sort='id', order='desc', category='0_0', quality_ if order_ not in order_keys: flask.abort(400) + # funky, es sort is default asc, prefixed by '-' if desc + if "desc" == order: + es_sort = "-" + es_sort + filter_keys = { '0': None, '1': (models.TorrentFlags.REMAKE, False), @@ -126,28 +145,37 @@ def search(term='', user=None, sort='id', order='desc', category='0_0', quality_ if flask.g.user: same_user = flask.g.user.id == user + s = Search(using=es_client, index='nyaav2') if term: query = db.session.query(models.TorrentNameSearch) + s = s.query("query_string", default_field="display_name", default_operator="AND", query=term) else: query = models.Torrent.query # Filter by user if user: + s = s.filter("term", uploader_id=user) + query = query.filter(models.Torrent.uploader_id == user) # If admin, show everything if not admin: # If user is not logged in or the accessed feed doesn't belong to user, # hide anonymous torrents belonging to the queried user if not same_user: + # TODO adapt to es syntax query = query.filter(models.Torrent.flags.op('&')( int(models.TorrentFlags.ANONYMOUS | models.TorrentFlags.DELETED)).is_(False)) if main_category: + s = s.filter("term", main_category_id=main_cat_id) query = query.filter(models.Torrent.main_category_id == main_cat_id) elif sub_category: + s = s.filter("term", main_category_id=main_cat_id) + s = s.filter("term", sub_category_id=sub_cat_id) query = query.filter((models.Torrent.main_category_id == main_cat_id) & (models.Torrent.sub_category_id == sub_cat_id)) + # TODO i dunno what this means in es if filter_tuple: query = query.filter(models.Torrent.flags.op('&')(int(filter_tuple[0])).is_(filter_tuple[1])) @@ -157,6 +185,7 @@ def search(term='', user=None, sort='id', order='desc', category='0_0', quality_ int(models.TorrentFlags.HIDDEN | models.TorrentFlags.DELETED)).is_(False)) if term: + # note already handled in es for item in shlex.split(term, posix=False): if len(item) >= 2: query = query.filter(FullTextSearch( @@ -166,14 +195,22 @@ def search(term='', user=None, sort='id', order='desc', category='0_0', quality_ if sort.class_ != models.Torrent: query = query.join(sort.class_) + s = s.sort(es_sort) query = query.order_by(getattr(sort, order)()) + per = app.config['RESULTS_PER_PAGE'] if rss: - query = query.limit(app.config['RESULTS_PER_PAGE']) + pass + #query = query.limit(app.config['RESULTS_PER_PAGE']) else: - query = query.paginate_faste(page, per_page=app.config['RESULTS_PER_PAGE'], step=5) + # page is 1-based? + s = s[(page-1)*per:page*per] + #query = query.paginate_faste(page, per_page=app.config['RESULTS_PER_PAGE'], step=5) - return query + #return query + from pprint import pprint + print(json.dumps(s.to_dict())) + return s.execute() @app.errorhandler(404) @@ -445,6 +482,7 @@ def activate_user(payload): user.status = models.UserStatusType.ACTIVE + db.session.add(user) db.session.commit() diff --git a/nyaa/templates/search_results.html b/nyaa/templates/search_results.html index 95d18d5..4bce368 100644 --- a/nyaa/templates/search_results.html +++ b/nyaa/templates/search_results.html @@ -8,7 +8,7 @@ {{ caller() }} {% endmacro %} -{% if torrent_query.items %} +{% if torrent_query.hits.total > 0 %}
@@ -45,26 +45,26 @@ - {% for torrent in torrent_query.items %} + {% for torrent in torrent_query %} - {% set cat_id = (torrent.main_category.id|string) + '_' + (torrent.sub_category.id|string) %} + {% set cat_id = (torrent.main_category_id|string) + '_' + (torrent.sub_category_id|string) %} {% set icon_dir = config.SITE_FLAVOR %} - + - + {% if config.ENABLE_SHOW_STATS %} - - - + + + {% endif %} {% endfor %} @@ -75,7 +75,9 @@

No results found

{% endif %} +{#
{% from "bootstrap/pagination.html" import render_pagination %} {{ render_pagination(torrent_query) }}
+#}
- + {{ torrent.display_name | escape }}{{ torrent.display_name | escape }} {% if torrent.has_torrent %}{% endif %} {{ torrent.filesize | filesizeformat(True) }}{{ torrent.created_time.strftime('%Y-%m-%d %H:%M') }}{{ torrent.created_time }}{{ torrent.stats.seed_count }}{{ torrent.stats.leech_count }}{{ torrent.stats.download_count }}{{ torrent.seed_count }}{{ torrent.leech_count }}{{ torrent.download_count }}