WIP hack in es as the provider for search results

real sketch. lots of stuff is still broken. But! you can
make elasticsearch q= style queries and it shows up properly.
only first page works; need to adapt pager to elasticsearch's "total-hits" thing.

import_to_es.py

@@ -35,6 +35,10 @@ def mk_es(t):
         "_type": "torrent",
         "_index": "nyaav2",
         "_source": {
+            # we're also indexing the id as a number so you can
+            # order by it. seems like this is just equivalent to
+            # order by created_time, but oh well
+            "id": t.id,
             "display_name": t.display_name,
             "created_time": t.created_time,
             "updated_time": t.updated_time,

nyaa/routes.py

@@ -27,6 +27,11 @@ from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 from email.utils import formatdate
 
+from elasticsearch import Elasticsearch
+from elasticsearch_dsl import Search, Q
+
+es_client = Elasticsearch()
+
 DEBUG_API = False
 
 
@@ -67,6 +72,16 @@ def search(term='', user=None, sort='id', order='desc', category='0_0', quality_
     sort_ = sort.lower()
     if sort_ not in sort_keys:
         flask.abort(400)
+
+    # XXX gross why are all the names subtly different
+    es_sort = ({
+        'id': 'id',
+        'size': 'filesize',
+        'name': 'display_name',
+        'seeders': 'seed_count',
+        'leechers': 'leech_count',
+        'downloads': 'download_count'
+    })[sort]
     sort = sort_keys[sort]
 
     order_keys = {
@@ -78,6 +93,10 @@ def search(term='', user=None, sort='id', order='desc', category='0_0', quality_
     if order_ not in order_keys:
         flask.abort(400)
 
+    # funky, es sort is default asc, prefixed by '-' if desc
+    if "desc" == order:
+        es_sort = "-" + es_sort
+
     filter_keys = {
         '0': None,
         '1': (models.TorrentFlags.REMAKE, False),
@@ -126,28 +145,37 @@ def search(term='', user=None, sort='id', order='desc', category='0_0', quality_
     if flask.g.user:
         same_user = flask.g.user.id == user
 
+    s = Search(using=es_client, index='nyaav2')
     if term:
         query = db.session.query(models.TorrentNameSearch)
+        s = s.query("query_string", default_field="display_name", default_operator="AND", query=term)
     else:
         query = models.Torrent.query
 
     # Filter by user
     if user:
+        s = s.filter("term", uploader_id=user)
+
         query = query.filter(models.Torrent.uploader_id == user)
         # If admin, show everything
         if not admin:
             # If user is not logged in or the accessed feed doesn't belong to user,
             # hide anonymous torrents belonging to the queried user
             if not same_user:
+                # TODO adapt to es syntax
                 query = query.filter(models.Torrent.flags.op('&')(
                     int(models.TorrentFlags.ANONYMOUS | models.TorrentFlags.DELETED)).is_(False))
 
     if main_category:
+        s = s.filter("term", main_category_id=main_cat_id)
         query = query.filter(models.Torrent.main_category_id == main_cat_id)
     elif sub_category:
+        s = s.filter("term", main_category_id=main_cat_id)
+        s = s.filter("term", sub_category_id=sub_cat_id)
         query = query.filter((models.Torrent.main_category_id == main_cat_id) &
                              (models.Torrent.sub_category_id == sub_cat_id))
 
+    # TODO i dunno what this means in es
     if filter_tuple:
         query = query.filter(models.Torrent.flags.op('&')(int(filter_tuple[0])).is_(filter_tuple[1]))
 
@@ -157,6 +185,7 @@ def search(term='', user=None, sort='id', order='desc', category='0_0', quality_
             int(models.TorrentFlags.HIDDEN | models.TorrentFlags.DELETED)).is_(False))
 
     if term:
+        # note already handled in es
         for item in shlex.split(term, posix=False):
             if len(item) >= 2:
                 query = query.filter(FullTextSearch(
@@ -166,14 +195,22 @@ def search(term='', user=None, sort='id', order='desc', category='0_0', quality_
     if sort.class_ != models.Torrent:
         query = query.join(sort.class_)
 
+    s = s.sort(es_sort)
     query = query.order_by(getattr(sort, order)())
 
+    per = app.config['RESULTS_PER_PAGE']
     if rss:
-        query = query.limit(app.config['RESULTS_PER_PAGE'])
+        pass
+        #query = query.limit(app.config['RESULTS_PER_PAGE'])
     else:
-        query = query.paginate_faste(page, per_page=app.config['RESULTS_PER_PAGE'], step=5)
+        # page is 1-based?
+        s = s[(page-1)*per:page*per]
+        #query = query.paginate_faste(page, per_page=app.config['RESULTS_PER_PAGE'], step=5)
 
-    return query
+    #return query
+    from pprint import pprint
+    print(json.dumps(s.to_dict()))
+    return s.execute()
 
 
 @app.errorhandler(404)
@@ -445,6 +482,7 @@ def activate_user(payload):
 
     user.status = models.UserStatusType.ACTIVE
 
+
     db.session.add(user)
     db.session.commit()
 

nyaa/templates/search_results.html

@@ -8,7 +8,7 @@
 	{{ caller() }}
 </th>
 {% endmacro %}
-{% if torrent_query.items %}
+{% if torrent_query.hits.total > 0 %}
 <div class="table-responsive">
 	<table class="table table-bordered table-hover table-striped torrent-list">
 		<thead>
@@ -45,26 +45,26 @@
 			</tr>
 		</thead>
 		<tbody>
-			{% for torrent in torrent_query.items %}
+			{% for torrent in torrent_query %}
 			<tr class="{% if torrent.deleted %}deleted{% elif torrent.hidden %}warning{% elif torrent.remake %}danger{% elif torrent.trusted %}success{% else %}default{% endif %}">
-				{% set cat_id = (torrent.main_category.id|string) + '_' + (torrent.sub_category.id|string) %}
+				{% set cat_id = (torrent.main_category_id|string) + '_' + (torrent.sub_category_id|string) %}
 				{% set icon_dir = config.SITE_FLAVOR %}
 				<td style="padding:0 4px;">
-				<a href="/?c={{ cat_id }}" title="{{ torrent.main_category.name }} - {{ torrent.sub_category.name }}">
+				<a href="/?c={{ cat_id }}" title="{{ torrent.main_category_id }} - {{ torrent.sub_category_id }}">
 					<img src="/static/img/icons/{{ icon_dir }}/{{ cat_id }}.png">
 				</a>
 				</td>
-				<td><a href="{{ url_for('view_torrent', torrent_id=torrent.id) }}">{{ torrent.display_name | escape }}</a></td>
+				<td><a href="{{ url_for('view_torrent', torrent_id=torrent.meta.id) }}">{{ torrent.display_name | escape }}</a></td>
 				<td style="white-space: nowrap;text-align: center;">
 					{% if torrent.has_torrent %}<a href="{{ url_for('download_torrent', torrent_id=torrent.id) }}"><i class="fa fa-fw fa-download"></i></a>{% endif %}
 					<a href="{{ torrent.magnet_uri }}"><i class="fa fa-fw fa-magnet"></i></a>
 				</td>
 				<td class="text-center">{{ torrent.filesize | filesizeformat(True) }}</td>
-				<td class="text-center" data-timestamp="{{ torrent.created_utc_timestamp|int }}">{{ torrent.created_time.strftime('%Y-%m-%d %H:%M') }}</td>
+                <td class="text-center" {#data-timestamp="{{ torrent.created_time|int }}"#}>{{ torrent.created_time }}</td>
 				{% if config.ENABLE_SHOW_STATS %}
-				<td class="text-center" style="color: green;">{{ torrent.stats.seed_count }}</td>
+				<td class="text-center" style="color: green;">{{ torrent.seed_count }}</td>
-				<td class="text-center" style="color: red;">{{ torrent.stats.leech_count }}</td>
+				<td class="text-center" style="color: red;">{{ torrent.leech_count }}</td>
-				<td class="text-center">{{ torrent.stats.download_count }}</td>
+				<td class="text-center">{{ torrent.download_count }}</td>
 				{% endif %}
 			</tr>
 			{% endfor %}
@@ -75,7 +75,9 @@
 <h3>No results found</h3>
 {% endif %}
 
+{#
 <center>
 	{% from "bootstrap/pagination.html" import render_pagination %}
 	{{ render_pagination(torrent_query) }}
 </center>
+#}