mirror of https://gitlab.com/SIGBUS/nyaa.git
WIP hack in es as the provider for search results
real sketch. lots of stuff is still broken. But! you can make elasticsearch q= style queries and it shows up properly. only first page works; need to adapt pager to elasticsearch's "total-hits" thing.
This commit is contained in:
queue
parent
d7d24ef49e
commit
3cbe2e4221
|
|
@ -35,6 +35,10 @@ def mk_es(t):
|
|||
"_type": "torrent",
|
||||
"_index": "nyaav2",
|
||||
"_source": {
|
||||
# we're also indexing the id as a number so you can
|
||||
# order by it. seems like this is just equivalent to
|
||||
# order by created_time, but oh well
|
||||
"id": t.id,
|
||||
"display_name": t.display_name,
|
||||
"created_time": t.created_time,
|
||||
"updated_time": t.updated_time,
|
||||
|
|
|
|||
|
|
@ -27,6 +27,11 @@ from email.mime.multipart import MIMEMultipart
|
|||
from email.mime.text import MIMEText
|
||||
from email.utils import formatdate
|
||||
|
||||
from elasticsearch import Elasticsearch
|
||||
from elasticsearch_dsl import Search, Q
|
||||
|
||||
es_client = Elasticsearch()
|
||||
|
||||
DEBUG_API = False
|
||||
|
||||
|
||||
|
|
@ -67,6 +72,16 @@ def search(term='', user=None, sort='id', order='desc', category='0_0', quality_
|
|||
sort_ = sort.lower()
|
||||
if sort_ not in sort_keys:
|
||||
flask.abort(400)
|
||||
|
||||
# XXX gross why are all the names subtly different
|
||||
es_sort = ({
|
||||
'id': 'id',
|
||||
'size': 'filesize',
|
||||
'name': 'display_name',
|
||||
'seeders': 'seed_count',
|
||||
'leechers': 'leech_count',
|
||||
'downloads': 'download_count'
|
||||
})[sort]
|
||||
sort = sort_keys[sort]
|
||||
|
||||
order_keys = {
|
||||
|
|
@ -78,6 +93,10 @@ def search(term='', user=None, sort='id', order='desc', category='0_0', quality_
|
|||
if order_ not in order_keys:
|
||||
flask.abort(400)
|
||||
|
||||
# funky, es sort is default asc, prefixed by '-' if desc
|
||||
if "desc" == order:
|
||||
es_sort = "-" + es_sort
|
||||
|
||||
filter_keys = {
|
||||
'0': None,
|
||||
'1': (models.TorrentFlags.REMAKE, False),
|
||||
|
|
@ -126,28 +145,37 @@ def search(term='', user=None, sort='id', order='desc', category='0_0', quality_
|
|||
if flask.g.user:
|
||||
same_user = flask.g.user.id == user
|
||||
|
||||
s = Search(using=es_client, index='nyaav2')
|
||||
if term:
|
||||
query = db.session.query(models.TorrentNameSearch)
|
||||
s = s.query("query_string", default_field="display_name", default_operator="AND", query=term)
|
||||
else:
|
||||
query = models.Torrent.query
|
||||
|
||||
# Filter by user
|
||||
if user:
|
||||
s = s.filter("term", uploader_id=user)
|
||||
|
||||
query = query.filter(models.Torrent.uploader_id == user)
|
||||
# If admin, show everything
|
||||
if not admin:
|
||||
# If user is not logged in or the accessed feed doesn't belong to user,
|
||||
# hide anonymous torrents belonging to the queried user
|
||||
if not same_user:
|
||||
# TODO adapt to es syntax
|
||||
query = query.filter(models.Torrent.flags.op('&')(
|
||||
int(models.TorrentFlags.ANONYMOUS | models.TorrentFlags.DELETED)).is_(False))
|
||||
|
||||
if main_category:
|
||||
s = s.filter("term", main_category_id=main_cat_id)
|
||||
query = query.filter(models.Torrent.main_category_id == main_cat_id)
|
||||
elif sub_category:
|
||||
s = s.filter("term", main_category_id=main_cat_id)
|
||||
s = s.filter("term", sub_category_id=sub_cat_id)
|
||||
query = query.filter((models.Torrent.main_category_id == main_cat_id) &
|
||||
(models.Torrent.sub_category_id == sub_cat_id))
|
||||
|
||||
# TODO i dunno what this means in es
|
||||
if filter_tuple:
|
||||
query = query.filter(models.Torrent.flags.op('&')(int(filter_tuple[0])).is_(filter_tuple[1]))
|
||||
|
||||
|
|
@ -157,6 +185,7 @@ def search(term='', user=None, sort='id', order='desc', category='0_0', quality_
|
|||
int(models.TorrentFlags.HIDDEN | models.TorrentFlags.DELETED)).is_(False))
|
||||
|
||||
if term:
|
||||
# note already handled in es
|
||||
for item in shlex.split(term, posix=False):
|
||||
if len(item) >= 2:
|
||||
query = query.filter(FullTextSearch(
|
||||
|
|
@ -166,14 +195,22 @@ def search(term='', user=None, sort='id', order='desc', category='0_0', quality_
|
|||
if sort.class_ != models.Torrent:
|
||||
query = query.join(sort.class_)
|
||||
|
||||
s = s.sort(es_sort)
|
||||
query = query.order_by(getattr(sort, order)())
|
||||
|
||||
per = app.config['RESULTS_PER_PAGE']
|
||||
if rss:
|
||||
query = query.limit(app.config['RESULTS_PER_PAGE'])
|
||||
pass
|
||||
#query = query.limit(app.config['RESULTS_PER_PAGE'])
|
||||
else:
|
||||
query = query.paginate_faste(page, per_page=app.config['RESULTS_PER_PAGE'], step=5)
|
||||
# page is 1-based?
|
||||
s = s[(page-1)*per:page*per]
|
||||
#query = query.paginate_faste(page, per_page=app.config['RESULTS_PER_PAGE'], step=5)
|
||||
|
||||
return query
|
||||
#return query
|
||||
from pprint import pprint
|
||||
print(json.dumps(s.to_dict()))
|
||||
return s.execute()
|
||||
|
||||
|
||||
@app.errorhandler(404)
|
||||
|
|
@ -445,6 +482,7 @@ def activate_user(payload):
|
|||
|
||||
user.status = models.UserStatusType.ACTIVE
|
||||
|
||||
|
||||
db.session.add(user)
|
||||
db.session.commit()
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
{{ caller() }}
|
||||
</th>
|
||||
{% endmacro %}
|
||||
{% if torrent_query.items %}
|
||||
{% if torrent_query.hits.total > 0 %}
|
||||
<div class="table-responsive">
|
||||
<table class="table table-bordered table-hover table-striped torrent-list">
|
||||
<thead>
|
||||
|
|
@ -45,26 +45,26 @@
|
|||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for torrent in torrent_query.items %}
|
||||
{% for torrent in torrent_query %}
|
||||
<tr class="{% if torrent.deleted %}deleted{% elif torrent.hidden %}warning{% elif torrent.remake %}danger{% elif torrent.trusted %}success{% else %}default{% endif %}">
|
||||
{% set cat_id = (torrent.main_category.id|string) + '_' + (torrent.sub_category.id|string) %}
|
||||
{% set cat_id = (torrent.main_category_id|string) + '_' + (torrent.sub_category_id|string) %}
|
||||
{% set icon_dir = config.SITE_FLAVOR %}
|
||||
<td style="padding:0 4px;">
|
||||
<a href="/?c={{ cat_id }}" title="{{ torrent.main_category.name }} - {{ torrent.sub_category.name }}">
|
||||
<a href="/?c={{ cat_id }}" title="{{ torrent.main_category_id }} - {{ torrent.sub_category_id }}">
|
||||
<img src="/static/img/icons/{{ icon_dir }}/{{ cat_id }}.png">
|
||||
</a>
|
||||
</td>
|
||||
<td><a href="{{ url_for('view_torrent', torrent_id=torrent.id) }}">{{ torrent.display_name | escape }}</a></td>
|
||||
<td><a href="{{ url_for('view_torrent', torrent_id=torrent.meta.id) }}">{{ torrent.display_name | escape }}</a></td>
|
||||
<td style="white-space: nowrap;text-align: center;">
|
||||
{% if torrent.has_torrent %}<a href="{{ url_for('download_torrent', torrent_id=torrent.id) }}"><i class="fa fa-fw fa-download"></i></a>{% endif %}
|
||||
<a href="{{ torrent.magnet_uri }}"><i class="fa fa-fw fa-magnet"></i></a>
|
||||
</td>
|
||||
<td class="text-center">{{ torrent.filesize | filesizeformat(True) }}</td>
|
||||
<td class="text-center" data-timestamp="{{ torrent.created_utc_timestamp|int }}">{{ torrent.created_time.strftime('%Y-%m-%d %H:%M') }}</td>
|
||||
<td class="text-center" {#data-timestamp="{{ torrent.created_time|int }}"#}>{{ torrent.created_time }}</td>
|
||||
{% if config.ENABLE_SHOW_STATS %}
|
||||
<td class="text-center" style="color: green;">{{ torrent.stats.seed_count }}</td>
|
||||
<td class="text-center" style="color: red;">{{ torrent.stats.leech_count }}</td>
|
||||
<td class="text-center">{{ torrent.stats.download_count }}</td>
|
||||
<td class="text-center" style="color: green;">{{ torrent.seed_count }}</td>
|
||||
<td class="text-center" style="color: red;">{{ torrent.leech_count }}</td>
|
||||
<td class="text-center">{{ torrent.download_count }}</td>
|
||||
{% endif %}
|
||||
</tr>
|
||||
{% endfor %}
|
||||
|
|
@ -75,7 +75,9 @@
|
|||
<h3>No results found</h3>
|
||||
{% endif %}
|
||||
|
||||
{#
|
||||
<center>
|
||||
{% from "bootstrap/pagination.html" import render_pagination %}
|
||||
{{ render_pagination(torrent_query) }}
|
||||
</center>
|
||||
#}
|
||||
|
|
|
|||
Loading…
Reference in New Issue