a3x/nyaa
1
0
Fork 0
mirror of https://gitlab.com/SIGBUS/nyaa.git synced 2024-12-22 14:00:00 +00:00
Code Issues Projects Releases Wiki Activity

Optimize MySQL COUNT queries and pagination

Browse source 
Also leaves 'Torrent.trackers' as 'select' for the joining, since we don't need it on listings
This commit is contained in:
TheAMM 2017-05-29 18:27:34 +03:00
parent 7b2bfc57ee
commit 2b331c307e
4 changed files with 53 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
nyaa/fix_paginate.py
View file

@ -1,30 +1,28 @@
import sqlalchemy
from flask_sqlalchemy import Pagination, BaseQuery
from flask import abort
def paginate_faste(self, page=1, per_page=50, max_page=None, step=5):
def paginate_faste(self, page=1, per_page=50, max_page=None, step=5, count_query=None):
if page < 1:
abort(404)
if max_page and page > max_page:
abort(404)
# Count all items
if count_query is not None:
total_query_count = count_query.scalar()
else:
total_query_count = self.count()
# Grab items on current page
items = self.limit(per_page).offset((page - 1) * per_page).all()
if not items and page != 1:
abort(404)
# No need to count if we're on the first page and there are fewer
# items than we expected.
if page == 1 and len(items) < per_page:
total = len(items)
else:
if max_page:
total = self.order_by(None).limit(per_page * min((page + step), max_page)).count()
else:
total = self.order_by(None).limit(per_page * (page + step)).count()
return Pagination(self, page, per_page, total, items)
return Pagination(self, page, per_page, total_query_count, items)
BaseQuery.paginate_faste = paginate_faste

2
nyaa/models.py
View file

@ -162,7 +162,7 @@ class TorrentBase(DeclarativeHelperBase):
@declarative.declared_attr
def trackers(cls):
return db.relationship(cls._flavor_prefix('TorrentTrackers'), uselist=True,
cascade="all, delete-orphan", lazy='joined',
cascade="all, delete-orphan", lazy='select',
order_by=cls._flavor_prefix('TorrentTrackers.order'))
@declarative.declared_attr

57
nyaa/search.py
View file

@ -7,6 +7,7 @@ import shlex
from nyaa import app, db
from nyaa import models
import sqlalchemy
import sqlalchemy_fulltext.modes as FullTextMode
from sqlalchemy_fulltext import FullTextSearch
from elasticsearch import Elasticsearch
@ -183,6 +184,24 @@ def search_elastic(term='', user=None, sort='id', order='desc',
return s.execute()
class QueryPairCaller(object):
''' Simple stupid class to filter one or more queries with the same args '''
def __init__(self, *items):
self.items = list(items)
def __getattr__(self, name):
# Create and return a wrapper that will call item.foobar(*args, **kwargs) for all items
def wrapper(*args, **kwargs):
for i in range(len(self.items)):
method = getattr(self.items[i], name)
if not callable(method):
raise Exception('Attribute %r is not callable' % method)
self.items[i] = method(*args, **kwargs)
return self
return wrapper
def search_db(term='', user=None, sort='id', order='desc', category='0_0',
quality_filter='0', page=1, rss=False, admin=False,
logged_in_user=None, per_page=75):
@ -259,18 +278,23 @@ def search_db(term='', user=None, sort='id', order='desc', category='0_0',
if logged_in_user:
same_user = logged_in_user.id == user
if term:
query = db.session.query(models.TorrentNameSearch)
else:
query = models.Torrent.query
model_class = models.TorrentNameSearch if term else models.Torrent
query = db.session.query(model_class)
# This is... eh. Optimize the COUNT() query since MySQL is bad at that.
# See http://docs.sqlalchemy.org/en/rel_1_1/orm/query.html#sqlalchemy.orm.query.Query.count
# Wrap the queries into the helper class to deduplicate code and apply filters to both in one go
count_query = db.session.query(sqlalchemy.func.count(model_class.id))
qpc = QueryPairCaller(query, count_query)
# User view (/user/username)
if user:
query = query.filter(models.Torrent.uploader_id == user)
qpc.filter(models.Torrent.uploader_id == user)
if not admin:
# Hide all DELETED torrents if regular user
query = query.filter(models.Torrent.flags.op('&')(
qpc.filter(models.Torrent.flags.op('&')(
int(models.TorrentFlags.DELETED)).is_(False))
# If logged in user is not the same as the user being viewed,
# show only torrents that aren't hidden or anonymous
@ -281,41 +305,42 @@ def search_db(term='', user=None, sort='id', order='desc', category='0_0',
# On RSS pages in user view,
# show only torrents that aren't hidden or anonymous no matter what
if not same_user or rss:
query = query.filter(models.Torrent.flags.op('&')(
qpc.filter(models.Torrent.flags.op('&')(
int(models.TorrentFlags.HIDDEN | models.TorrentFlags.ANONYMOUS)).is_(False))
# General view (homepage, general search view)
else:
if not admin:
# Hide all DELETED torrents if regular user
query = query.filter(models.Torrent.flags.op('&')(
qpc.filter(models.Torrent.flags.op('&')(
int(models.TorrentFlags.DELETED)).is_(False))
# If logged in, show all torrents that aren't hidden unless they belong to you
# On RSS pages, show all public torrents and nothing more.
if logged_in_user and not rss:
query = query.filter(
qpc.filter(
(models.Torrent.flags.op('&')(int(models.TorrentFlags.HIDDEN)).is_(False)) |
(models.Torrent.uploader_id == logged_in_user.id))
# Otherwise, show all torrents that aren't hidden
else:
query = query.filter(models.Torrent.flags.op('&')(
qpc.filter(models.Torrent.flags.op('&')(
int(models.TorrentFlags.HIDDEN)).is_(False))
if main_category:
query = query.filter(models.Torrent.main_category_id == main_cat_id)
qpc.filter(models.Torrent.main_category_id == main_cat_id)
elif sub_category:
query = query.filter((models.Torrent.main_category_id == main_cat_id) &
(models.Torrent.sub_category_id == sub_cat_id))
qpc.filter((models.Torrent.main_category_id == main_cat_id) &
(models.Torrent.sub_category_id == sub_cat_id))
if filter_tuple:
query = query.filter(models.Torrent.flags.op('&')(
qpc.filter(models.Torrent.flags.op('&')(
int(filter_tuple[0])).is_(filter_tuple[1]))
if term:
for item in shlex.split(term, posix=False):
if len(item) >= 2:
query = query.filter(FullTextSearch(
qpc.filter(FullTextSearch(
item, models.TorrentNameSearch, FullTextMode.NATURAL))
query, count_query = qpc.items
# Sort and order
if sort.class_ != models.Torrent:
query = query.join(sort.class_)
@ -325,6 +350,6 @@ def search_db(term='', user=None, sort='id', order='desc', category='0_0',
if rss:
query = query.limit(per_page)
else:
query = query.paginate_faste(page, per_page=per_page, step=5)
query = query.paginate_faste(page, per_page=per_page, step=5, count_query=count_query)
return query

2
nyaa/templates/bootstrap/pagination.html
View file

@ -30,7 +30,7 @@
<li{% if not pagination.has_prev %} class="disabled"{% endif %}><a href="{{_arg_url_for(endpoint, url_args, p=pagination.prev_num) if pagination.has_prev else '#'}}">{{prev}}</a></li>
{%- endif -%}
{%- for page in pagination.iter_pages(left_edge=2, left_current=6, right_current=6, right_edge=2) %}
{%- for page in pagination.iter_pages(left_edge=2, left_current=6, right_current=6, right_edge=0) %}
{% if page %}
{% if page != pagination.page %}
<li><a href="{{_arg_url_for(endpoint, url_args, p=page)}}">{{page}}</a></li>