From 2b331c307e07629a7759fc4d9e7e584ff0f1caf3 Mon Sep 17 00:00:00 2001 From: TheAMM Date: Mon, 29 May 2017 18:27:34 +0300 Subject: [PATCH] Optimize MySQL COUNT queries and pagination Also leaves 'Torrent.trackers' as 'select' for the joining, since we don't need it on listings --- nyaa/fix_paginate.py | 22 +++++---- nyaa/models.py | 2 +- nyaa/search.py | 57 +++++++++++++++++------- nyaa/templates/bootstrap/pagination.html | 2 +- 4 files changed, 53 insertions(+), 30 deletions(-) diff --git a/nyaa/fix_paginate.py b/nyaa/fix_paginate.py index 59334b3..147a0df 100644 --- a/nyaa/fix_paginate.py +++ b/nyaa/fix_paginate.py @@ -1,30 +1,28 @@ +import sqlalchemy from flask_sqlalchemy import Pagination, BaseQuery from flask import abort -def paginate_faste(self, page=1, per_page=50, max_page=None, step=5): +def paginate_faste(self, page=1, per_page=50, max_page=None, step=5, count_query=None): if page < 1: abort(404) if max_page and page > max_page: abort(404) + # Count all items + if count_query is not None: + total_query_count = count_query.scalar() + else: + total_query_count = self.count() + + # Grab items on current page items = self.limit(per_page).offset((page - 1) * per_page).all() if not items and page != 1: abort(404) - # No need to count if we're on the first page and there are fewer - # items than we expected. - if page == 1 and len(items) < per_page: - total = len(items) - else: - if max_page: - total = self.order_by(None).limit(per_page * min((page + step), max_page)).count() - else: - total = self.order_by(None).limit(per_page * (page + step)).count() - - return Pagination(self, page, per_page, total, items) + return Pagination(self, page, per_page, total_query_count, items) BaseQuery.paginate_faste = paginate_faste diff --git a/nyaa/models.py b/nyaa/models.py index 22d349f..74e983e 100644 --- a/nyaa/models.py +++ b/nyaa/models.py @@ -162,7 +162,7 @@ class TorrentBase(DeclarativeHelperBase): @declarative.declared_attr def trackers(cls): return db.relationship(cls._flavor_prefix('TorrentTrackers'), uselist=True, - cascade="all, delete-orphan", lazy='joined', + cascade="all, delete-orphan", lazy='select', order_by=cls._flavor_prefix('TorrentTrackers.order')) @declarative.declared_attr diff --git a/nyaa/search.py b/nyaa/search.py index 976ad5c..4eac57a 100644 --- a/nyaa/search.py +++ b/nyaa/search.py @@ -7,6 +7,7 @@ import shlex from nyaa import app, db from nyaa import models +import sqlalchemy import sqlalchemy_fulltext.modes as FullTextMode from sqlalchemy_fulltext import FullTextSearch from elasticsearch import Elasticsearch @@ -183,6 +184,24 @@ def search_elastic(term='', user=None, sort='id', order='desc', return s.execute() +class QueryPairCaller(object): + ''' Simple stupid class to filter one or more queries with the same args ''' + def __init__(self, *items): + self.items = list(items) + + def __getattr__(self, name): + # Create and return a wrapper that will call item.foobar(*args, **kwargs) for all items + def wrapper(*args, **kwargs): + for i in range(len(self.items)): + method = getattr(self.items[i], name) + if not callable(method): + raise Exception('Attribute %r is not callable' % method) + self.items[i] = method(*args, **kwargs) + return self + + return wrapper + + def search_db(term='', user=None, sort='id', order='desc', category='0_0', quality_filter='0', page=1, rss=False, admin=False, logged_in_user=None, per_page=75): @@ -259,18 +278,23 @@ def search_db(term='', user=None, sort='id', order='desc', category='0_0', if logged_in_user: same_user = logged_in_user.id == user - if term: - query = db.session.query(models.TorrentNameSearch) - else: - query = models.Torrent.query + model_class = models.TorrentNameSearch if term else models.Torrent + + query = db.session.query(model_class) + + # This is... eh. Optimize the COUNT() query since MySQL is bad at that. + # See http://docs.sqlalchemy.org/en/rel_1_1/orm/query.html#sqlalchemy.orm.query.Query.count + # Wrap the queries into the helper class to deduplicate code and apply filters to both in one go + count_query = db.session.query(sqlalchemy.func.count(model_class.id)) + qpc = QueryPairCaller(query, count_query) # User view (/user/username) if user: - query = query.filter(models.Torrent.uploader_id == user) + qpc.filter(models.Torrent.uploader_id == user) if not admin: # Hide all DELETED torrents if regular user - query = query.filter(models.Torrent.flags.op('&')( + qpc.filter(models.Torrent.flags.op('&')( int(models.TorrentFlags.DELETED)).is_(False)) # If logged in user is not the same as the user being viewed, # show only torrents that aren't hidden or anonymous @@ -281,41 +305,42 @@ def search_db(term='', user=None, sort='id', order='desc', category='0_0', # On RSS pages in user view, # show only torrents that aren't hidden or anonymous no matter what if not same_user or rss: - query = query.filter(models.Torrent.flags.op('&')( + qpc.filter(models.Torrent.flags.op('&')( int(models.TorrentFlags.HIDDEN | models.TorrentFlags.ANONYMOUS)).is_(False)) # General view (homepage, general search view) else: if not admin: # Hide all DELETED torrents if regular user - query = query.filter(models.Torrent.flags.op('&')( + qpc.filter(models.Torrent.flags.op('&')( int(models.TorrentFlags.DELETED)).is_(False)) # If logged in, show all torrents that aren't hidden unless they belong to you # On RSS pages, show all public torrents and nothing more. if logged_in_user and not rss: - query = query.filter( + qpc.filter( (models.Torrent.flags.op('&')(int(models.TorrentFlags.HIDDEN)).is_(False)) | (models.Torrent.uploader_id == logged_in_user.id)) # Otherwise, show all torrents that aren't hidden else: - query = query.filter(models.Torrent.flags.op('&')( + qpc.filter(models.Torrent.flags.op('&')( int(models.TorrentFlags.HIDDEN)).is_(False)) if main_category: - query = query.filter(models.Torrent.main_category_id == main_cat_id) + qpc.filter(models.Torrent.main_category_id == main_cat_id) elif sub_category: - query = query.filter((models.Torrent.main_category_id == main_cat_id) & - (models.Torrent.sub_category_id == sub_cat_id)) + qpc.filter((models.Torrent.main_category_id == main_cat_id) & + (models.Torrent.sub_category_id == sub_cat_id)) if filter_tuple: - query = query.filter(models.Torrent.flags.op('&')( + qpc.filter(models.Torrent.flags.op('&')( int(filter_tuple[0])).is_(filter_tuple[1])) if term: for item in shlex.split(term, posix=False): if len(item) >= 2: - query = query.filter(FullTextSearch( + qpc.filter(FullTextSearch( item, models.TorrentNameSearch, FullTextMode.NATURAL)) + query, count_query = qpc.items # Sort and order if sort.class_ != models.Torrent: query = query.join(sort.class_) @@ -325,6 +350,6 @@ def search_db(term='', user=None, sort='id', order='desc', category='0_0', if rss: query = query.limit(per_page) else: - query = query.paginate_faste(page, per_page=per_page, step=5) + query = query.paginate_faste(page, per_page=per_page, step=5, count_query=count_query) return query diff --git a/nyaa/templates/bootstrap/pagination.html b/nyaa/templates/bootstrap/pagination.html index a9e9654..f2d8abc 100644 --- a/nyaa/templates/bootstrap/pagination.html +++ b/nyaa/templates/bootstrap/pagination.html @@ -30,7 +30,7 @@ {{prev}} {%- endif -%} - {%- for page in pagination.iter_pages(left_edge=2, left_current=6, right_current=6, right_edge=2) %} + {%- for page in pagination.iter_pages(left_edge=2, left_current=6, right_current=6, right_edge=0) %} {% if page %} {% if page != pagination.page %}
  • {{page}}