Optimize MySQL COUNT queries and pagination

Also leaves 'Torrent.trackers' as 'select' for the joining, since we don't need it on listings

nyaa/fix_paginate.py

@@ -1,30 +1,28 @@
+import sqlalchemy
 from flask_sqlalchemy import Pagination, BaseQuery
 from flask import abort
 
 
-def paginate_faste(self, page=1, per_page=50, max_page=None, step=5):
+def paginate_faste(self, page=1, per_page=50, max_page=None, step=5, count_query=None):
     if page < 1:
         abort(404)
 
     if max_page and page > max_page:
         abort(404)
 
+    # Count all items
+    if count_query is not None:
+        total_query_count = count_query.scalar()
+    else:
+        total_query_count = self.count()
+
+    # Grab items on current page
     items = self.limit(per_page).offset((page - 1) * per_page).all()
 
     if not items and page != 1:
         abort(404)
 
-    # No need to count if we're on the first page and there are fewer
+    return Pagination(self, page, per_page, total_query_count, items)
-    # items than we expected.
-    if page == 1 and len(items) < per_page:
-        total = len(items)
-    else:
-        if max_page:
-            total = self.order_by(None).limit(per_page * min((page + step), max_page)).count()
-        else:
-            total = self.order_by(None).limit(per_page * (page + step)).count()
-
-    return Pagination(self, page, per_page, total, items)
 
 
 BaseQuery.paginate_faste = paginate_faste

nyaa/models.py

@@ -162,7 +162,7 @@ class TorrentBase(DeclarativeHelperBase):
     @declarative.declared_attr
     def trackers(cls):
         return db.relationship(cls._flavor_prefix('TorrentTrackers'), uselist=True,
-                               cascade="all, delete-orphan", lazy='joined',
+                               cascade="all, delete-orphan", lazy='select',
                                order_by=cls._flavor_prefix('TorrentTrackers.order'))
 
     @declarative.declared_attr

nyaa/search.py

@@ -7,6 +7,7 @@ import shlex
 from nyaa import app, db
 from nyaa import models
 
+import sqlalchemy
 import sqlalchemy_fulltext.modes as FullTextMode
 from sqlalchemy_fulltext import FullTextSearch
 from elasticsearch import Elasticsearch
@@ -183,6 +184,24 @@ def search_elastic(term='', user=None, sort='id', order='desc',
     return s.execute()
 
 
+class QueryPairCaller(object):
+    ''' Simple stupid class to filter one or more queries with the same args '''
+    def __init__(self, *items):
+        self.items = list(items)
+
+    def __getattr__(self, name):
+        # Create and return a wrapper that will call item.foobar(*args, **kwargs) for all items
+        def wrapper(*args, **kwargs):
+            for i in range(len(self.items)):
+                method = getattr(self.items[i], name)
+                if not callable(method):
+                    raise Exception('Attribute %r is not callable' % method)
+                self.items[i] = method(*args, **kwargs)
+            return self
+
+        return wrapper
+
+
 def search_db(term='', user=None, sort='id', order='desc', category='0_0',
               quality_filter='0', page=1, rss=False, admin=False,
               logged_in_user=None, per_page=75):
@@ -259,18 +278,23 @@ def search_db(term='', user=None, sort='id', order='desc', category='0_0',
     if logged_in_user:
         same_user = logged_in_user.id == user
 
-    if term:
+    model_class = models.TorrentNameSearch if term else models.Torrent
-        query = db.session.query(models.TorrentNameSearch)
+
-    else:
+    query = db.session.query(model_class)
-        query = models.Torrent.query
+
+    # This is... eh. Optimize the COUNT() query since MySQL is bad at that.
+    # See http://docs.sqlalchemy.org/en/rel_1_1/orm/query.html#sqlalchemy.orm.query.Query.count
+    # Wrap the queries into the helper class to deduplicate code and apply filters to both in one go
+    count_query = db.session.query(sqlalchemy.func.count(model_class.id))
+    qpc = QueryPairCaller(query, count_query)
 
     # User view (/user/username)
     if user:
-        query = query.filter(models.Torrent.uploader_id == user)
+        qpc.filter(models.Torrent.uploader_id == user)
 
         if not admin:
             # Hide all DELETED torrents if regular user
-            query = query.filter(models.Torrent.flags.op('&')(
+            qpc.filter(models.Torrent.flags.op('&')(
                 int(models.TorrentFlags.DELETED)).is_(False))
             # If logged in user is not the same as the user being viewed,
             # show only torrents that aren't hidden or anonymous
@@ -281,41 +305,42 @@ def search_db(term='', user=None, sort='id', order='desc', category='0_0',
             # On RSS pages in user view,
             # show only torrents that aren't hidden or anonymous no matter what
             if not same_user or rss:
-                query = query.filter(models.Torrent.flags.op('&')(
+                qpc.filter(models.Torrent.flags.op('&')(
                     int(models.TorrentFlags.HIDDEN | models.TorrentFlags.ANONYMOUS)).is_(False))
     # General view (homepage, general search view)
     else:
         if not admin:
             # Hide all DELETED torrents if regular user
-            query = query.filter(models.Torrent.flags.op('&')(
+            qpc.filter(models.Torrent.flags.op('&')(
                 int(models.TorrentFlags.DELETED)).is_(False))
             # If logged in, show all torrents that aren't hidden unless they belong to you
             # On RSS pages, show all public torrents and nothing more.
             if logged_in_user and not rss:
-                query = query.filter(
+                qpc.filter(
                     (models.Torrent.flags.op('&')(int(models.TorrentFlags.HIDDEN)).is_(False)) |
                     (models.Torrent.uploader_id == logged_in_user.id))
             # Otherwise, show all torrents that aren't hidden
             else:
-                query = query.filter(models.Torrent.flags.op('&')(
+                qpc.filter(models.Torrent.flags.op('&')(
                     int(models.TorrentFlags.HIDDEN)).is_(False))
 
     if main_category:
-        query = query.filter(models.Torrent.main_category_id == main_cat_id)
+        qpc.filter(models.Torrent.main_category_id == main_cat_id)
     elif sub_category:
-        query = query.filter((models.Torrent.main_category_id == main_cat_id) &
+        qpc.filter((models.Torrent.main_category_id == main_cat_id) &
                    (models.Torrent.sub_category_id == sub_cat_id))
 
     if filter_tuple:
-        query = query.filter(models.Torrent.flags.op('&')(
+        qpc.filter(models.Torrent.flags.op('&')(
             int(filter_tuple[0])).is_(filter_tuple[1]))
 
     if term:
         for item in shlex.split(term, posix=False):
             if len(item) >= 2:
-                query = query.filter(FullTextSearch(
+                qpc.filter(FullTextSearch(
                     item, models.TorrentNameSearch, FullTextMode.NATURAL))
 
+    query, count_query = qpc.items
     # Sort and order
     if sort.class_ != models.Torrent:
         query = query.join(sort.class_)
@@ -325,6 +350,6 @@ def search_db(term='', user=None, sort='id', order='desc', category='0_0',
     if rss:
         query = query.limit(per_page)
     else:
-        query = query.paginate_faste(page, per_page=per_page, step=5)
+        query = query.paginate_faste(page, per_page=per_page, step=5, count_query=count_query)
 
     return query

nyaa/templates/bootstrap/pagination.html

@@ -30,7 +30,7 @@
     <li{% if not pagination.has_prev %} class="disabled"{% endif %}><a href="{{_arg_url_for(endpoint, url_args, p=pagination.prev_num) if pagination.has_prev else '#'}}">{{prev}}</a></li>
   {%- endif -%}
 
-  {%- for page in pagination.iter_pages(left_edge=2, left_current=6, right_current=6, right_edge=2) %}
+  {%- for page in pagination.iter_pages(left_edge=2, left_current=6, right_current=6, right_edge=0) %}
     {% if page %}
       {% if page != pagination.page %}
         <li><a href="{{_arg_url_for(endpoint, url_args, p=page)}}">{{page}}</a></li>