From 18dd7cd66fd908ec372bdd4489f5400b032e94c5 Mon Sep 17 00:00:00 2001
From: shm0o <shm0o@users.noreply.github.com>
Date: Thu, 25 May 2017 15:01:54 -0400
Subject: [PATCH 01/30] static_cachebuster: build the url with url_for

---
 nyaa/routes.py             | 17 ++++++++---------
 nyaa/templates/layout.html | 12 ++++++------
 2 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/nyaa/routes.py b/nyaa/routes.py
index 8893567..b7ff9fe 100644
--- a/nyaa/routes.py
+++ b/nyaa/routes.py
@@ -52,7 +52,7 @@ def redirect_url():
 
 
 @app.template_global()
-def static_cachebuster(static_filename):
+def static_cachebuster(filename):
     ''' Adds a ?t=<mtime> cachebuster to the given path, if the file exists.
         Results are cached in memory and persist until app restart! '''
     # Instead of timestamps, we could use commit hashes (we already load it in __init__)
@@ -61,19 +61,18 @@ def static_cachebuster(static_filename):
 
     if app.debug:
         # Do not bust cache on debug (helps debugging)
-        return static_filename
+        return flask.url_for('static', filename=filename)
 
     # Get file mtime if not already cached.
-    if static_filename not in _static_cache:
-        file_path = os.path.join(app.config['BASE_DIR'], 'nyaa', static_filename[1:])
+    if filename not in _static_cache:
+        file_path = os.path.join(app.static_folder, filename)
+        file_mtime = None
         if os.path.exists(file_path):
             file_mtime = int(os.path.getmtime(file_path))
-            _static_cache[static_filename] = static_filename + '?t=' + str(file_mtime)
-        else:
-            # Throw a warning?
-            _static_cache[static_filename] = static_filename
 
-    return _static_cache[static_filename]
+        _static_cache[filename] = file_mtime
+
+    return flask.url_for('static', filename=filename, t=_static_cache[filename])
 
 
 @app.template_global()
diff --git a/nyaa/templates/layout.html b/nyaa/templates/layout.html
index e0184be..906289b 100755
--- a/nyaa/templates/layout.html
+++ b/nyaa/templates/layout.html
@@ -25,10 +25,10 @@
 			make the navbar not look awful on tablets.
 		-->
 		{# These are extracted here for the dark mode toggle #}
-		{% set bootstrap_light = static_cachebuster('/static/css/bootstrap.min.css') %}
-		{% set bootstrap_dark = static_cachebuster('/static/css/bootstrap-dark.min.css') %}
+		{% set bootstrap_light = static_cachebuster('css/bootstrap.min.css') %}
+		{% set bootstrap_dark = static_cachebuster('css/bootstrap-dark.min.css') %}
 		<link href="{{ bootstrap_light }}" rel="stylesheet" id="bsThemeLink">
-		<link href="{{ static_cachebuster('/static/css/bootstrap-xl-mod.css') }}" rel="stylesheet">
+		<link href="{{ static_cachebuster('css/bootstrap-xl-mod.css') }}" rel="stylesheet">
 		<!--
 			This theme changer script needs to be inline and right under the above stylesheet link to prevent FOUC (Flash Of Unstyled Content)
 			Development version is commented out in static/js/main.js at the bottom of the file
@@ -38,15 +38,15 @@
 		<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous" />
 
 		<!-- Custom styles for this template -->
-		<link href="{{ static_cachebuster('/static/css/main.css') }}" rel="stylesheet">
+		<link href="{{ static_cachebuster('css/main.css') }}" rel="stylesheet">
 
 		<!-- Core JavaScript -->
 		<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.2.1/jquery.min.js" integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4=" crossorigin="anonymous"></script>
 		<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script>
 		<script src="https://cdnjs.cloudflare.com/ajax/libs/commonmark/0.27.0/commonmark.min.js" integrity="sha256-10JreQhQG80GtKuzsioj0K46DlaB/CK/EG+NuG0q97E=" crossorigin="anonymous"></script>
 		<!-- Modified to not apply border-radius to selectpickers and stuff so our navbar looks cool -->
-		<script src="{{ static_cachebuster('/static/js/bootstrap-select.js') }}"></script>
-		<script src="{{ static_cachebuster('/static/js/main.js') }}"></script>
+		<script src="{{ static_cachebuster('js/bootstrap-select.js') }}"></script>
+		<script src="{{ static_cachebuster('js/main.js') }}"></script>
 
 		<!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
 		<!--[if lt IE 9]>

From 8b7712b1d0c9dea49ff795d3f3edf44d06553a5b Mon Sep 17 00:00:00 2001
From: shm0o <shm0o@users.noreply.github.com>
Date: Thu, 25 May 2017 19:03:56 -0400
Subject: [PATCH 02/30] templates/view.html: use url_for for the 'edit_torrent'
 endpoint

---
 nyaa/templates/view.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nyaa/templates/view.html b/nyaa/templates/view.html
index a4f9c9e..4834635 100644
--- a/nyaa/templates/view.html
+++ b/nyaa/templates/view.html
@@ -10,7 +10,7 @@
 	<div class="panel-heading"{% if torrent.hidden %} style="background-color: darkgray;"{% endif %}>
 		<h3 class="panel-title">
 			{% if can_edit %}
-			<a href="{{ request.url }}/edit" title="Edit torrent"><i class="fa fa-fw fa-pencil"></i></a>
+			<a href="{{ url_for('edit_torrent', torrent_id=torrent.id) }}" title="Edit torrent"><i class="fa fa-fw fa-pencil"></i></a>
 			{% endif %}
 			{{ torrent.display_name }}
 		</h3>

From c178f905b05e51b7dae9361980cb93b2724ed2f7 Mon Sep 17 00:00:00 2001
From: shm0o <shm0o@users.noreply.github.com>
Date: Thu, 25 May 2017 19:05:25 -0400
Subject: [PATCH 03/30] templates/search_results.html: use url_for for the
 appropriate endpoints

---
 nyaa/templates/search_results.html | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/nyaa/templates/search_results.html b/nyaa/templates/search_results.html
index f4c5ff2..a73858a 100644
--- a/nyaa/templates/search_results.html
+++ b/nyaa/templates/search_results.html
@@ -12,7 +12,7 @@
 {% if special_results is defined and not search.user %}
 {% if special_results.first_word_user %}
 <div class="alert alert-info">
-	<a href="/user/{{ special_results.first_word_user.username }}{{ modify_query(q=special_results.query_sans_user)[1:] }}">Click here to see only results uploaded by {{ special_results.first_word_user.username }}</a>
+	<a href="{{ url_for('view_user', user_name=special_results.first_word_user.username) }}{{ modify_query(q=special_results.query_sans_user)[1:] }}">Click here to see only results uploaded by {{ special_results.first_word_user.username }}</a>
 </div>
 {% endif %}
 {% endif %}
@@ -61,11 +61,11 @@
 				{% set icon_dir = config.SITE_FLAVOR %}
 				<td style="padding:0 4px;">
 				{% if use_elastic %}
-				<a href="/?c={{ cat_id }}" title="{{ category_name(cat_id) }}">
+				<a href="{{ url_for('home', c=cat_id) }}" title="{{ category_name(cat_id) }}">
 				{% else %}
-				<a href="/?c={{ cat_id }}" title="{{ torrent.main_category.name }} - {{ torrent.sub_category.name }}">
+				<a href="{{ url_for('home', c=cat_id) }}" title="{{ torrent.main_category.name }} - {{ torrent.sub_category.name }}">
 				{% endif %}
-					<img src="/static/img/icons/{{ icon_dir }}/{{ cat_id }}.png" alt="{{ category_name(cat_id) }}">
+					<img src="{{ url_for('static', filename='img/icons/%s/%s.png'|format(icon_dir, cat_id)) }}" alt="{{ category_name(cat_id) }}">
 				</a>
 				</td>
 				{% if use_elastic %}

From 3edc73a518c5267251c17b08b853da06177a5618 Mon Sep 17 00:00:00 2001
From: shm0o <shm0o@users.noreply.github.com>
Date: Thu, 25 May 2017 19:08:55 -0400
Subject: [PATCH 04/30] templates/layout.html: use url_for for the appropriate
 endpoints

---
 nyaa/templates/layout.html | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/nyaa/templates/layout.html b/nyaa/templates/layout.html
index 906289b..91bb818 100755
--- a/nyaa/templates/layout.html
+++ b/nyaa/templates/layout.html
@@ -6,9 +6,9 @@
 
 		<meta name="viewport" content="width=device-width">
 		<meta http-equiv="X-UA-Compatible" content="IE=edge">
-		<link rel="shortcut icon" type="image/png" href="/static/favicon.png">
-		<link rel="icon" type="image/png" href="/static/favicon.png">
-		<link rel="mask-icon" href="/static/pinned-tab.svg" color="#3582F7">
+		<link rel="shortcut icon" type="image/png" href="{{ url_for('static', filename='favicon.png') }}">
+		<link rel="icon" type="image/png" href="{{ url_for('static', filename='favicon.png') }}">
+		<link rel="mask-icon" href="{{ url_for('static', filename='pinned-tab.svg') }}" color="#3582F7">
 		<link rel="alternate" type="application/rss+xml" href="{% if rss_filter %}{{ url_for('home', page='rss', _external=True, **rss_filter) }}{% else %}{{ url_for('home', page='rss', _external=True) }}{% endif %}" />
 
 		<meta property="og:site_name" content="{{ config.SITE_NAME }}">
@@ -65,21 +65,21 @@
 					<span class="icon-bar"></span>
 					<span class="icon-bar"></span>
 					</button>
-					<a class="navbar-brand" href="/">{{ config.SITE_NAME }}</a>
+					<a class="navbar-brand" href="{{ url_for('home') }}">{{ config.SITE_NAME }}</a>
 				</div>
 				{% set search_username = (user.username + ("'" if user.username[-1] == 's' else "'s")) if user_page else None %}
 				{% set search_placeholder = 'Search {} torrents...'.format(search_username) if user_page else 'Search...' %}
 				<div id="navbar" class="navbar-collapse collapse">
 					<ul class="nav navbar-nav">
-						<li {% if request.path == "/upload" %} class="active"{% endif %}><a href="/upload">Upload</a></li>
+						<li {% if request.path == url_for('upload') %}class="active"{% endif %}><a href="{{ url_for('upload') }}">Upload</a></li>
 						<li class="dropdown">
 							<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
 								About
 								<span class="caret"></span>
 							</a>
 							<ul class="dropdown-menu">
-								<li {% if request.path == "/rules" %} class="active"{% endif %}><a href="/rules">Rules</a></li>
-								<li {% if request.path == "/help" %} class="active"{% endif %}><a href="/help">Help</a></li>
+								<li {% if request.path == url_for('site_rules') %}class="active"{% endif %}><a href="{{ url_for('site_rules') }}">Rules</a></li>
+								<li {% if request.path == url_for('site_help') %}class="active"{% endif %}><a href="{{ url_for('site_help') }}">Help</a></li>
 							</ul>
 						</li>
 						<li><a href="{% if rss_filter %}{{ url_for('home', page='rss', **rss_filter) }}{% else %}{{ url_for('home', page='rss') }}{% endif %}">RSS</a></li>
@@ -116,13 +116,13 @@
 									</a>
 								</li>
 								<li>
-									<a href="/profile">
+									<a href="{{ url_for('profile') }}">
 										<i class="fa fa-gear fa-fw"></i>
 										Profile
 									</a>
 								</li>
 								<li>
-									<a href="/logout">
+									<a href="{{ url_for('logout') }}">
 										<i class="fa fa-times fa-fw"></i>
 										Logout
 									</a>
@@ -142,13 +142,13 @@
 							</a>
 							<ul class="dropdown-menu">
 								<li>
-									<a href="/login">
+									<a href="{{ url_for('login') }}">
 										<i class="fa fa-sign-in fa-fw"></i>
 										Login
 									</a>
 								</li>
 								<li>
-									<a href="/register">
+									<a href="{{ url_for('register') }}">
 										<i class="fa fa-pencil fa-fw"></i>
 										Register
 									</a>
@@ -204,7 +204,7 @@
 						{% if user_page %}
 						<form class="navbar-form navbar-right form" action="{{ url_for('view_user', user_name=user.username) }}" method="get">
 						{% else %}
-						<form class="navbar-form navbar-right form" action="/" method="get">
+						<form class="navbar-form navbar-right form" action="{{ url_for('home') }}" method="get">
 						{% endif %}
 							
 							<input type="text" class="form-control" name="q" placeholder="{{ search_placeholder }}" value="{{ search["term"] if search is defined else '' }}">
@@ -240,7 +240,7 @@
 					{% if user_page %}
 					<form class="navbar-form navbar-right form" action="{{ url_for('view_user', user_name=user.username) }}" method="get">
 					{% else %}
-					<form class="navbar-form navbar-right form" action="/" method="get">
+					<form class="navbar-form navbar-right form" action="{{ url_for('home') }}" method="get">
 					{% endif %}
 						<div class="input-group search-container hidden-xs hidden-sm">
 							<input type="text" class="form-control search-bar" name="q" placeholder="{{ search_placeholder }}" value="{{ search["term"] if search is defined else '' }}">

From 6c80557e39e7add5878a6fa4fb362cdf894a0c59 Mon Sep 17 00:00:00 2001
From: TheAMM <the.actual.amm@gmail.com>
Date: Fri, 26 May 2017 16:08:46 +0300
Subject: [PATCH 05/30] SCHEMA CHANGE: Add comment_count to Torrent

with a helper function on Torrent to update the count.
---
 ...ceb2cb4d7c_add_comment_count_to_torrent.py | 44 +++++++++++++++++++
 nyaa/models.py                                | 10 +++++
 2 files changed, 54 insertions(+)
 create mode 100644 migrations/versions/2bceb2cb4d7c_add_comment_count_to_torrent.py

diff --git a/migrations/versions/2bceb2cb4d7c_add_comment_count_to_torrent.py b/migrations/versions/2bceb2cb4d7c_add_comment_count_to_torrent.py
new file mode 100644
index 0000000..49b949a
--- /dev/null
+++ b/migrations/versions/2bceb2cb4d7c_add_comment_count_to_torrent.py
@@ -0,0 +1,44 @@
+"""Add comment_count to Torrent
+
+Revision ID: 2bceb2cb4d7c
+Revises: d0eeb8049623
+Create Date: 2017-05-26 15:07:21.114331
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '2bceb2cb4d7c'
+down_revision = 'd0eeb8049623'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('nyaa_torrents', sa.Column('comment_count', sa.Integer(), nullable=False))
+    op.create_index(op.f('ix_nyaa_torrents_comment_count'), 'nyaa_torrents', ['comment_count'], unique=False)
+
+    try:
+        op.add_column('sukebei_torrents', sa.Column('comment_count', sa.Integer(), nullable=False))
+        op.create_index(op.f('ix_sukebei_torrents_comment_count'), 'sukebei_torrents', ['comment_count'], unique=False)    
+    except (sa.exc.OperationalError, sa.exc.ProgrammingError):
+        print('Could not upgrade sukebei!')
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_nyaa_torrents_comment_count'), table_name='nyaa_torrents')
+    op.drop_column('nyaa_torrents', 'comment_count')
+
+    try:
+        op.drop_index(op.f('ix_sukebei_torrents_comment_count'), table_name='sukebei_torrents')
+        op.drop_column('sukebei_torrents', 'comment_count')
+    except (sa.exc.OperationalError, sa.exc.ProgrammingError):
+        print('Could not downgrade sukebei!')
+
+    # ### end Alembic commands ###
diff --git a/nyaa/models.py b/nyaa/models.py
index 8b27679..5d386da 100755
--- a/nyaa/models.py
+++ b/nyaa/models.py
@@ -67,6 +67,8 @@ class Torrent(db.Model):
     uploader_ip = db.Column(db.Binary(length=16), default=None, nullable=True)
     has_torrent = db.Column(db.Boolean, nullable=False, default=False)
 
+    comment_count = db.Column(db.Integer, default=0, nullable=False, index=True)
+
     created_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow, nullable=False)
     updated_time = db.Column(db.DateTime(timezone=False),
                              default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
@@ -107,6 +109,10 @@ class Torrent(db.Model):
     def __repr__(self):
         return '<{0} #{1.id} \'{1.display_name}\' {1.filesize}b>'.format(type(self).__name__, self)
 
+    def update_comment_count(self):
+        self.comment_count = Comment.query.filter_by(torrent_id = self.id).count()
+        return self.comment_count
+
     @property
     def created_utc_timestamp(self):
         ''' Returns a UTC POSIX timestamp, as seconds '''
@@ -149,6 +155,8 @@ class Torrent(db.Model):
         if self.uploader_ip:
             return str(ip_address(self.uploader_ip))
 
+    # Flag getters and setters below
+
     @property
     def anonymous(self):
         return self.flags & TorrentFlags.ANONYMOUS
@@ -197,6 +205,8 @@ class Torrent(db.Model):
     def complete(self, value):
         self.flags = (self.flags & ~TorrentFlags.COMPLETE) | (value and TorrentFlags.COMPLETE)
 
+    # Class methods
+
     @classmethod
     def by_id(cls, id):
         return cls.query.get(id)

From b5ebebf98e9ec94231bad45dc6f9dc355404570a Mon Sep 17 00:00:00 2001
From: TheAMM <the.actual.amm@gmail.com>
Date: Fri, 26 May 2017 16:11:31 +0300
Subject: [PATCH 06/30] Update Torrent.comment_count on comment submit/delete

---
 nyaa/routes.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/nyaa/routes.py b/nyaa/routes.py
index 8893567..251ebe5 100644
--- a/nyaa/routes.py
+++ b/nyaa/routes.py
@@ -658,9 +658,10 @@ def view_torrent(torrent_id):
                 text=comment_text)
 
             db.session.add(comment)
-            db.session.commit()
+            db.session.flush()
 
-            torrent_count = models.Comment.query.filter_by(torrent_id=torrent.id).count()
+            torrent_count = torrent.update_comment_count()
+            db.session.commit()
 
             flask.flash('Comment successfully posted.', 'success')
 
@@ -686,6 +687,9 @@ def view_torrent(torrent_id):
 def delete_comment(torrent_id, comment_id):
     if not flask.g.user:
         flask.abort(403)
+    torrent = models.Torrent.by_id(torrent_id)
+    if not torrent:
+        flask.abort(404)
 
     comment = models.Comment.query.filter_by(id=comment_id).first()
     if not comment:
@@ -695,6 +699,8 @@ def delete_comment(torrent_id, comment_id):
         flask.abort(403)
 
     db.session.delete(comment)
+    db.session.flush()
+    torrent.update_comment_count()
     db.session.commit()
 
     flask.flash('Comment successfully deleted.', 'success')

From 9cd6c506ae0fc02f9bcbbf912867ec86b2ab025f Mon Sep 17 00:00:00 2001
From: TheAMM <the.actual.amm@gmail.com>
Date: Fri, 26 May 2017 16:12:47 +0300
Subject: [PATCH 07/30] Update ElasticSeach index and scripts for comment_count

---
 es_mapping.yml  | 2 ++
 import_to_es.py | 1 +
 sync_es.py      | 1 +
 3 files changed, 4 insertions(+)

diff --git a/es_mapping.yml b/es_mapping.yml
index 1fc72ad..3490fd9 100644
--- a/es_mapping.yml
+++ b/es_mapping.yml
@@ -91,6 +91,8 @@ mappings:
         type: long
       seed_count:
         type: long
+      comment_count:
+        type: long
       # these ids are really only for filtering, thus keyword
       uploader_id:
         type: keyword
diff --git a/import_to_es.py b/import_to_es.py
index 106cbc1..6832781 100755
--- a/import_to_es.py
+++ b/import_to_es.py
@@ -51,6 +51,7 @@ def mk_es(t):
             "uploader_id": t.uploader_id,
             "main_category_id": t.main_category_id,
             "sub_category_id": t.sub_category_id,
+            "comment_count": t.comment_count,
             # XXX all the bitflags are numbers
             "anonymous": bool(t.anonymous),
             "trusted": bool(t.trusted),
diff --git a/sync_es.py b/sync_es.py
index cafafb5..2f2bb09 100755
--- a/sync_es.py
+++ b/sync_es.py
@@ -86,6 +86,7 @@ def reindex_torrent(t, index_name):
         "uploader_id": t['uploader_id'],
         "main_category_id": t['main_category_id'],
         "sub_category_id": t['sub_category_id'],
+        "comment_count": t['comment_count'],
         # XXX all the bitflags are numbers
         "anonymous": bool(f & TorrentFlags.ANONYMOUS),
         "trusted": bool(f & TorrentFlags.TRUSTED),

From 90695cad11f3a81d1676b2a4da9225bfeff9fe46 Mon Sep 17 00:00:00 2001
From: Kfir Hadas <sharkykh@gmail.com>
Date: Tue, 23 May 2017 10:51:28 +0300
Subject: [PATCH 08/30] Add number of comments to torrent lists

Will only show when number of comments is above zero.
---
 nyaa/static/css/main.css           | 12 ++++++++++++
 nyaa/templates/search_results.html | 13 ++++++++-----
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/nyaa/static/css/main.css b/nyaa/static/css/main.css
index 3368282..785c78f 100755
--- a/nyaa/static/css/main.css
+++ b/nyaa/static/css/main.css
@@ -62,6 +62,18 @@ table.torrent-list tbody tr td a:visited {
 	color: #1d4568;
 }
 
+table.torrent-list tbody .comments {
+	float: right;
+	border: 1px solid #d7d7d7;
+	border-radius: 3px;
+	color: #383838;
+	padding: 0 5px 0 5px;
+	background-color: #ffffff;
+}
+table.torrent-list tbody .comments i {
+	padding-right: 2px;
+}
+
 #torrent-description img {
 	max-width: 100%;
 }
diff --git a/nyaa/templates/search_results.html b/nyaa/templates/search_results.html
index f4c5ff2..4d2dea5 100644
--- a/nyaa/templates/search_results.html
+++ b/nyaa/templates/search_results.html
@@ -68,11 +68,14 @@
 					<img src="/static/img/icons/{{ icon_dir }}/{{ cat_id }}.png" alt="{{ category_name(cat_id) }}">
 				</a>
 				</td>
-				{% if use_elastic %}
-				<td><a href="{{ url_for('view_torrent', torrent_id=torrent.meta.id) }}" title="{{ torrent.display_name | escape }}">{%if "highlight" in torrent.meta %}{{ torrent.meta.highlight.display_name[0] | safe }}{% else %}{{torrent.display_name}}{%endif%}</a></td>
-				{% else %}
-				<td><a href="{{ url_for('view_torrent', torrent_id=torrent.id) }}" title="{{ torrent.display_name | escape }}">{{ torrent.display_name | escape }}</a></td>
-				{% endif %}
+				<td>
+					{% if torrent.comments %}<span class="comments"><i class="fa fa-comments-o"></i>{{ torrent.comments | length }}</span>{% endif %}
+					{% if use_elastic %}
+					<a href="{{ url_for('view_torrent', torrent_id=torrent.meta.id) }}" title="{{ torrent.display_name | escape }}">{%if "highlight" in torrent.meta %}{{ torrent.meta.highlight.display_name[0] | safe }}{% else %}{{torrent.display_name}}{%endif%}</a>
+					{% else %}
+					<a href="{{ url_for('view_torrent', torrent_id=torrent.id) }}" title="{{ torrent.display_name | escape }}">{{ torrent.display_name | escape }}</a>
+					{% endif %}
+				</td>
 				<td style="white-space: nowrap;text-align: center;">
 					{% if torrent.has_torrent %}<a href="{{ url_for('download_torrent', torrent_id=torrent.id) }}"><i class="fa fa-fw fa-download"></i></a>{% endif %}
 					{% if use_elastic %}

From aad5a0c6ea770b90890a3f93b6c2a56d32fd28c3 Mon Sep 17 00:00:00 2001
From: Kfir Hadas <sharkykh@gmail.com>
Date: Thu, 25 May 2017 17:56:06 +0300
Subject: [PATCH 09/30] Update style

Add support for sorting comments (in advance),
add th for sorting by comments_count, add title to the counter box and fix it being under the torrent's name,
---
 nyaa/search.py                     |  2 ++
 nyaa/static/css/main.css           |  8 +++++++-
 nyaa/templates/search_results.html | 16 +++++++++++-----
 3 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/nyaa/search.py b/nyaa/search.py
index 9e22f84..4db2879 100644
--- a/nyaa/search.py
+++ b/nyaa/search.py
@@ -25,6 +25,7 @@ def search_elastic(term='', user=None, sort='id', order='desc',
         'id': 'id',
         'size': 'filesize',
         # 'name': 'display_name',  # This is slow and buggy
+        'comments': 'comments_count',
         'seeders': 'seed_count',
         'leechers': 'leech_count',
         'downloads': 'download_count'
@@ -190,6 +191,7 @@ def search_db(term='', user=None, sort='id', order='desc', category='0_0',
         'size': models.Torrent.filesize,
         # Disable this because we disabled this in search_elastic, for the sake of consistency:
         # 'name': models.Torrent.display_name,
+        'comments': models.Torrent.comment_count,
         'seeders': models.Statistic.seed_count,
         'leechers': models.Statistic.leech_count,
         'downloads': models.Statistic.download_count
diff --git a/nyaa/static/css/main.css b/nyaa/static/css/main.css
index 785c78f..1fe9f6f 100755
--- a/nyaa/static/css/main.css
+++ b/nyaa/static/css/main.css
@@ -62,12 +62,18 @@ table.torrent-list tbody tr td a:visited {
 	color: #1d4568;
 }
 
+/* comments count */
+table.torrent-list .hdr-comments {
+	border-left: hidden;
+}
+
 table.torrent-list tbody .comments {
+	position: relative;
 	float: right;
 	border: 1px solid #d7d7d7;
 	border-radius: 3px;
 	color: #383838;
-	padding: 0 5px 0 5px;
+	padding: 1px 5px;
 	background-color: #ffffff;
 }
 table.torrent-list tbody .comments i {
diff --git a/nyaa/templates/search_results.html b/nyaa/templates/search_results.html
index 4d2dea5..bcdc0b3 100644
--- a/nyaa/templates/search_results.html
+++ b/nyaa/templates/search_results.html
@@ -28,6 +28,9 @@
 				{% call render_column_header("hdr-name", "width:auto;") %}
 					<div>Name</div>
 				{% endcall %}
+				{% call render_column_header("hdr-comments", "width:50px;", center_text=True, sort_key="comments", header_title="Comments") %}
+					<i class="fa fa-comments-o"></i>
+				{% endcall %}
 				{% call render_column_header("hdr-link", "width:70px;", center_text=True) %}
 					<div>Link</div>
 				{% endcall %}
@@ -44,12 +47,10 @@
 				{% endcall %}
 				{% call render_column_header("hdr-leechers", "width:50px;", center_text=True, sort_key="leechers", header_title="Leeches") %}
 					<i class="fa fa-arrow-down" aria-hidden="true"></i>
-
 				{% endcall %}
 				{% call render_column_header("hdr-downloads", "width:50px;", center_text=True, sort_key="downloads", header_title="Completed downloads") %}
 					<i class="fa fa-check" aria-hidden="true"></i>
 				{% endcall %}
-
 				{% endif %}
 			</tr>
 		</thead>
@@ -68,15 +69,20 @@
 					<img src="/static/img/icons/{{ icon_dir }}/{{ cat_id }}.png" alt="{{ category_name(cat_id) }}">
 				</a>
 				</td>
-				<td>
-					{% if torrent.comments %}<span class="comments"><i class="fa fa-comments-o"></i>{{ torrent.comments | length }}</span>{% endif %}
+				<td colspan="2">
+					{% set com_count = torrent.comment_count %}
+					{% if com_count %}
+					<span class="comments" title="{{ '{c} comment{s}'.format(c=com_count, s='s' if com_count > 1 else '') }}">
+						<i class="fa fa-comments-o"></i>{{ com_count -}}
+					</span>
+					{% endif %}
 					{% if use_elastic %}
 					<a href="{{ url_for('view_torrent', torrent_id=torrent.meta.id) }}" title="{{ torrent.display_name | escape }}">{%if "highlight" in torrent.meta %}{{ torrent.meta.highlight.display_name[0] | safe }}{% else %}{{torrent.display_name}}{%endif%}</a>
 					{% else %}
 					<a href="{{ url_for('view_torrent', torrent_id=torrent.id) }}" title="{{ torrent.display_name | escape }}">{{ torrent.display_name | escape }}</a>
 					{% endif %}
 				</td>
-				<td style="white-space: nowrap;text-align: center;">
+				<td class="text-center" style="white-space: nowrap;">
 					{% if torrent.has_torrent %}<a href="{{ url_for('download_torrent', torrent_id=torrent.id) }}"><i class="fa fa-fw fa-download"></i></a>{% endif %}
 					{% if use_elastic %}
 					<a href="{{ create_magnet_from_es_info(torrent.display_name, torrent.info_hash) }}"><i class="fa fa-fw fa-magnet"></i></a>

From fa946844d1fe41edd84af1b4b8f0f1c4a493eaba Mon Sep 17 00:00:00 2001
From: Kfir Hadas <sharkykh@gmail.com>
Date: Fri, 26 May 2017 19:23:34 +0300
Subject: [PATCH 10/30] Style fixes

Comments box changed the height of the `<tr>`
Tweak icon/text sizes.
---
 nyaa/static/css/main.css | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/nyaa/static/css/main.css b/nyaa/static/css/main.css
index 1fe9f6f..4d70ea0 100755
--- a/nyaa/static/css/main.css
+++ b/nyaa/static/css/main.css
@@ -65,6 +65,11 @@ table.torrent-list tbody tr td a:visited {
 /* comments count */
 table.torrent-list .hdr-comments {
 	border-left: hidden;
+	font-size: medium;
+}
+
+table.torrent-list .hdr-comments i {
+	margin-right: 6px;
 }
 
 table.torrent-list tbody .comments {
@@ -73,7 +78,8 @@ table.torrent-list tbody .comments {
 	border: 1px solid #d7d7d7;
 	border-radius: 3px;
 	color: #383838;
-	padding: 1px 5px;
+	padding: 0 5px;
+	font-size: small;
 	background-color: #ffffff;
 }
 table.torrent-list tbody .comments i {

From d89f74893ba48a8a3415ab7d3a4a3273031df917 Mon Sep 17 00:00:00 2001
From: TheAMM <the.actual.amm@gmail.com>
Date: Fri, 26 May 2017 23:27:30 +0300
Subject: [PATCH 11/30] Remove username validator from login form to allow
 logging in with email

Since our previous regex was broken, the validator passed practically everything.
---
 nyaa/forms.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nyaa/forms.py b/nyaa/forms.py
index f21569b..36cf714 100644
--- a/nyaa/forms.py
+++ b/nyaa/forms.py
@@ -42,7 +42,7 @@ _username_validator = Regexp(
 
 
 class LoginForm(FlaskForm):
-    username = StringField('Username or email address', [DataRequired(), _username_validator])
+    username = StringField('Username or email address', [DataRequired()])
     password = PasswordField('Password', [DataRequired()])
 
 

From 6f4aad56abc28f498f0c57f882d6993dd0dce83c Mon Sep 17 00:00:00 2001
From: TheAMM <the.actual.amm@gmail.com>
Date: Sat, 27 May 2017 18:05:31 +0300
Subject: [PATCH 12/30] Update comment count migration script for #201

---
 .../versions/2bceb2cb4d7c_add_comment_count_to_torrent.py       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/migrations/versions/2bceb2cb4d7c_add_comment_count_to_torrent.py b/migrations/versions/2bceb2cb4d7c_add_comment_count_to_torrent.py
index 49b949a..aa88a8e 100644
--- a/migrations/versions/2bceb2cb4d7c_add_comment_count_to_torrent.py
+++ b/migrations/versions/2bceb2cb4d7c_add_comment_count_to_torrent.py
@@ -23,7 +23,7 @@ def upgrade():
 
     try:
         op.add_column('sukebei_torrents', sa.Column('comment_count', sa.Integer(), nullable=False))
-        op.create_index(op.f('ix_sukebei_torrents_comment_count'), 'sukebei_torrents', ['comment_count'], unique=False)    
+        op.create_index(op.f('ix_sukebei_torrents_comment_count'), 'sukebei_torrents', ['comment_count'], unique=False)
     except (sa.exc.OperationalError, sa.exc.ProgrammingError):
         print('Could not upgrade sukebei!')
 

From f7ed2c06d35e935c783ca810e912e68247c2c101 Mon Sep 17 00:00:00 2001
From: TheAMM <the.actual.amm@gmail.com>
Date: Sat, 27 May 2017 18:10:21 +0300
Subject: [PATCH 13/30] Jeez fine PEP8 I get it

---
 nyaa/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nyaa/models.py b/nyaa/models.py
index 5d386da..18178d6 100755
--- a/nyaa/models.py
+++ b/nyaa/models.py
@@ -110,7 +110,7 @@ class Torrent(db.Model):
         return '<{0} #{1.id} \'{1.display_name}\' {1.filesize}b>'.format(type(self).__name__, self)
 
     def update_comment_count(self):
-        self.comment_count = Comment.query.filter_by(torrent_id = self.id).count()
+        self.comment_count = Comment.query.filter_by(torrent_id=self.id).count()
         return self.comment_count
 
     @property

From 07cf0de6f991ee5db61c2b1996cc46f982328e18 Mon Sep 17 00:00:00 2001
From: TheAMM <the.actual.amm@gmail.com>
Date: Sat, 27 May 2017 17:33:48 +0300
Subject: [PATCH 14/30] Rewrite site-specific database models into declarative
 bases

This allows us to easily extend the bases into Nyaa and Sukebei models, with working FKs and relationships.

Note: the schema will (should!) otherwise match, however; the following index on Torrent was renamed:
uploader_flag_idx will be <prefix>_uploader_flag_idx, as we will be always creating both flavors.
The index rename should not affect SQA, but will potentially break migrations.
---
 nyaa/models.py | 398 +++++++++++++++++++++++++++++++++++++------------
 1 file changed, 305 insertions(+), 93 deletions(-)

diff --git a/nyaa/models.py b/nyaa/models.py
index 8b27679..e34c767 100755
--- a/nyaa/models.py
+++ b/nyaa/models.py
@@ -3,10 +3,13 @@ from enum import Enum, IntEnum
 from datetime import datetime, timezone
 from nyaa import app, db
 from nyaa.torrents import create_magnet
+
 from sqlalchemy import func, ForeignKeyConstraint, Index
+from sqlalchemy.ext import declarative
 from sqlalchemy_utils import ChoiceType, EmailType, PasswordType
-from werkzeug.security import generate_password_hash, check_password_hash
 from sqlalchemy_fulltext import FullText
+
+from werkzeug.security import generate_password_hash, check_password_hash
 from ipaddress import ip_address
 
 import re
@@ -36,6 +39,31 @@ else:
 UTC_EPOCH = datetime.utcfromtimestamp(0)
 
 
+class DeclarativeHelperBase(object):
+    ''' This class eases our nyaa-sukebei shenanigans by automatically adjusting
+        __tablename__ and providing class methods for renaming references. '''
+    # See http://docs.sqlalchemy.org/en/latest/orm/extensions/declarative/api.html
+
+    __tablename_base__ = None
+    __flavor__ = None
+
+    @classmethod
+    def _table_prefix_string(cls):
+        return cls.__flavor__.lower() + '_'
+
+    @classmethod
+    def _table_prefix(cls, table_name):
+        return cls._table_prefix_string() + table_name
+
+    @classmethod
+    def _flavor_prefix(cls, table_name):
+        return cls.__flavor__ + table_name
+
+    @declarative.declared_attr
+    def __tablename__(cls):
+        return cls._table_prefix(cls.__tablename_base__)
+
+
 class TorrentFlags(IntEnum):
     NONE = 0
     ANONYMOUS = 1
@@ -46,16 +74,13 @@ class TorrentFlags(IntEnum):
     DELETED = 32
 
 
-DB_TABLE_PREFIX = app.config['TABLE_PREFIX']
-
-
-class Torrent(db.Model):
-    __tablename__ = DB_TABLE_PREFIX + 'torrents'
+class TorrentBase(DeclarativeHelperBase):
+    __tablename_base__ = 'torrents'
 
     id = db.Column(db.Integer, primary_key=True)
     info_hash = db.Column(BinaryType(length=20), unique=True, nullable=False, index=True)
-    display_name = db.Column(
-        db.String(length=255, collation=COL_UTF8_GENERAL_CI), nullable=False, index=True)
+    display_name = db.Column(db.String(length=255, collation=COL_UTF8_GENERAL_CI),
+                             nullable=False, index=True)
     torrent_name = db.Column(db.String(length=255), nullable=False)
     information = db.Column(db.String(length=255), nullable=False)
     description = db.Column(DescriptionTextType(collation=COL_UTF8MB4_BIN), nullable=False)
@@ -63,45 +88,84 @@ class Torrent(db.Model):
     filesize = db.Column(db.BIGINT, default=0, nullable=False, index=True)
     encoding = db.Column(db.String(length=32), nullable=False)
     flags = db.Column(db.Integer, default=0, nullable=False, index=True)
-    uploader_id = db.Column(db.Integer, db.ForeignKey('users.id'), nullable=True)
+
+    @declarative.declared_attr
+    def uploader_id(cls):
+        # Even though this is same for both tables, declarative requires this
+        return db.Column(db.Integer, db.ForeignKey('users.id'), nullable=True)
+
     uploader_ip = db.Column(db.Binary(length=16), default=None, nullable=True)
     has_torrent = db.Column(db.Boolean, nullable=False, default=False)
 
     created_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow, nullable=False)
-    updated_time = db.Column(db.DateTime(timezone=False),
-                             default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
+    updated_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow,
+                             onupdate=datetime.utcnow, nullable=False)
+
+    @declarative.declared_attr
+    def main_category_id(cls):
+        fk = db.ForeignKey(cls._table_prefix('main_categories.id'))
+        return db.Column(db.Integer, fk, nullable=False)
 
-    main_category_id = db.Column(db.Integer, db.ForeignKey(
-        DB_TABLE_PREFIX + 'main_categories.id'), nullable=False)
     sub_category_id = db.Column(db.Integer, nullable=False)
-    redirect = db.Column(db.Integer, db.ForeignKey(
-        DB_TABLE_PREFIX + 'torrents.id'), nullable=True)
 
-    __table_args__ = (
-        Index('uploader_flag_idx', 'uploader_id', 'flags'),
-        ForeignKeyConstraint(
-            ['main_category_id', 'sub_category_id'],
-            [DB_TABLE_PREFIX + 'sub_categories.main_category_id',
-                DB_TABLE_PREFIX + 'sub_categories.id']
-        ), {}
-    )
+    @declarative.declared_attr
+    def redirect(cls):
+        fk = db.ForeignKey(cls._table_prefix('torrents.id'))
+        return db.Column(db.Integer, fk, nullable=True)
 
-    user = db.relationship('User', uselist=False, back_populates='torrents')
-    main_category = db.relationship('MainCategory', uselist=False,
-                                    back_populates='torrents', lazy="joined")
-    sub_category = db.relationship('SubCategory', uselist=False, backref='torrents', lazy="joined",
-                                   primaryjoin=(
-                                       "and_(SubCategory.id == foreign(Torrent.sub_category_id), "
-                                       "SubCategory.main_category_id == Torrent.main_category_id)"))
-    info = db.relationship('TorrentInfo', uselist=False,
-                           cascade="all, delete-orphan", back_populates='torrent')
-    filelist = db.relationship('TorrentFilelist', uselist=False,
+    @declarative.declared_attr
+    def __table_args__(cls):
+        return (
+                Index(cls._table_prefix('uploader_flag_idx'), 'uploader_id', 'flags'),
+                ForeignKeyConstraint(
+                    ['main_category_id', 'sub_category_id'],
+                    [cls._table_prefix('sub_categories.main_category_id'),
+                        cls._table_prefix('sub_categories.id')]
+                ), {}
+            )
+
+    @declarative.declared_attr
+    def user(cls):
+        return db.relationship('User', uselist=False, back_populates=cls._table_prefix('torrents'))
+
+    @declarative.declared_attr
+    def main_category(cls):
+        return db.relationship(cls._flavor_prefix('MainCategory'), uselist=False,
+                               back_populates='torrents', lazy="joined")
+
+    @declarative.declared_attr
+    def sub_category(cls):
+        join_sql = ("and_({0}SubCategory.id == foreign({0}Torrent.sub_category_id), "
+                    "{0}SubCategory.main_category_id == {0}Torrent.main_category_id)")
+        return db.relationship(cls._flavor_prefix('SubCategory'), uselist=False,
+                               backref='torrents', lazy="joined",
+                               primaryjoin=join_sql.format(cls.__flavor__))
+
+    @declarative.declared_attr
+    def info(cls):
+        return db.relationship(cls._flavor_prefix('TorrentInfo'), uselist=False,
                                cascade="all, delete-orphan", back_populates='torrent')
-    stats = db.relationship('Statistic', uselist=False,
-                            cascade="all, delete-orphan", back_populates='torrent', lazy='joined')
-    trackers = db.relationship('TorrentTrackers', uselist=True, cascade="all, delete-orphan",
-                               lazy='joined', order_by='TorrentTrackers.order')
-    comments = db.relationship('Comment', uselist=True,
+
+    @declarative.declared_attr
+    def filelist(cls):
+        return db.relationship(cls._flavor_prefix('TorrentFilelist'), uselist=False,
+                               cascade="all, delete-orphan", back_populates='torrent')
+
+    @declarative.declared_attr
+    def stats(cls):
+        return db.relationship(cls._flavor_prefix('Statistic'), uselist=False,
+                               cascade="all, delete-orphan", back_populates='torrent',
+                               lazy='joined')
+
+    @declarative.declared_attr
+    def trackers(cls):
+        return db.relationship(cls._flavor_prefix('TorrentTrackers'), uselist=True,
+                               cascade="all, delete-orphan", lazy='joined',
+                               order_by=cls._flavor_prefix('TorrentTrackers.order'))
+
+    @declarative.declared_attr
+    def comments(cls):
+        return db.relationship(cls._flavor_prefix('Comment'), uselist=True,
                                cascade="all, delete-orphan")
 
     def __repr__(self):
@@ -211,44 +275,57 @@ class Torrent(db.Model):
         return cls.by_info_hash(info_hash_bytes)
 
 
-class TorrentNameSearch(FullText, Torrent):
-    __fulltext_columns__ = ('display_name',)
+class TorrentFilelistBase(DeclarativeHelperBase):
+    __tablename_base__ = 'torrents_filelist'
 
-
-class TorrentFilelist(db.Model):
-    __tablename__ = DB_TABLE_PREFIX + 'torrents_filelist'
     __table_args__ = {'mysql_row_format': 'COMPRESSED'}
 
-    torrent_id = db.Column(db.Integer, db.ForeignKey(
-        DB_TABLE_PREFIX + 'torrents.id', ondelete="CASCADE"), primary_key=True)
+    @declarative.declared_attr
+    def torrent_id(cls):
+        fk = db.ForeignKey(cls._table_prefix('torrents.id'), ondelete="CASCADE")
+        return db.Column(db.Integer, fk, primary_key=True)
+
     filelist_blob = db.Column(MediumBlobType, nullable=True)
 
-    torrent = db.relationship('Torrent', uselist=False, back_populates='filelist')
+    @declarative.declared_attr
+    def torrent(cls):
+        return db.relationship(cls._flavor_prefix('Torrent'), uselist=False,
+                               back_populates='filelist')
 
 
-class TorrentInfo(db.Model):
-    __tablename__ = DB_TABLE_PREFIX + 'torrents_info'
+class TorrentInfoBase(DeclarativeHelperBase):
+    __tablename_base__ = 'torrents_info'
+
     __table_args__ = {'mysql_row_format': 'COMPRESSED'}
 
-    torrent_id = db.Column(db.Integer, db.ForeignKey(
-        DB_TABLE_PREFIX + 'torrents.id', ondelete="CASCADE"), primary_key=True)
+    @declarative.declared_attr
+    def torrent_id(cls):
+        return db.Column(db.Integer, db.ForeignKey(
+            cls._table_prefix('torrents.id'), ondelete="CASCADE"), primary_key=True)
     info_dict = db.Column(MediumBlobType, nullable=True)
 
-    torrent = db.relationship('Torrent', uselist=False, back_populates='info')
+    @declarative.declared_attr
+    def torrent(cls):
+        return db.relationship(cls._flavor_prefix('Torrent'), uselist=False, back_populates='info')
 
 
-class Statistic(db.Model):
-    __tablename__ = DB_TABLE_PREFIX + 'statistics'
+class StatisticBase(DeclarativeHelperBase):
+    __tablename_base__ = 'statistics'
 
-    torrent_id = db.Column(db.Integer, db.ForeignKey(
-        DB_TABLE_PREFIX + 'torrents.id', ondelete="CASCADE"), primary_key=True)
+    @declarative.declared_attr
+    def torrent_id(cls):
+        fk = db.ForeignKey(cls._table_prefix('torrents.id'), ondelete="CASCADE")
+        return db.Column(db.Integer, fk, primary_key=True)
 
     seed_count = db.Column(db.Integer, default=0, nullable=False, index=True)
     leech_count = db.Column(db.Integer, default=0, nullable=False, index=True)
     download_count = db.Column(db.Integer, default=0, nullable=False, index=True)
     last_updated = db.Column(db.DateTime(timezone=False))
 
-    torrent = db.relationship('Torrent', uselist=False, back_populates='stats')
+    @declarative.declared_attr
+    def torrent(cls):
+        return db.relationship(cls._flavor_prefix('Torrent'), uselist=False,
+                               back_populates='stats')
 
 
 class Trackers(db.Model):
@@ -264,30 +341,43 @@ class Trackers(db.Model):
         return cls.query.filter_by(uri=uri).first()
 
 
-class TorrentTrackers(db.Model):
-    __tablename__ = DB_TABLE_PREFIX + 'torrent_trackers'
+class TorrentTrackersBase(DeclarativeHelperBase):
+    __tablename_base__ = 'torrent_trackers'
+
+    @declarative.declared_attr
+    def torrent_id(cls):
+        fk = db.ForeignKey(cls._table_prefix('torrents.id'), ondelete="CASCADE")
+        return db.Column(db.Integer, fk, primary_key=True)
+
+    @declarative.declared_attr
+    def tracker_id(cls):
+        fk = db.ForeignKey('trackers.id', ondelete="CASCADE")
+        return db.Column(db.Integer, fk, primary_key=True)
 
-    torrent_id = db.Column(db.Integer, db.ForeignKey(
-        DB_TABLE_PREFIX + 'torrents.id', ondelete="CASCADE"), primary_key=True)
-    tracker_id = db.Column(db.Integer, db.ForeignKey(
-        'trackers.id', ondelete="CASCADE"), primary_key=True)
     order = db.Column(db.Integer, nullable=False, index=True)
 
-    tracker = db.relationship('Trackers', uselist=False, lazy='joined')
+    @declarative.declared_attr
+    def tracker(cls):
+        return db.relationship('Trackers', uselist=False, lazy='joined')
 
     @classmethod
     def by_torrent_id(cls, torrent_id):
         return cls.query.filter_by(torrent_id=torrent_id).order_by(cls.order.desc())
 
 
-class MainCategory(db.Model):
-    __tablename__ = DB_TABLE_PREFIX + 'main_categories'
+class MainCategoryBase(DeclarativeHelperBase):
+    __tablename_base__ = 'main_categories'
 
     id = db.Column(db.Integer, primary_key=True)
     name = db.Column(db.String(length=64), nullable=False)
 
-    sub_categories = db.relationship('SubCategory', back_populates='main_category')
-    torrents = db.relationship('Torrent', back_populates='main_category')
+    @declarative.declared_attr
+    def sub_categories(cls):
+        return db.relationship(cls._flavor_prefix('SubCategory'), back_populates='main_category')
+
+    @declarative.declared_attr
+    def torrents(cls):
+        return db.relationship(cls._flavor_prefix('Torrent'), back_populates='main_category')
 
     def get_category_ids(self):
         return (self.id, 0)
@@ -301,18 +391,22 @@ class MainCategory(db.Model):
         return cls.query.get(id)
 
 
-class SubCategory(db.Model):
-    __tablename__ = DB_TABLE_PREFIX + 'sub_categories'
+class SubCategoryBase(DeclarativeHelperBase):
+    __tablename_base__ = 'sub_categories'
 
     id = db.Column(db.Integer, primary_key=True)
-    main_category_id = db.Column(db.Integer, db.ForeignKey(
-        DB_TABLE_PREFIX + 'main_categories.id'), primary_key=True)
+
+    @declarative.declared_attr
+    def main_category_id(cls):
+        fk = db.ForeignKey(cls._table_prefix('main_categories.id'))
+        return db.Column(db.Integer, fk, primary_key=True)
+
     name = db.Column(db.String(length=64), nullable=False)
 
-    main_category = db.relationship('MainCategory', uselist=False, back_populates='sub_categories')
-#    torrents = db.relationship('Torrent', back_populates='sub_category'),
-#        primaryjoin="and_(Torrent.sub_category_id == foreign(SubCategory.id), "
-#                    "Torrent.main_category_id == SubCategory.main_category_id)")
+    @declarative.declared_attr
+    def main_category(cls):
+        return db.relationship(cls._flavor_prefix('MainCategory'), uselist=False,
+                               back_populates='sub_categories')
 
     def get_category_ids(self):
         return (self.main_category_id, self.id)
@@ -326,17 +420,27 @@ class SubCategory(db.Model):
         return cls.query.get((sub_cat_id, main_cat_id))
 
 
-class Comment(db.Model):
-    __tablename__ = DB_TABLE_PREFIX + 'comments'
+class CommentBase(DeclarativeHelperBase):
+    __tablename_base__ = 'comments'
 
     id = db.Column(db.Integer, primary_key=True)
-    torrent_id = db.Column(db.Integer, db.ForeignKey(
-        DB_TABLE_PREFIX + 'torrents.id', ondelete='CASCADE'), nullable=False)
-    user_id = db.Column(db.Integer, db.ForeignKey('users.id', ondelete='CASCADE'))
+
+    @declarative.declared_attr
+    def torrent_id(cls):
+        return db.Column(db.Integer, db.ForeignKey(
+                cls._table_prefix('torrents.id'), ondelete='CASCADE'), nullable=False)
+
+    @declarative.declared_attr
+    def user_id(cls):
+        return db.Column(db.Integer, db.ForeignKey('users.id', ondelete='CASCADE'))
+
     created_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow)
     text = db.Column(db.String(length=255, collation=COL_UTF8MB4_BIN), nullable=False)
 
-    user = db.relationship('User', uselist=False, back_populates='comments', lazy="joined")
+    @declarative.declared_attr
+    def user(cls):
+        return db.relationship('User', uselist=False,
+                               back_populates=cls._table_prefix('comments'), lazy="joined")
 
     def __repr__(self):
         return '<Comment %r>' % self.id
@@ -376,9 +480,11 @@ class User(db.Model):
     last_login_date = db.Column(db.DateTime(timezone=False), default=None, nullable=True)
     last_login_ip = db.Column(db.Binary(length=16), default=None, nullable=True)
 
-    torrents = db.relationship('Torrent', back_populates='user', lazy='dynamic')
-    comments = db.relationship('Comment', back_populates='user', lazy='dynamic')
-    # session = db.relationship('Session', uselist=False, back_populates='user')
+    nyaa_torrents = db.relationship('NyaaTorrent', back_populates='user', lazy='dynamic')
+    nyaa_comments = db.relationship('NyaaComment', back_populates='user', lazy='dynamic')
+
+    sukebei_torrents = db.relationship('SukebeiTorrent', back_populates='user', lazy='dynamic')
+    sukebei_comments = db.relationship('SukebeiComment', back_populates='user', lazy='dynamic')
 
     def __init__(self, username, email, password):
         self.username = username
@@ -464,12 +570,118 @@ class User(db.Model):
         return self.level >= UserLevelType.TRUSTED
 
 
-# class Session(db.Model):
-#    __tablename__ = 'sessions'
-#
-#    session_id = db.Column(db.Integer, primary_key=True)
-#    user_id = db.Column(db.Integer, db.ForeignKey('users.id'), nullable=False)
-#    login_ip = db.Column(db.Binary(length=16), nullable=True)
-#    login_date = db.Column(db.DateTime(timezone=False), nullable=True)
-#
-#    user = db.relationship('User', back_populates='session')
+# Actually declare our site-specific classes
+
+# Torrent
+class NyaaTorrent(TorrentBase, db.Model):
+    __flavor__ = 'Nyaa'
+
+
+class SukebeiTorrent(TorrentBase, db.Model):
+    __flavor__ = 'Sukebei'
+
+
+# Fulltext models for MySQL
+if app.config['USE_MYSQL']:
+    class NyaaTorrentNameSearch(FullText, NyaaTorrent):
+        __fulltext_columns__ = ('display_name',)
+        __table_args__ = {'extend_existing': True}
+
+    class SukebeiTorrentNameSearch(FullText, SukebeiTorrent):
+        __fulltext_columns__ = ('display_name',)
+        __table_args__ = {'extend_existing': True}
+else:
+    # Bogus classes for Sqlite
+    class NyaaTorrentNameSearch(object):
+        pass
+
+    class SukebeiTorrentNameSearch(object):
+        pass
+
+
+# TorrentFilelist
+class NyaaTorrentFilelist(TorrentFilelistBase, db.Model):
+    __flavor__ = 'Nyaa'
+
+
+class SukebeiTorrentFilelist(TorrentFilelistBase, db.Model):
+    __flavor__ = 'Sukebei'
+
+
+# TorrentInfo
+class NyaaTorrentInfo(TorrentInfoBase, db.Model):
+    __flavor__ = 'Nyaa'
+
+
+class SukebeiTorrentInfo(TorrentInfoBase, db.Model):
+    __flavor__ = 'Sukebei'
+
+
+# Statistic
+class NyaaStatistic(StatisticBase, db.Model):
+    __flavor__ = 'Nyaa'
+
+
+class SukebeiStatistic(StatisticBase, db.Model):
+    __flavor__ = 'Sukebei'
+
+
+# TorrentTrackers
+class NyaaTorrentTrackers(TorrentTrackersBase, db.Model):
+    __flavor__ = 'Nyaa'
+
+
+class SukebeiTorrentTrackers(TorrentTrackersBase, db.Model):
+    __flavor__ = 'Sukebei'
+
+
+# MainCategory
+class NyaaMainCategory(MainCategoryBase, db.Model):
+    __flavor__ = 'Nyaa'
+
+
+class SukebeiMainCategory(MainCategoryBase, db.Model):
+    __flavor__ = 'Sukebei'
+
+
+# SubCategory
+class NyaaSubCategory(SubCategoryBase, db.Model):
+    __flavor__ = 'Nyaa'
+
+
+class SukebeiSubCategory(SubCategoryBase, db.Model):
+    __flavor__ = 'Sukebei'
+
+
+# Comment
+class NyaaComment(CommentBase, db.Model):
+    __flavor__ = 'Nyaa'
+
+
+class SukebeiComment(CommentBase, db.Model):
+    __flavor__ = 'Sukebei'
+
+
+# Choose our defaults for models.Torrent etc
+if app.config['SITE_FLAVOR'] == 'nyaa':
+    Torrent = NyaaTorrent
+    TorrentFilelist = NyaaTorrentFilelist
+    TorrentInfo = NyaaTorrentInfo
+    Statistic = NyaaStatistic
+    TorrentTrackers = NyaaTorrentTrackers
+    MainCategory = NyaaMainCategory
+    SubCategory = NyaaSubCategory
+    Comment = NyaaComment
+
+    TorrentNameSearch = NyaaTorrentNameSearch
+elif app.config['SITE_FLAVOR'] == 'sukebei':
+    Torrent = SukebeiTorrent
+    TorrentFilelist = SukebeiTorrentFilelist
+    TorrentInfo = SukebeiTorrentInfo
+    Statistic = SukebeiStatistic
+    TorrentTrackers = SukebeiTorrentTrackers
+    MainCategory = SukebeiMainCategory
+    SubCategory = SukebeiSubCategory
+    Comment = SukebeiComment
+
+    TorrentNameSearch = SukebeiTorrentNameSearch

From 1b3e05339d6129fbe2ceace866b757aaf8783c33 Mon Sep 17 00:00:00 2001
From: TheAMM <the.actual.amm@gmail.com>
Date: Sat, 27 May 2017 17:39:13 +0300
Subject: [PATCH 15/30] Update db_create.py to add categories for both Nyaa and
 Sukebei

Also includes a reminder to stamp the database version for Alembic.
---
 db_create.py | 73 +++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 49 insertions(+), 24 deletions(-)

diff --git a/db_create.py b/db_create.py
index d881377..06a1783 100755
--- a/db_create.py
+++ b/db_create.py
@@ -1,35 +1,60 @@
+#!/usr/bin/env python3
 import sys
+import sqlalchemy
 from nyaa import app, db, models
 
-# Create tables
+NYAA_CATEGORIES = [
+    ('Anime', ['Anime Music Video', 'English-translated', 'Non-English-translated', 'Raw']),
+    ('Audio', ['Lossless', 'Lossy']),
+    ('Literature', ['English-translated', 'Non-English-translated', 'Raw']),
+    ('Live Action', ['English-translated', 'Idol/Promotional Video', 'Non-English-translated', 'Raw']),
+    ('Pictures', ['Graphics', 'Photos']),
+    ('Software', ['Applications', 'Games']),
+]
 
-db.create_all()
 
-# Insert categories and insert if it doesn't eixst
-existing_cats = models.MainCategory.query.all()
-if not existing_cats:
-    if app.config['SITE_FLAVOR'] == 'nyaa':
-        CATEGORIES = [
-            ('Anime', ['Anime Music Video', 'English-translated', 'Non-English-translated', 'Raw']),
-            ('Audio', ['Lossless', 'Lossy']),
-            ('Literature', ['English-translated', 'Non-English-translated', 'Raw']),
-            ('Live Action', ['English-translated', 'Idol/Promotional Video', 'Non-English-translated', 'Raw']),
-            ('Pictures', ['Graphics', 'Photos']),
-            ('Software', ['Applications', 'Games']),
-        ]
-    elif app.config['SITE_FLAVOR'] == 'sukebei':
-        CATEGORIES = [
-            ('Art', ['Anime', 'Doujinshi', 'Games', 'Manga', 'Pictures']),
-            ('Real Life', ['Photobooks / Pictures', 'Videos']),
-        ]
-    else:
-        CATEGORIES = []
+SUKEBEI_CATEGORIES = [
+    ('Art', ['Anime', 'Doujinshi', 'Games', 'Manga', 'Pictures']),
+    ('Real Life', ['Photobooks / Pictures', 'Videos']),
+]
 
-    for main_cat_name, sub_cat_names in CATEGORIES:
-        main_cat = models.MainCategory(name=main_cat_name)
+
+def add_categories(categories, main_class, sub_class):
+    for main_cat_name, sub_cat_names in categories:
+        main_cat = main_class(name=main_cat_name)
         for i, sub_cat_name in enumerate(sub_cat_names):
             # Composite keys can't autoincrement, set sub_cat id manually (1-index)
-            sub_cat = models.SubCategory(id=i+1, name=sub_cat_name, main_category=main_cat)
+            sub_cat = sub_class(id=i+1, name=sub_cat_name, main_category=main_cat)
         db.session.add(main_cat)
 
+
+if __name__ == '__main__':
+    # Test for the user table, assume db is empty if it's not created
+    database_empty = False
+    try:
+        models.User.query.first()
+    except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
+        database_empty = True
+
+
+    print('Creating all tables...')
+    db.create_all()
+
+
+    nyaa_category_test = models.NyaaMainCategory.query.first()
+    if not nyaa_category_test:
+        print('Adding Nyaa categories...')
+        add_categories(NYAA_CATEGORIES, models.NyaaMainCategory, models.NyaaSubCategory)
+
+    sukebei_category_test = models.SukebeiMainCategory.query.first()
+    if not sukebei_category_test:
+        print('Adding Sukebei categories...')
+        add_categories(SUKEBEI_CATEGORIES, models.SukebeiMainCategory, models.SukebeiSubCategory)
+
     db.session.commit()
+
+    if database_empty:
+        print('Remember to run the following to mark the database up-to-date for Alembic:')
+        print('./db_migrate.py db stamp head')
+        # Technically we should be able to do this here, but when you have
+        # Flask-Migrate and Flask-SQA and everything... I didn't get it working.
\ No newline at end of file

From 6f9341d3f1aefab00f7905d2f59290e57e1895f2 Mon Sep 17 00:00:00 2001
From: TheAMM <the.actual.amm@gmail.com>
Date: Sat, 27 May 2017 17:40:53 +0300
Subject: [PATCH 16/30] Add initial database state for Alembic, update existing
 migrations

Existing migrations will apply changes to both Nyaa and Sukebei
(adjusted manually - in the future Alembic should be able to create operations
for both, thanks to having both models loaded)

Added a down_revision for previously-earliest revision.
---
 .../3001f79b7722_add_torrents.uploader_ip.py  |  17 +-
 .../97ddefed1834_initial_database_state.py    | 166 ++++++++++++++++++
 .../versions/d0eeb8049623_add_comments.py     |  41 ++---
 3 files changed, 189 insertions(+), 35 deletions(-)
 create mode 100644 migrations/versions/97ddefed1834_initial_database_state.py

diff --git a/migrations/versions/3001f79b7722_add_torrents.uploader_ip.py b/migrations/versions/3001f79b7722_add_torrents.uploader_ip.py
index 152c440..03ed87c 100644
--- a/migrations/versions/3001f79b7722_add_torrents.uploader_ip.py
+++ b/migrations/versions/3001f79b7722_add_torrents.uploader_ip.py
@@ -11,20 +11,19 @@ import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
 revision = '3001f79b7722'
-down_revision = None
+down_revision = '97ddefed1834'
 branch_labels = None
 depends_on = None
 
+TABLE_PREFIXES = ('nyaa', 'sukebei')
+
 
 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.add_column('nyaa_torrents', sa.Column('uploader_ip', sa.Binary(), nullable=True))
-    op.add_column('sukebei_torrents', sa.Column('uploader_ip', sa.Binary(), nullable=True))
-    # ### end Alembic commands ###
+    for prefix in TABLE_PREFIXES:
+        op.add_column(prefix + '_torrents', sa.Column('uploader_ip', sa.Binary(), nullable=True))
+        # ### end Alembic commands ###
 
 
 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_column('nyaa_torrents', 'uploader_ip')
-    op.drop_column('sukebei_torrents', 'uploader_ip')
-    # ### end Alembic commands ###
+    for prefix in TABLE_PREFIXES:
+        op.drop_column(prefix + '_torrents', 'uploader_ip')
diff --git a/migrations/versions/97ddefed1834_initial_database_state.py b/migrations/versions/97ddefed1834_initial_database_state.py
new file mode 100644
index 0000000..fcb1dcb
--- /dev/null
+++ b/migrations/versions/97ddefed1834_initial_database_state.py
@@ -0,0 +1,166 @@
+"""Initial database state
+
+Revision ID: 97ddefed1834
+Revises: 
+Create Date: 2017-05-26 18:46:14.440040
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlalchemy_utils
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = '97ddefed1834'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+TABLE_PREFIXES = ('nyaa', 'sukebei')
+
+def upgrade():
+    # Shared tables
+    op.create_table('users',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('username', sa.String(length=32, collation='ascii_general_ci'), nullable=False),
+        sa.Column('email', sqlalchemy_utils.types.email.EmailType(length=255), nullable=True),
+        
+        # These are actually PasswordType, UserStatusType and UserLevelType,
+        # but database-wise binary and integers are what's being used
+        sa.Column('password_hash', sa.Binary(length=255), nullable=False),
+        sa.Column('status', sa.Integer(), nullable=False),
+        sa.Column('level', sa.Integer(), nullable=False),
+
+        sa.Column('created_time', sa.DateTime(), nullable=True),
+        sa.Column('last_login_date', sa.DateTime(), nullable=True),
+        sa.Column('last_login_ip', sa.Binary(), nullable=True),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('email'),
+        sa.UniqueConstraint('username')
+    )
+
+    op.create_table('trackers',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('uri', sa.String(length=255, collation='utf8_general_ci'), nullable=False),
+        sa.Column('disabled', sa.Boolean(), nullable=False),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('uri')
+    )
+
+    # Nyaa and Sukebei
+    for prefix in TABLE_PREFIXES:
+        # Main categories
+        op.create_table(prefix + '_main_categories',
+            sa.Column('id', sa.Integer(), nullable=False),
+            sa.Column('name', sa.String(length=64), nullable=False),
+            sa.PrimaryKeyConstraint('id')
+        )
+        # Sub categories
+        op.create_table(prefix + '_sub_categories',
+            sa.Column('id', sa.Integer(), nullable=False),
+            sa.Column('main_category_id', sa.Integer(), nullable=False),
+            sa.Column('name', sa.String(length=64), nullable=False),
+            sa.ForeignKeyConstraint(['main_category_id'], [prefix + '_main_categories.id'], ),
+            sa.PrimaryKeyConstraint('id', 'main_category_id')
+        )
+        # Main torrent table
+        op.create_table(prefix + '_torrents',
+            sa.Column('id', sa.Integer(), nullable=False),
+            sa.Column('info_hash', sa.BINARY(length=20), nullable=False),
+            sa.Column('display_name', sa.String(length=255, collation='utf8_general_ci'), nullable=False),
+            sa.Column('torrent_name', sa.String(length=255), nullable=False),
+            sa.Column('information', sa.String(length=255), nullable=False),
+            sa.Column('description', mysql.TEXT(collation='utf8mb4_bin'), nullable=False),
+            sa.Column('filesize', sa.BIGINT(), nullable=False),
+            sa.Column('encoding', sa.String(length=32), nullable=False),
+            sa.Column('flags', sa.Integer(), nullable=False),
+            sa.Column('uploader_id', sa.Integer(), nullable=True),
+            sa.Column('has_torrent', sa.Boolean(), nullable=False),
+            sa.Column('created_time', sa.DateTime(), nullable=False),
+            sa.Column('updated_time', sa.DateTime(), nullable=False),
+            sa.Column('main_category_id', sa.Integer(), nullable=False),
+            sa.Column('sub_category_id', sa.Integer(), nullable=False),
+            sa.Column('redirect', sa.Integer(), nullable=True),
+            sa.ForeignKeyConstraint(['main_category_id', 'sub_category_id'], [prefix + '_sub_categories.main_category_id', prefix + '_sub_categories.id'], ),
+            sa.ForeignKeyConstraint(['main_category_id'], [prefix + '_main_categories.id'], ),
+            sa.ForeignKeyConstraint(['redirect'], [prefix + '_torrents.id'], ),
+            sa.ForeignKeyConstraint(['uploader_id'], ['users.id'], ),
+            sa.PrimaryKeyConstraint('id')
+        )
+        op.create_index(op.f('ix_' + prefix + '_torrents_display_name'), prefix + '_torrents', ['display_name'], unique=False)
+        op.create_index(op.f('ix_' + prefix + '_torrents_filesize'), prefix + '_torrents', ['filesize'], unique=False)
+        op.create_index(op.f('ix_' + prefix + '_torrents_flags'), prefix + '_torrents', ['flags'], unique=False)
+        op.create_index(op.f('ix_' + prefix + '_torrents_info_hash'), prefix + '_torrents', ['info_hash'], unique=True)
+        op.create_index(prefix + '_uploader_flag_idx', prefix + '_torrents', ['uploader_id', 'flags'], unique=False)
+        
+        # Statistics for torrents
+        op.create_table(prefix + '_statistics',
+            sa.Column('torrent_id', sa.Integer(), nullable=False),
+            sa.Column('seed_count', sa.Integer(), nullable=False),
+            sa.Column('leech_count', sa.Integer(), nullable=False),
+            sa.Column('download_count', sa.Integer(), nullable=False),
+            sa.Column('last_updated', sa.DateTime(), nullable=True),
+            sa.ForeignKeyConstraint(['torrent_id'], [prefix + '_torrents.id'], ondelete='CASCADE'),
+            sa.PrimaryKeyConstraint('torrent_id')
+        )
+        op.create_index(op.f('ix_' + prefix + '_statistics_download_count'), prefix + '_statistics', ['download_count'], unique=False)
+        op.create_index(op.f('ix_' + prefix + '_statistics_leech_count'), prefix + '_statistics', ['leech_count'], unique=False)
+        op.create_index(op.f('ix_' + prefix + '_statistics_seed_count'), prefix + '_statistics', ['seed_count'], unique=False)
+        
+        # Trackers relationships for torrents
+        op.create_table(prefix + '_torrent_trackers',
+            sa.Column('torrent_id', sa.Integer(), nullable=False),
+            sa.Column('tracker_id', sa.Integer(), nullable=False),
+            sa.Column('order', sa.Integer(), nullable=False),
+            sa.ForeignKeyConstraint(['torrent_id'], [prefix + '_torrents.id'], ondelete='CASCADE'),
+            sa.ForeignKeyConstraint(['tracker_id'], ['trackers.id'], ondelete='CASCADE'),
+            sa.PrimaryKeyConstraint('torrent_id', 'tracker_id')
+        )
+        op.create_index(op.f('ix_' + prefix + '_torrent_trackers_order'), prefix + '_torrent_trackers', ['order'], unique=False)
+        
+        # Torrent filelists
+        op.create_table(prefix + '_torrents_filelist',
+            sa.Column('torrent_id', sa.Integer(), nullable=False),
+            sa.Column('filelist_blob', mysql.MEDIUMBLOB(), nullable=True),
+            sa.ForeignKeyConstraint(['torrent_id'], [prefix + '_torrents.id'], ondelete='CASCADE'),
+            sa.PrimaryKeyConstraint('torrent_id'),
+            mysql_row_format='COMPRESSED'
+        )
+        
+        # Torrent info_dicts
+        op.create_table(prefix + '_torrents_info',
+            sa.Column('torrent_id', sa.Integer(), nullable=False),
+            sa.Column('info_dict', mysql.MEDIUMBLOB(), nullable=True),
+            sa.ForeignKeyConstraint(['torrent_id'], [prefix + '_torrents.id'], ondelete='CASCADE'),
+            sa.PrimaryKeyConstraint('torrent_id'),
+            mysql_row_format='COMPRESSED'
+        )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # Note: this may fail. It's better to just drop all tables instead (or reset the database)
+    
+    # Nyaa and Sukebei
+    for prefix in TABLE_PREFIXES:
+        op.drop_table(prefix + '_torrents_info')
+        op.drop_table(prefix + '_torrents_filelist')
+        op.drop_index(op.f('ix_' + prefix + '_torrent_trackers_order'), table_name=prefix + '_torrent_trackers')
+        op.drop_table(prefix + '_torrent_trackers')
+        op.drop_index(op.f('ix_' + prefix + '_statistics_seed_count'), table_name=prefix + '_statistics')
+        op.drop_index(op.f('ix_' + prefix + '_statistics_leech_count'), table_name=prefix + '_statistics')
+        op.drop_index(op.f('ix_' + prefix + '_statistics_download_count'), table_name=prefix + '_statistics')
+        op.drop_table(prefix + '_statistics')
+        op.drop_table(prefix + '_torrents')
+        op.drop_index(prefix + '_uploader_flag_idx', table_name=prefix + '_torrents')
+        op.drop_index(op.f('ix_' + prefix + '_torrents_info_hash'), table_name=prefix + '_torrents')
+        op.drop_index(op.f('ix_' + prefix + '_torrents_flags'), table_name=prefix + '_torrents')
+        op.drop_index(op.f('ix_' + prefix + '_torrents_filesize'), table_name=prefix + '_torrents')
+        op.drop_index(op.f('ix_' + prefix + '_torrents_display_name'), table_name=prefix + '_torrents')
+        op.drop_table(prefix + '_sub_categories')
+        op.drop_table(prefix + '_main_categories')
+
+    # Shared tables
+    op.drop_table('users')
+    op.drop_table('trackers')
+    # ### end Alembic commands ###
diff --git a/migrations/versions/d0eeb8049623_add_comments.py b/migrations/versions/d0eeb8049623_add_comments.py
index f6fab5b..47e2a7a 100644
--- a/migrations/versions/d0eeb8049623_add_comments.py
+++ b/migrations/versions/d0eeb8049623_add_comments.py
@@ -15,34 +15,23 @@ down_revision = '3001f79b7722'
 branch_labels = None
 depends_on = None
 
+TABLE_PREFIXES = ('nyaa', 'sukebei')
+
 
 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('nyaa_comments',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('torrent_id', sa.Integer(), nullable=False),
-    sa.Column('user_id', sa.Integer(), nullable=True),
-    sa.Column('created_time', sa.DateTime(), nullable=True),
-    sa.Column('text', sa.String(length=255, collation='utf8mb4_bin'), nullable=False),
-    sa.ForeignKeyConstraint(['torrent_id'], ['nyaa_torrents.id'], ondelete='CASCADE'),
-    sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('sukebei_comments',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('torrent_id', sa.Integer(), nullable=False),
-    sa.Column('user_id', sa.Integer(), nullable=True),
-    sa.Column('created_time', sa.DateTime(), nullable=True),
-    sa.Column('text', sa.String(length=255, collation='utf8mb4_bin'), nullable=False),
-    sa.ForeignKeyConstraint(['torrent_id'], ['sukebei_torrents.id'], ondelete='CASCADE'),
-    sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'),
-    sa.PrimaryKeyConstraint('id')
-    )
-    # ### end Alembic commands ###
+    for prefix in TABLE_PREFIXES:
+        op.create_table(prefix + '_comments',
+            sa.Column('id', sa.Integer(), nullable=False),
+            sa.Column('torrent_id', sa.Integer(), nullable=False),
+            sa.Column('user_id', sa.Integer(), nullable=True),
+            sa.Column('created_time', sa.DateTime(), nullable=True),
+            sa.Column('text', sa.String(length=255, collation='utf8mb4_bin'), nullable=False),
+            sa.ForeignKeyConstraint(['torrent_id'], [prefix + '_torrents.id'], ondelete='CASCADE'),
+            sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'),
+            sa.PrimaryKeyConstraint('id')
+        )
 
 
 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table('nyaa_comments')
-    op.drop_table('sukebei_comments')
-    # ### end Alembic commands ###
+    for prefix in TABLE_PREFIXES:
+        op.drop_table(prefix + '_comments')

From eac41d561ab5fd7dff27157b58995203d00e8248 Mon Sep 17 00:00:00 2001
From: TheAMM <the.actual.amm@gmail.com>
Date: Sat, 27 May 2017 17:43:27 +0300
Subject: [PATCH 17/30] Fix db_migrate.py shebang for venvs

Allows one to call it using ./db_migrate.py without errors
---
 db_migrate.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db_migrate.py b/db_migrate.py
index 8d4f8f0..1f480e7 100755
--- a/db_migrate.py
+++ b/db_migrate.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 from nyaa import app, db
 from flask_script import Manager

From 260dc2cdc8c12e26661fb0e8ecd11af04bcd884c Mon Sep 17 00:00:00 2001
From: TheAMM <the.actual.amm@gmail.com>
Date: Sat, 27 May 2017 21:33:20 +0300
Subject: [PATCH 18/30] Forbid caching when debugging

---
 nyaa/__init__.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/nyaa/__init__.py b/nyaa/__init__.py
index d8c04bf..9ef9bfb 100644
--- a/nyaa/__init__.py
+++ b/nyaa/__init__.py
@@ -22,6 +22,15 @@ if app.config['DEBUG']:
     app.config['DEBUG_TB_INTERCEPT_REDIRECTS'] = False
     toolbar = DebugToolbarExtension(app)
     app.logger.setLevel(logging.DEBUG)
+
+    # Forbid caching
+    @app.after_request
+    def forbid_cache(request):
+        request.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate, max-age=0'
+        request.headers['Pragma'] = 'no-cache'
+        request.headers['Expires'] = '0'
+        return request
+
 else:
     app.logger.setLevel(logging.WARNING)
 

From 16194a12a39a911bb03bfd3a20797456c7c98185 Mon Sep 17 00:00:00 2001
From: TheAMM <the.actual.amm@gmail.com>
Date: Sat, 27 May 2017 21:37:26 +0300
Subject: [PATCH 19/30] Fix typo in ES search sort keys

---
 nyaa/search.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nyaa/search.py b/nyaa/search.py
index 4db2879..976ad5c 100644
--- a/nyaa/search.py
+++ b/nyaa/search.py
@@ -25,7 +25,7 @@ def search_elastic(term='', user=None, sort='id', order='desc',
         'id': 'id',
         'size': 'filesize',
         # 'name': 'display_name',  # This is slow and buggy
-        'comments': 'comments_count',
+        'comments': 'comment_count',
         'seeders': 'seed_count',
         'leechers': 'leech_count',
         'downloads': 'download_count'

From 2d04cf3525a359c546915a7894e5fb5a17297c0a Mon Sep 17 00:00:00 2001
From: TheAMM <the.actual.amm@gmail.com>
Date: Sat, 27 May 2017 22:46:35 +0300
Subject: [PATCH 20/30] Update comment count migration, calculate initial
 counts during upgrade

How fancy!
---
 ...ceb2cb4d7c_add_comment_count_to_torrent.py | 32 ++++++++++++-------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/migrations/versions/2bceb2cb4d7c_add_comment_count_to_torrent.py b/migrations/versions/2bceb2cb4d7c_add_comment_count_to_torrent.py
index aa88a8e..3114e27 100644
--- a/migrations/versions/2bceb2cb4d7c_add_comment_count_to_torrent.py
+++ b/migrations/versions/2bceb2cb4d7c_add_comment_count_to_torrent.py
@@ -15,30 +15,38 @@ down_revision = 'd0eeb8049623'
 branch_labels = None
 depends_on = None
 
+COMMENT_UPDATE_SQL = '''UPDATE {0}_torrents
+                        SET comment_count = (
+                            SELECT COUNT(*) FROM {0}_comments
+                            WHERE {0}_torrents.id = {0}_comments.torrent_id
+                        );'''
+
 
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
     op.add_column('nyaa_torrents', sa.Column('comment_count', sa.Integer(), nullable=False))
     op.create_index(op.f('ix_nyaa_torrents_comment_count'), 'nyaa_torrents', ['comment_count'], unique=False)
 
-    try:
-        op.add_column('sukebei_torrents', sa.Column('comment_count', sa.Integer(), nullable=False))
-        op.create_index(op.f('ix_sukebei_torrents_comment_count'), 'sukebei_torrents', ['comment_count'], unique=False)
-    except (sa.exc.OperationalError, sa.exc.ProgrammingError):
-        print('Could not upgrade sukebei!')
-
+    op.add_column('sukebei_torrents', sa.Column('comment_count', sa.Integer(), nullable=False))
+    op.create_index(op.f('ix_sukebei_torrents_comment_count'), 'sukebei_torrents', ['comment_count'], unique=False)
     # ### end Alembic commands ###
 
+    connection = op.get_bind()
+
+    print('Updating comment counts on nyaa_torrents...')
+    connection.execute(sa.sql.text(COMMENT_UPDATE_SQL.format('nyaa')))
+    print('Done.')
+
+    print('Updating comment counts on sukebei_torrents...')
+    connection.execute(sa.sql.text(COMMENT_UPDATE_SQL.format('sukebei')))
+    print('Done.')
+
 
 def downgrade():
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_index(op.f('ix_nyaa_torrents_comment_count'), table_name='nyaa_torrents')
     op.drop_column('nyaa_torrents', 'comment_count')
 
-    try:
-        op.drop_index(op.f('ix_sukebei_torrents_comment_count'), table_name='sukebei_torrents')
-        op.drop_column('sukebei_torrents', 'comment_count')
-    except (sa.exc.OperationalError, sa.exc.ProgrammingError):
-        print('Could not downgrade sukebei!')
-
+    op.drop_index(op.f('ix_sukebei_torrents_comment_count'), table_name='sukebei_torrents')
+    op.drop_column('sukebei_torrents', 'comment_count')
     # ### end Alembic commands ###

From 79dae38f37ffe7923d47042f83dfe5b9e4857975 Mon Sep 17 00:00:00 2001
From: TheAMM <the.actual.amm@gmail.com>
Date: Sat, 27 May 2017 23:19:38 +0300
Subject: [PATCH 21/30] Patch db_migrate.py to always use the 'db' command

This means you can (have to) use the Flask-Migrate commands directly:
"./db_migrate.py stamp head" etc instead of "./db_migrate.py db stamp head"
---
 db_create.py  | 2 +-
 db_migrate.py | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/db_create.py b/db_create.py
index 06a1783..8d2eed8 100755
--- a/db_create.py
+++ b/db_create.py
@@ -55,6 +55,6 @@ if __name__ == '__main__':
 
     if database_empty:
         print('Remember to run the following to mark the database up-to-date for Alembic:')
-        print('./db_migrate.py db stamp head')
+        print('./db_migrate.py stamp head')
         # Technically we should be able to do this here, but when you have
         # Flask-Migrate and Flask-SQA and everything... I didn't get it working.
\ No newline at end of file
diff --git a/db_migrate.py b/db_migrate.py
index 1f480e7..07989d1 100755
--- a/db_migrate.py
+++ b/db_migrate.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
+import sys
 from nyaa import app, db
 from flask_script import Manager
 from flask_migrate import Migrate, MigrateCommand
@@ -10,4 +11,12 @@ manager = Manager(app)
 manager.add_command("db", MigrateCommand)
 
 if __name__ == "__main__":
+	# Patch sys.argv to default to 'db'
+	argv_contents = sys.argv[:]
+	
+	sys.argv.clear()
+	sys.argv.append(argv_contents[0])
+	sys.argv.append('db')
+	sys.argv.extend(argv_contents[1:])
+
 	manager.run()

From 0cbed3870e9a65fd6a37d0df8c72d83858b4fb18 Mon Sep 17 00:00:00 2001
From: TheAMM <the.actual.amm@gmail.com>
Date: Sun, 28 May 2017 02:12:48 +0300
Subject: [PATCH 22/30] Update import_to_es.py to index both torrent flavors,
 rename sync_es config

import_to_es.py will now also SHOW MASTER STATUS now
Changed progressbar dependency
---
 config_es_sync.json         | 11 ------
 es_sync_config.example.json | 11 ++++++
 import_to_es.py             | 69 ++++++++++++++++++++++++++-----------
 requirements.txt            |  2 +-
 4 files changed, 60 insertions(+), 33 deletions(-)
 delete mode 100644 config_es_sync.json
 create mode 100644 es_sync_config.example.json

diff --git a/config_es_sync.json b/config_es_sync.json
deleted file mode 100644
index d2cb889..0000000
--- a/config_es_sync.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
-"save_loc": "/tmp/pos.json",
-"mysql_host": "127.0.0.1",
-"mysql_port": 13306,
-"mysql_user": "root",
-"mysql_password": "dunnolol",
-"database": "nyaav2",
-"internal_queue_depth": 10000,
-"es_chunk_size": 10000,
-"flush_interval": 5
-}
diff --git a/es_sync_config.example.json b/es_sync_config.example.json
new file mode 100644
index 0000000..b2dc524
--- /dev/null
+++ b/es_sync_config.example.json
@@ -0,0 +1,11 @@
+{
+    "save_loc": "/tmp/pos.json",
+    "mysql_host": "127.0.0.1",
+    "mysql_port": 3306,
+    "mysql_user": "nyaa",
+    "mysql_password": "some_password",
+    "database": "nyaav2",
+    "internal_queue_depth": 10000,
+    "es_chunk_size": 10000,
+    "flush_interval": 5
+}
diff --git a/import_to_es.py b/import_to_es.py
index 6832781..b3bc3b1 100755
--- a/import_to_es.py
+++ b/import_to_es.py
@@ -5,22 +5,16 @@ which is assumed to already exist.
 This is a one-shot deal, so you'd either need to complement it
 with a cron job or some binlog-reading thing (TODO)
 """
-from nyaa import app
-from nyaa.models import Torrent
+import sys
+import json
+from nyaa import app, db, models
+
 from elasticsearch import Elasticsearch
 from elasticsearch.client import IndicesClient
 from elasticsearch import helpers
-import progressbar
-import sys
 
-bar = progressbar.ProgressBar(
-        max_value=Torrent.query.count(),
-        widgets=[
-            progressbar.SimpleProgress(),
-            ' [', progressbar.Timer(), '] ',
-            progressbar.Bar(),
-            ' (', progressbar.ETA(), ') ',
-            ])
+# This should be progressbar33
+import progressbar
 
 es = Elasticsearch(timeout=30)
 ic = IndicesClient(es)
@@ -32,11 +26,11 @@ ic = IndicesClient(es)
 # we don't want to reindex all the user's torrents just because they
 # changed their name, and we don't really want to FTS search on the user anyway.
 # Maybe it's more convenient to derefence though.
-def mk_es(t):
+def mk_es(t, index_name):
     return {
         "_id": t.id,
         "_type": "torrent",
-        "_index": app.config['ES_INDEX_NAME'],
+        "_index": index_name,
         "_source": {
             # we're also indexing the id as a number so you can
             # order by it. seems like this is just equivalent to
@@ -73,7 +67,7 @@ def mk_es(t):
 # page through an sqlalchemy query, like the per_fetch but
 # doesn't break the eager joins its doing against the stats table.
 # annoying that this isn't built in somehow.
-def page_query(query, limit=sys.maxsize, batch_size=10000):
+def page_query(query, limit=sys.maxsize, batch_size=10000, progress_bar=None):
     start = 0
     while True:
         # XXX very inelegant way to do this, i'm confus
@@ -89,13 +83,46 @@ def page_query(query, limit=sys.maxsize, batch_size=10000):
             yield(thing)
         if not had_things or stop == limit:
             break
-        bar.update(start)
+        if progress_bar:
+            progress_bar.update(start)
         start = min(limit, start + batch_size)
 
-# turn off refreshes while bulk loading
-ic.put_settings(body={'index': {'refresh_interval': '-1'}}, index=app.config['ES_INDEX_NAME'])
+FLAVORS = [
+    ('nyaa', models.NyaaTorrent),
+    ('sukebei', models.SukebeiTorrent)
+]
 
-helpers.bulk(es, (mk_es(t) for t in page_query(Torrent.query)), chunk_size=10000)
+# Get binlog status from mysql
+master_status = db.engine.execute('SHOW MASTER STATUS;').fetchone()
 
-# restore to near-enough real time
-ic.put_settings(body={'index': {'refresh_interval': '30s'}}, index=app.config['ES_INDEX_NAME'])
+position_json = {
+    'log_file': master_status[0], 
+    'log_pos': master_status[1]
+}
+
+print('Save the following in the file configured in your ES sync config JSON:')
+print(json.dumps(position_json))
+
+for flavor, torrent_class in FLAVORS:
+    print('Importing torrents for index', flavor, 'from', torrent_class)
+    bar = progressbar.ProgressBar(
+        maxval=torrent_class.query.count(),
+        widgets=[ progressbar.SimpleProgress(),
+                  ' [', progressbar.Timer(), '] ',
+                  progressbar.Bar(),
+                  ' (', progressbar.ETA(), ') ',
+            ])
+
+    # turn off refreshes while bulk loading
+    ic.put_settings(body={'index': {'refresh_interval': '-1'}}, index=flavor)
+
+    bar.start()
+    helpers.bulk(es, (mk_es(t, flavor) for t in page_query(torrent_class.query, progress_bar=bar)), chunk_size=10000)
+    bar.finish()
+
+    # Refresh the index immideately
+    ic.refresh(index=flavor)
+    print('Index refresh done.')
+
+    # restore to near-enough real time
+    ic.put_settings(body={'index': {'refresh_interval': '30s'}}, index=flavor)
diff --git a/requirements.txt b/requirements.txt
index 2a184ff..43e57e3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -28,7 +28,7 @@ mysqlclient==1.3.10
 orderedset==2.0
 packaging==16.8
 passlib==1.7.1
-progressbar2==3.20.0
+progressbar33==2.40.0
 pycodestyle==2.3.1
 pycparser==2.17
 PyMySQL==0.7.11

From 05713892f75770a619336201ff43e2db46583e8f Mon Sep 17 00:00:00 2001
From: TheAMM <the.actual.amm@gmail.com>
Date: Sun, 28 May 2017 02:15:39 +0300
Subject: [PATCH 23/30] Update config.example.py: remove TABLE_PREFIX, enable
 stat display by default

---
 config.example.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/config.example.py b/config.example.py
index 3b07da0..aa50cd8 100644
--- a/config.example.py
+++ b/config.example.py
@@ -7,7 +7,7 @@ USE_EMAIL_VERIFICATION = False
 USE_MYSQL = True
 
 # Enable this once stat integration is done
-ENABLE_SHOW_STATS = False
+ENABLE_SHOW_STATS = True
 
 BASE_DIR = os.path.abspath(os.path.dirname(__file__))
 if USE_MYSQL:
@@ -21,7 +21,6 @@ SECRET_KEY = '***'
 
 # Prefix for running multiple sites, user table will not be prefixed.
 SITE_FLAVOR = 'nyaa' # 'nyaa' or 'sukebei'
-TABLE_PREFIX = SITE_FLAVOR + '_'
 
 # for recaptcha and email verification:
 # keys for localhost. Change as appropriate when actual domain is registered.
@@ -33,16 +32,16 @@ MAIL_FROM_ADDRESS = '***'
 SMTP_USERNAME = '***'
 SMTP_PASSWORD = '***'
 
-# What the site identifies itself as.
+# What the site identifies itself as. This affects templates, not database stuff.
 SITE_NAME = 'Nyaa'
 
 # The maximum number of files a torrent can contain
 # until the site says "Too many files to display."
 MAX_FILES_VIEW = 1000
 
-# """
+#
 # Setting to make sure main announce url is present in torrent
-# """
+#
 ENFORCE_MAIN_ANNOUNCE_URL = False
 MAIN_ANNOUNCE_URL = ''
 
@@ -51,10 +50,11 @@ BACKUP_TORRENT_FOLDER = 'torrents'
 #
 # Search Options
 #
-# Max ES search results, do not set over 10000
 RESULTS_PER_PAGE = 75
 
+# See README.MD on Elasticsearch setup
 USE_ELASTIC_SEARCH = False
 ENABLE_ELASTIC_SEARCH_HIGHLIGHT = False
+# Max ES search results, do not set over 10000
 ES_MAX_SEARCH_RESULT = 1000
 ES_INDEX_NAME = SITE_FLAVOR  # we create indicies named nyaa or sukebei
\ No newline at end of file

From 926c85058e92fb4fd3203ef70345c6220bb8211b Mon Sep 17 00:00:00 2001
From: TheAMM <the.actual.amm@gmail.com>
Date: Sun, 28 May 2017 02:16:25 +0300
Subject: [PATCH 24/30] Overhaul README.md with more helpful and more
 up-to-date instructions

---
 README.md | 146 ++++++++++++++++++++++++++++++++----------------------
 1 file changed, 87 insertions(+), 59 deletions(-)

diff --git a/README.md b/README.md
index ab46758..8ff9856 100644
--- a/README.md
+++ b/README.md
@@ -1,82 +1,110 @@
 # NyaaV2
 
-## Setup
+## Setting up for development
+This project uses Python 3.6. There are features used that do not exist in 3.5, so make sure to use Python 3.6.
+This guide also assumes you 1) are using Linux and 2) are somewhat capable with the commandline.
+It's not impossible to run Nyaa on Windows, but this guide doesn't focus on that.
 
+### Code Quality:
+- Before we get any deeper, remember to follow PEP8 style guidelines and run `./lint.sh` before committing.
+    - You may also use `pycodestyle nyaa/ --show-source --max-line-length=100´ to see a list of warnings/problems instead of having `lint.sh` making modifications for you
+- Other than PEP8, try to keep your code clean and easy to understand, as well. It's only polite!
+
+### Setting up Pyenv
+pyenv eases the use of different Python versions, and as not all Linux distros offer 3.6 packages, it's right up our alley.
 - Install dependencies https://github.com/pyenv/pyenv/wiki/Common-build-problems
 - Install `pyenv` https://github.com/pyenv/pyenv/blob/master/README.md#installation
 - Install `pyenv-virtualenv` https://github.com/pyenv/pyenv-virtualenv/blob/master/README.md
-- `pyenv install 3.6.1`
-- `pyenv virtualenv 3.6.1 nyaa`
-- `pyenv activate nyaa`
+- Install Python 3.6.1 with `pyenv` and create a virtualenv for the project:
+    - `pyenv install 3.6.1`
+    - `pyenv virtualenv 3.6.1 nyaa`
+    - `pyenv activate nyaa`
 - Install dependencies with `pip install -r requirements.txt`
 - Copy `config.example.py` into `config.py`
-- Change TABLE_PREFIX to `nyaa_` or `sukebei_` depending on the site
+    - Change `SITE_FLAVOR` in your `config.py` depending on which instance you want to host
 
-### Setting up MySQL/MariaDB database for advanced functionality
+### Setting up MySQL/MariaDB database
+You *may* use SQLite but the current support for it in this project is outdated and rather unsupported.
 - Enable `USE_MYSQL` flag in config.py
 - Install latest mariadb by following instructions here https://downloads.mariadb.org/mariadb/repositories/
     - Tested versions: `mysql  Ver 15.1 Distrib 10.0.30-MariaDB, for debian-linux-gnu (x86_64) using readline 5.2`
-- Run the following commands logged in as your root db user:
+- Run the following commands logged in as your root db user (substitute for your own `config.py` values if desired):
     - `CREATE USER 'test'@'localhost' IDENTIFIED BY 'test123';`
-    - `GRANT ALL PRIVILEGES ON * . * TO 'test'@'localhost';`
+    - `GRANT ALL PRIVILEGES ON *.* TO 'test'@'localhost';`
     - `FLUSH PRIVILEGES;`
     - `CREATE DATABASE nyaav2 DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;`
-- To setup and import nyaa_maria_vx.sql:
-    - `mysql -u <user> -p nyaav2`
-    - `DROP DATABASE nyaav2;`
-    - `CREATE DATABASE nyaav2 DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;`
-    - `SOURCE ~/path/to/database/nyaa_maria_vx.sql`
 
 ### Finishing up
-- Run `python db_create.py` to create the database
-- Load the .sql file
-    - `mysql -u user -p nyaav2`
-    - `SOURCE cocks.sql`
-    - Remember to change the default user password to an empty string to disable logging in
+- Run `python db_create.py` to create the database and import categories
+    - Follow the advice of `db_create.py` and run `./db_migrate.py stamp head` to mark the database version for Alembic
 - Start the dev server with `python run.py`
-- When you are finished developing, deactivate your virtualenv with `source deactivate`
+- When you are finished developing, deactivate your virtualenv with `pyenv deactivate` or `source deactivate` (or just close your shell session)
 
-## Enabling ElasticSearch
+You're now ready for simple testing and development!
+Continue below to learn about database migrations and enabling the advanced search engine, Elasticsearch.
 
-### Basics
-- Install jdk `sudo apt-get install openjdk-8-jdk`
-- Install elasticsearch https://www.elastic.co/guide/en/elasticsearch/reference/current/deb.html
-- `sudo systemctl enable elasticsearch.service`
-- `sudo systemctl start elasticsearch.service`
-- Run `curl -XGET 'localhost:9200'` and make sure ES is running
-- Optional: install Kabana as a search frontend for ES
-
-### Enable MySQL Binlogging
-- Add the `[mariadb]` bin-log section to my.cnf and reload mysql server
-- Connect to mysql
-- `SHOW VARIABLES LIKE 'binlog_format';`
-    - Make sure it shows ROW
-- Connect to root user
-- `GRANT REPLICATION SLAVE ON *.* TO 'test'@'localhost';` where test is the user you will be running `sync_es.py` with
-
-### Setting up ES
-- Run `./create_es.sh` and this creates two indicies: `nyaa` and `sukebei`
-- The output should show `acknowledged: true` twice
-- The safest bet is to disable the webapp here to ensure there's no database writes
-- Run `python import_to_es.py` with `SITE_FLAVOR` set to `nyaa`
-- Run `python import_to_es.py` with `SITE_FLAVOR` set to `sukebei`
-- These will take some time to run as it's indexing
-
-### Setting up sync_es.py
-- Sync_es.py keeps the ElasticSearch index updated by reading the BinLog
-- Configure the MySQL options with the user where you granted the REPLICATION permissions
-- Connect to MySQL, run `SHOW MASTER STATUS;`.
-- Copy the output to `/var/lib/sync_es_position.json` with the contents `{"log_file": "FILE", "log_pos": POSITION}` and replace FILENAME with File (something like master1-bin.000002) in the SQL output and POSITION (something like 892528513) with Position
-- Set up `sync_es.py` as a service and run it, preferably as the system/root
-- Make sure `sync_es.py` runs within venv with the right dependencies
-
-Enable the `USE_ELASTIC_SEARCH` flag in `config.py`, restart the application, and you're good to go.
 
 ## Database migrations
-- Uses [flask-Migrate](https://flask-migrate.readthedocs.io/)
-- Run `./db_migrate.py db migrate` to generate the migration script after database model changes.
-- Take a look at the result in `migrations/versions/...` to make sure nothing went wrong.
-- Run `./db_migrate.py db upgrade` to upgrade your database.
+- Database migrations are done with [flask-Migrate](https://flask-migrate.readthedocs.io/), a wrapper around [Alembic](http://alembic.zzzcomputing.com/en/latest/).
+- If someone has made changes in the database schema and included a new migration script:
+    - If your database has never been marked by Alembic (you're on a database from before the migrations), run `./db_migrate.py stamp head` before pulling the new migration script(s).
+        - If you already have the new scripts, check the output of `./db_migrate.py history` instead and choose a hash that matches your current database state, then run `./db_migrate.py stamp <hash>`.
+    - Update your branch (eg. `git fetch && git rebase origin/master`)
+    - Run `./db_migrate.py upgrade head` to run the migration. Done!
+- If *you* have made a change in the database schema:
+    - Save your changes in `models.py` and ensure the database schema matches the previous version (ie. your new tables/columns are not added to the live database)
+    - Run `./db_migrate.py migrate -m "Short description of changes"` to automatically generate a migration script for the changes
+      - Check the script (`migrations/versions/...`) and make sure it works! Alembic may not able to notice all changes.
+    - Run `./db_migrate.py upgrade` to run the migration and verify the upgrade works.
+       - (Run `./db_migrate.py downgrade` to verify the downgrade works as well, then upgrade again)
 
-## Code Quality:
-- Remember to follow PEP8 style guidelines and run `./lint.sh` before committing.
+
+## Setting up and enabling Elasticsearch
+
+### Installing Elasticsearch
+- Install JDK with `sudo apt-get install openjdk-8-jdk`
+- Install [Elasticsearch](https://www.elastic.co/downloads/elasticsearch)
+    - [From packages...](https://www.elastic.co/guide/en/elasticsearch/reference/current/deb.html)
+        - Enable the service:
+            - `sudo systemctl enable elasticsearch.service`
+            - `sudo systemctl start elasticsearch.service`
+    - or [simply extracting the archives and running the files](https://www.elastic.co/guide/en/elasticsearch/reference/current/_installation.html), if you don't feel like permantently installing ES
+- Run `curl -XGET 'localhost:9200'` and make sure ES is running
+    - Optional: install [Kibana](https://www.elastic.co/products/kibana) as a search debug frontend for ES
+
+### Setting up ES
+- Run `./create_es.sh` to create the indices for the torrents: `nyaa` and `sukebei`
+    - The output should show `acknowledged: true` twice
+- Stop the Nyaa app if you haven't already
+- Run `python import_to_es.py` to import all the torrents (on nyaa and sukebei) into the ES indices.
+    - This may take some time to run if you have plenty of torrents in your database.
+
+Enable the `USE_ELASTIC_SEARCH` flag in `config.py` and (re)start the application.
+Elasticsearch should now be functional! The ES indices won't be updated "live" with the current setup, continue below for instructions on how to hook Elasticsearch up to MySQL binlog.
+
+However, take note that binglog is not necessary for simple ES testing and development; you can simply run `import_to_es.py` from time to time to reindex all the torrents.
+
+### Enabling MySQL Binlogging
+- Edit your MariaDB/MySQL server configuration and add the following under `[mariadb]`:
+    ```
+    log-bin
+    server_id=1
+    log-basename=master1
+    binlog-format=row
+    ```
+- Restart MariaDB/MySQL (`sudo service mysql restart`)
+- Copy the example configuration (`es_sync_config.example.json`) as `es_sync_config.json` and adjust options in it to your liking (verify the connection options!)
+- Connect to mysql as root
+    - Verify that the result of `SHOW VARIABLES LIKE 'binlog_format';` is `ROW`
+    - Execute `GRANT REPLICATION SLAVE ON *.* TO 'username'@'localhost';` to allow your configured user access to the binlog
+
+
+### Setting up sync_es.py
+`sync_es.py` keeps the Elasticsearch indices updated by reading the binlog and pushing the changes to the ES indices.
+- Make sure `es_sync_config.json` is configured with the user you grated the `REPLICATION` permissions
+- Run `import_to_es.py` and copy the outputted JSON into the file specified by `save_loc` in your `es_sync_config.json`
+- Run `sync_es.py` as-is *or*, for actual deployment, set it up as a service and run it, preferably as the system/root
+    - Make sure `sync_es.py` runs within the venv with the right dependencies!
+
+You're done! The script should now be feeding updates from the database to Elasticsearch.
+Take note, however, that the specified ES index refresh interval is 30 seconds, which may feel like a long time on local development. Feel free to adjust it or [poke Elasticsearch yourself!](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html)
\ No newline at end of file

From 75aff3835dd9adcd2da7d6e37ab4e463b10e0f6f Mon Sep 17 00:00:00 2001
From: Anna-Maria Meriniemi <The.Actual.AMM@gmail.com>
Date: Sun, 28 May 2017 02:23:45 +0300
Subject: [PATCH 25/30] GFM, you lied to me about newlines!

---
 README.md | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 8ff9856..9a8dc39 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,8 @@
 # NyaaV2
 
 ## Setting up for development
-This project uses Python 3.6. There are features used that do not exist in 3.5, so make sure to use Python 3.6.
-This guide also assumes you 1) are using Linux and 2) are somewhat capable with the commandline.
+This project uses Python 3.6. There are features used that do not exist in 3.5, so make sure to use Python 3.6.   
+This guide also assumes you 1) are using Linux and 2) are somewhat capable with the commandline.   
 It's not impossible to run Nyaa on Windows, but this guide doesn't focus on that.
 
 ### Code Quality:
@@ -40,7 +40,7 @@ You *may* use SQLite but the current support for it in this project is outdated
 - Start the dev server with `python run.py`
 - When you are finished developing, deactivate your virtualenv with `pyenv deactivate` or `source deactivate` (or just close your shell session)
 
-You're now ready for simple testing and development!
+You're now ready for simple testing and development!   
 Continue below to learn about database migrations and enabling the advanced search engine, Elasticsearch.
 
 
@@ -79,8 +79,8 @@ Continue below to learn about database migrations and enabling the advanced sear
 - Run `python import_to_es.py` to import all the torrents (on nyaa and sukebei) into the ES indices.
     - This may take some time to run if you have plenty of torrents in your database.
 
-Enable the `USE_ELASTIC_SEARCH` flag in `config.py` and (re)start the application.
-Elasticsearch should now be functional! The ES indices won't be updated "live" with the current setup, continue below for instructions on how to hook Elasticsearch up to MySQL binlog.
+Enable the `USE_ELASTIC_SEARCH` flag in `config.py` and (re)start the application.   
+Elasticsearch should now be functional! The ES indices won't be updated "live" with the current setup, continue below for instructions on how to hook Elasticsearch up to MySQL binlog.   
 
 However, take note that binglog is not necessary for simple ES testing and development; you can simply run `import_to_es.py` from time to time to reindex all the torrents.
 
@@ -106,5 +106,5 @@ However, take note that binglog is not necessary for simple ES testing and devel
 - Run `sync_es.py` as-is *or*, for actual deployment, set it up as a service and run it, preferably as the system/root
     - Make sure `sync_es.py` runs within the venv with the right dependencies!
 
-You're done! The script should now be feeding updates from the database to Elasticsearch.
-Take note, however, that the specified ES index refresh interval is 30 seconds, which may feel like a long time on local development. Feel free to adjust it or [poke Elasticsearch yourself!](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html)
\ No newline at end of file
+You're done! The script should now be feeding updates from the database to Elasticsearch.   
+Take note, however, that the specified ES index refresh interval is 30 seconds, which may feel like a long time on local development. Feel free to adjust it or [poke Elasticsearch yourself!](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html)

From 32fa021f34a02872c66abf27167ac15d9d0ee506 Mon Sep 17 00:00:00 2001
From: TheAMM <the.actual.amm@gmail.com>
Date: Sun, 28 May 2017 02:46:04 +0300
Subject: [PATCH 26/30] Turn comments icon on search result into link to
 comments

Also cleans up torrent.id/torrent.meta.id
---
 nyaa/templates/search_results.html | 11 ++++++-----
 nyaa/templates/view.html           |  2 +-
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/nyaa/templates/search_results.html b/nyaa/templates/search_results.html
index fedfb2c..8313888 100644
--- a/nyaa/templates/search_results.html
+++ b/nyaa/templates/search_results.html
@@ -70,20 +70,21 @@
 				</a>
 				</td>
 				<td colspan="2">
+					{% set torrent_id = torrent.meta.id if use_elastic else torrent.id %}
 					{% set com_count = torrent.comment_count %}
 					{% if com_count %}
-					<span class="comments" title="{{ '{c} comment{s}'.format(c=com_count, s='s' if com_count > 1 else '') }}">
+					<a href="{{ url_for('view_torrent', torrent_id=torrent_id, _anchor='comments') }}" class="comments" title="{{ '{c} comment{s}'.format(c=com_count, s='s' if com_count > 1 else '') }}">
 						<i class="fa fa-comments-o"></i>{{ com_count -}}
-					</span>
+					</a>
 					{% endif %}
 					{% if use_elastic %}
-					<a href="{{ url_for('view_torrent', torrent_id=torrent.meta.id) }}" title="{{ torrent.display_name | escape }}">{%if "highlight" in torrent.meta %}{{ torrent.meta.highlight.display_name[0] | safe }}{% else %}{{torrent.display_name}}{%endif%}</a>
+					<a href="{{ url_for('view_torrent', torrent_id=torrent_id) }}" title="{{ torrent.display_name | escape }}">{%if "highlight" in torrent.meta %}{{ torrent.meta.highlight.display_name[0] | safe }}{% else %}{{torrent.display_name}}{%endif%}</a>
 					{% else %}
-					<a href="{{ url_for('view_torrent', torrent_id=torrent.id) }}" title="{{ torrent.display_name | escape }}">{{ torrent.display_name | escape }}</a>
+					<a href="{{ url_for('view_torrent', torrent_id=torrent_id) }}" title="{{ torrent.display_name | escape }}">{{ torrent.display_name | escape }}</a>
 					{% endif %}
 				</td>
 				<td class="text-center" style="white-space: nowrap;">
-					{% if torrent.has_torrent %}<a href="{{ url_for('download_torrent', torrent_id=torrent.id) }}"><i class="fa fa-fw fa-download"></i></a>{% endif %}
+					{% if torrent.has_torrent %}<a href="{{ url_for('download_torrent', torrent_id=torrent_id) }}"><i class="fa fa-fw fa-download"></i></a>{% endif %}
 					{% if use_elastic %}
 					<a href="{{ create_magnet_from_es_info(torrent.display_name, torrent.info_hash) }}"><i class="fa fa-fw fa-magnet"></i></a>
 					{% else %}
diff --git a/nyaa/templates/view.html b/nyaa/templates/view.html
index 4834635..cfc1adc 100644
--- a/nyaa/templates/view.html
+++ b/nyaa/templates/view.html
@@ -128,7 +128,7 @@
 </div>
 {% endif %}
 
-<div class="panel panel-default">
+<div id="comments" class="panel panel-default">
 	<div class="panel-heading">
 		<h3 class="panel-title">
 			Comments - {{ comments | length }}

From 8fecc08d19527df45cfb82a8812bd81fc08e795f Mon Sep 17 00:00:00 2001
From: TheAMM <the.actual.amm@gmail.com>
Date: Sun, 28 May 2017 02:47:47 +0300
Subject: [PATCH 27/30] Keep it short, keep it simple

Keep it not retarded
---
 db_migrate.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/db_migrate.py b/db_migrate.py
index 07989d1..c02ff2f 100755
--- a/db_migrate.py
+++ b/db_migrate.py
@@ -12,11 +12,6 @@ manager.add_command("db", MigrateCommand)
 
 if __name__ == "__main__":
 	# Patch sys.argv to default to 'db'
-	argv_contents = sys.argv[:]
-	
-	sys.argv.clear()
-	sys.argv.append(argv_contents[0])
-	sys.argv.append('db')
-	sys.argv.extend(argv_contents[1:])
+	sys.argv.insert(1, 'db')
 
 	manager.run()

From 7acfa97a472050ae64ffa47ba99c7e5b33c0410a Mon Sep 17 00:00:00 2001
From: zaclimon <isaacpateau05@gmail.com>
Date: Sat, 27 May 2017 21:11:55 -0400
Subject: [PATCH 28/30] requirements.txt: Adapt progressbar33 version

While installing progressbar33 2.40.0, an error occurs:

Could not find a version that satisfies the requirement
progressbar33==2.40.0 (from -r requirements.txt (line 31)) (from
versions: 2.4)
No matching distribution found for progressbar33==2.40.0 (from -r
requirements.txt (line 31))

Reflect the correct version as shown on pypi which is 2.4.
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 43e57e3..705ed83 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -28,7 +28,7 @@ mysqlclient==1.3.10
 orderedset==2.0
 packaging==16.8
 passlib==1.7.1
-progressbar33==2.40.0
+progressbar33==2.4
 pycodestyle==2.3.1
 pycparser==2.17
 PyMySQL==0.7.11

From 33852a55bf6daf638cba0c084d9883a5ad22790e Mon Sep 17 00:00:00 2001
From: queue <queue@hakase.org>
Date: Sun, 28 May 2017 19:46:38 -0600
Subject: [PATCH 29/30] sync_es: die when killed

---
 sync_es.py | 29 +++++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/sync_es.py b/sync_es.py
index 2f2bb09..f52ca2b 100755
--- a/sync_es.py
+++ b/sync_es.py
@@ -24,6 +24,10 @@ database into es, at the expense of redoing a (small) amount of indexing.
 
 This uses multithreading so we don't have to block on socket io (both binlog
 reading and es POSTing). asyncio soon™
+
+This script will exit on any sort of exception, so you'll want to use your
+supervisor's restart functionality, e.g. Restart=failure in systemd, or
+the poor man's `while true; do sync_es.py; sleep 1; done` in tmux.
 """
 from elasticsearch import Elasticsearch
 from elasticsearch.helpers import bulk, BulkIndexError
@@ -133,14 +137,31 @@ def delet_this(row, index_name):
         '_type': 'torrent',
         '_id': str(row['values']['id'])}
 
+# we could try to make this script robust to errors from es or mysql, but since
+# the only thing we can do is "clear state and retry", it's easier to leave
+# this to the supervisor. If we we carrying around heavier state in-process,
+# it'd be more worth it to handle errors ourselves.
+# 
+# Apparently there's no setDefaultUncaughtExceptionHandler in threading, and
+# sys.excepthook is also broken, so this gives us the same
+# exit-if-anything-happens semantics. 
+class ExitingThread(Thread):
+    def run(self):
+        try:
+            self.run_happy()
+        except:
+            log.exception("something happened")
+            # sys.exit only exits the thread, lame
+            import os
+            os._exit(1)
 
-class BinlogReader(Thread):
+class BinlogReader(ExitingThread):
     # write_buf is the Queue we communicate with
     def __init__(self, write_buf):
         Thread.__init__(self)
         self.write_buf = write_buf
 
-    def run(self):
+    def run_happy(self):
         with open(SAVE_LOC) as f:
             pos = json.load(f)
 
@@ -229,7 +250,7 @@ class BinlogReader(Thread):
             else:
               raise Exception(f"unknown table {s.table}")
 
-class EsPoster(Thread):
+class EsPoster(ExitingThread):
     # read_buf is the queue of stuff to bulk post
     def __init__(self, read_buf, chunk_size=1000, flush_interval=5):
         Thread.__init__(self)
@@ -237,7 +258,7 @@ class EsPoster(Thread):
         self.chunk_size = chunk_size
         self.flush_interval = flush_interval
 
-    def run(self):
+    def run_happy(self):
         es = Elasticsearch(timeout=30)
 
         last_save = time.time()

From eceb8824dc73a7300138073c8fa4792b6f688b76 Mon Sep 17 00:00:00 2001
From: queue <queue@hakase.org>
Date: Sun, 28 May 2017 20:14:14 -0600
Subject: [PATCH 30/30] sync_es: fix flush_interval behavior during slow times

instead of flushing every N seconds, it flushed N seconds after
the last change, which could drag out to N seconds * M batch size
if there are few updates. Practically this doesn't change anything
since stuff is always happening.

Also fix not writing a save point if nothing is happening. Also
practically does nothing, but for correctness.
---
 sync_es.py | 65 ++++++++++++++++++++++++++++++++----------------------
 1 file changed, 39 insertions(+), 26 deletions(-)

diff --git a/sync_es.py b/sync_es.py
index f52ca2b..444c905 100755
--- a/sync_es.py
+++ b/sync_es.py
@@ -263,54 +263,67 @@ class EsPoster(ExitingThread):
 
         last_save = time.time()
         since_last = 0
+        # XXX keep track of last posted position for save points, awkward
+        posted_log_file = None
+        posted_log_pos = None
 
         while True:
             actions = []
-            while len(actions) < self.chunk_size:
+            now = time.time()
+            # wait up to flush_interval seconds after starting the batch
+            deadline = now + self.flush_interval
+            while len(actions) < self.chunk_size and now < deadline:
+                timeout = deadline - now
                 try:
                     # grab next event from queue with metadata that creepily
                     # updates, surviving outside the scope of the loop
                     ((log_file, log_pos, timestamp), action) = \
-                            self.read_buf.get(block=True, timeout=self.flush_interval)
+                            self.read_buf.get(block=True, timeout=timeout)
                     actions.append(action)
+                    now = time.time()
                 except Empty:
                     # nothing new for the whole interval
                     break
 
-            if not actions:
-                # nothing to post
-                log.debug("no changes...")
-                continue
+            if actions:
+                # XXX "time" to get histogram of no events per bulk
+                stats.timing('actions_per_bulk', len(actions))
 
-            # XXX "time" to get histogram of no events per bulk
-            stats.timing('actions_per_bulk', len(actions))
-
-            try:
-                with stats.timer('post_bulk'):
-                    bulk(es, actions, chunk_size=self.chunk_size)
-            except BulkIndexError as bie:
-                 # in certain cases where we're really out of sync, we update a
-                 # stat when the torrent doc is, causing a "document missing"
-                 # error from es, with no way to suppress that server-side.
-                 # Thus ignore that type of error if it's the only problem
-                for e in bie.errors:
-                    try:
-                        if e['update']['error']['type'] != 'document_missing_exception':
+                try:
+                    with stats.timer('post_bulk'):
+                        bulk(es, actions, chunk_size=self.chunk_size)
+                except BulkIndexError as bie:
+                     # in certain cases where we're really out of sync, we update a
+                     # stat when the torrent doc is, causing a "document missing"
+                     # error from es, with no way to suppress that server-side.
+                     # Thus ignore that type of error if it's the only problem
+                    for e in bie.errors:
+                        try:
+                            if e['update']['error']['type'] != 'document_missing_exception':
+                                raise bie
+                        except KeyError:
                             raise bie
-                    except KeyError:
-                        raise bie
 
-            # how far we're behind, wall clock
-            stats.gauge('process_latency', int((time.time() - timestamp) * 1000))
+                # how far we've gotten in the actual log
+                posted_log_file = log_file
+                posted_log_pos = log_pos
+
+                # how far we're behind, wall clock
+                stats.gauge('process_latency', int((time.time() - timestamp) * 1000))
+            else:
+                log.debug("no changes...")
 
             since_last += len(actions)
-            if since_last >= 10000 or (time.time() - last_save) > 10:
+            # TODO instead of this manual timeout loop, could move this to another queue/thread
+            if posted_log_file is not None and (since_last >= 10000 or (time.time() - last_save) > 10):
                 log.info(f"saving position {log_file}/{log_pos}, {time.time() - timestamp:,.3f} seconds behind")
                 with stats.timer('save_pos'):
                     with open(SAVE_LOC, 'w') as f:
-                        json.dump({"log_file": log_file, "log_pos": log_pos}, f)
+                        json.dump({"log_file": posted_log_file, "log_pos": posted_log_pos}, f)
                 last_save = time.time()
                 since_last = 0
+                posted_log_file = None
+                posted_log_pos = None
 
 # in-memory queue between binlog and es. The bigger it is, the more events we
 # can parse in memory while waiting for es to catch up, at the expense of heap.