Prepare for app factory [1 of 2] (#315)

* Move db, assets, debug toolbar and fix_paginate into nyaa.extensions
* Change all `from nyaa import db` imports to `from nyaa.extensions import db`
* Move `nyaa.torrents.create_magnet_from_es_info` context processor into template-utils blueprint
* Fix tools (wrap in `with app.app_context():` where needed)

db_create.py

@@ -1,7 +1,8 @@
 #!/usr/bin/env python3
-import sys
 import sqlalchemy
-from nyaa import app, db, models
+
+from nyaa import app, models
+from nyaa.extensions import db
 
 NYAA_CATEGORIES = [
     ('Anime', ['Anime Music Video', 'English-translated', 'Non-English-translated', 'Raw']),
@@ -29,32 +30,31 @@ def add_categories(categories, main_class, sub_class):
 
 
 if __name__ == '__main__':
-    # Test for the user table, assume db is empty if it's not created
+    with app.app_context():
-    database_empty = False
+        # Test for the user table, assume db is empty if it's not created
-    try:
+        database_empty = False
-        models.User.query.first()
+        try:
-    except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
+            models.User.query.first()
-        database_empty = True
+        except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
+            database_empty = True
 
+        print('Creating all tables...')
+        db.create_all()
 
-    print('Creating all tables...')
+        nyaa_category_test = models.NyaaMainCategory.query.first()
-    db.create_all()
+        if not nyaa_category_test:
+            print('Adding Nyaa categories...')
+            add_categories(NYAA_CATEGORIES, models.NyaaMainCategory, models.NyaaSubCategory)
 
+        sukebei_category_test = models.SukebeiMainCategory.query.first()
+        if not sukebei_category_test:
+            print('Adding Sukebei categories...')
+            add_categories(SUKEBEI_CATEGORIES, models.SukebeiMainCategory, models.SukebeiSubCategory)
 
-    nyaa_category_test = models.NyaaMainCategory.query.first()
+        db.session.commit()
-    if not nyaa_category_test:
-        print('Adding Nyaa categories...')
-        add_categories(NYAA_CATEGORIES, models.NyaaMainCategory, models.NyaaSubCategory)
 
-    sukebei_category_test = models.SukebeiMainCategory.query.first()
+        if database_empty:
-    if not sukebei_category_test:
+            print('Remember to run the following to mark the database up-to-date for Alembic:')
-        print('Adding Sukebei categories...')
+            print('./db_migrate.py stamp head')
-        add_categories(SUKEBEI_CATEGORIES, models.SukebeiMainCategory, models.SukebeiSubCategory)
+            # Technically we should be able to do this here, but when you have
-
+            # Flask-Migrate and Flask-SQA and everything... I didn't get it working.
-    db.session.commit()
-
-    if database_empty:
-        print('Remember to run the following to mark the database up-to-date for Alembic:')
-        print('./db_migrate.py stamp head')
-        # Technically we should be able to do this here, but when you have
-        # Flask-Migrate and Flask-SQA and everything... I didn't get it working.

db_migrate.py

@@ -1,17 +1,20 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 import sys
-from nyaa import app, db
+
 from flask_script import Manager
 from flask_migrate import Migrate, MigrateCommand
 
+from nyaa import app
+from nyaa.extensions import db
+
 migrate = Migrate(app, db)
 
 manager = Manager(app)
 manager.add_command("db", MigrateCommand)
 
 if __name__ == "__main__":
-	# Patch sys.argv to default to 'db'
+    # Patch sys.argv to default to 'db'
-	sys.argv.insert(1, 'db')
+    sys.argv.insert(1, 'db')
 
-	manager.run()
+    manager.run()

import_to_es.py

@@ -7,14 +7,15 @@ with a cron job or some binlog-reading thing (TODO)
 """
 import sys
 import json
-from nyaa import app, db, models
 
+# This should be progressbar33
+import progressbar
 from elasticsearch import Elasticsearch
 from elasticsearch.client import IndicesClient
 from elasticsearch import helpers
 
-# This should be progressbar33
+from nyaa import app, models
-import progressbar
+from nyaa.extensions import db
 
 es = Elasticsearch(timeout=30)
 ic = IndicesClient(es)
@@ -93,7 +94,8 @@ FLAVORS = [
 ]
 
 # Get binlog status from mysql
-master_status = db.engine.execute('SHOW MASTER STATUS;').fetchone()
+with app.app_context():
+    master_status = db.engine.execute('SHOW MASTER STATUS;').fetchone()
 
 position_json = {
     'log_file': master_status[0], 

nyaa/__init__.py

@@ -2,26 +2,20 @@ import logging
 import os
 
 import flask
-from flask_assets import Bundle, Environment  # noqa F401
+from flask_assets import Bundle  # noqa F401
-from flask_debugtoolbar import DebugToolbarExtension
-from flask_sqlalchemy import SQLAlchemy
 
-from nyaa import fix_paginate  # noqa F401
+from nyaa.extensions import assets, db, fix_paginate, toolbar
 
 app = flask.Flask(__name__)
 app.config.from_object('config')
 
-# Database
-app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
-app.config['MYSQL_DATABASE_CHARSET'] = 'utf8mb4'
-
 # Don't refresh cookie each request
 app.config['SESSION_REFRESH_EACH_REQUEST'] = False
 
 # Debugging
 if app.config['DEBUG']:
     app.config['DEBUG_TB_INTERCEPT_REDIRECTS'] = False
-    toolbar = DebugToolbarExtension(app)
+    toolbar.init_app(app)
     app.logger.setLevel(logging.DEBUG)
 
     # Forbid caching
@@ -63,9 +57,13 @@ app.jinja_env.add_extension('jinja2.ext.do')
 app.jinja_env.lstrip_blocks = True
 app.jinja_env.trim_blocks = True
 
-db = SQLAlchemy(app)
+# Database
+fix_paginate()  # This has to be before the database is initialized
+app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
+app.config['MYSQL_DATABASE_CHARSET'] = 'utf8mb4'
+db.init_app(app)
 
-assets = Environment(app)
+assets.init_app(app)
 
 # css = Bundle('style.scss', filters='libsass',
 #             output='style.css', depends='**/*.scss')

nyaa/api_handler.py

@@ -6,7 +6,8 @@ import re
 
 import flask
 
-from nyaa import backend, bencode, db, forms, models, utils
+from nyaa import backend, bencode, forms, models, utils
+from nyaa.extensions import db
 from nyaa.views.torrents import _create_upload_category_choices
 
 api_blueprint = flask.Blueprint('api', __name__)

nyaa/backend.py

@@ -7,7 +7,8 @@ from werkzeug import secure_filename
 
 from orderedset import OrderedSet
 
-from nyaa import app, db, models, utils
+from nyaa import app, models, utils
+from nyaa.extensions import db
 
 
 @utils.cached_function

nyaa/extensions.py

@@ -0,0 +1,34 @@
+from flask import abort
+from flask_assets import Environment
+from flask_debugtoolbar import DebugToolbarExtension
+from flask_sqlalchemy import BaseQuery, Pagination, SQLAlchemy
+
+assets = Environment()
+db = SQLAlchemy()
+toolbar = DebugToolbarExtension()
+
+
+def fix_paginate():
+
+    def paginate_faste(self, page=1, per_page=50, max_page=None, step=5, count_query=None):
+        if page < 1:
+            abort(404)
+
+        if max_page and page > max_page:
+            abort(404)
+
+        # Count all items
+        if count_query is not None:
+            total_query_count = count_query.scalar()
+        else:
+            total_query_count = self.count()
+
+        # Grab items on current page
+        items = self.limit(per_page).offset((page - 1) * per_page).all()
+
+        if not items and page != 1:
+            abort(404)
+
+        return Pagination(self, page, per_page, total_query_count, items)
+
+    BaseQuery.paginate_faste = paginate_faste

nyaa/fix_paginate.py

@@ -1,27 +0,0 @@
-from flask import abort
-from flask_sqlalchemy import BaseQuery, Pagination
-
-
-def paginate_faste(self, page=1, per_page=50, max_page=None, step=5, count_query=None):
-    if page < 1:
-        abort(404)
-
-    if max_page and page > max_page:
-        abort(404)
-
-    # Count all items
-    if count_query is not None:
-        total_query_count = count_query.scalar()
-    else:
-        total_query_count = self.count()
-
-    # Grab items on current page
-    items = self.limit(per_page).offset((page - 1) * per_page).all()
-
-    if not items and page != 1:
-        abort(404)
-
-    return Pagination(self, page, per_page, total_query_count, items)
-
-
-BaseQuery.paginate_faste = paginate_faste

nyaa/models.py

@@ -15,7 +15,8 @@ from sqlalchemy.ext import declarative
 from sqlalchemy_fulltext import FullText
 from sqlalchemy_utils import ChoiceType, EmailType, PasswordType
 
-from nyaa import app, db
+from nyaa import app
+from nyaa.extensions import db
 from nyaa.torrents import create_magnet
 
 if app.config['USE_MYSQL']:

nyaa/search.py

@@ -10,7 +10,8 @@ from elasticsearch import Elasticsearch
 from elasticsearch_dsl import Q, Search
 from sqlalchemy_fulltext import FullTextSearch
 
-from nyaa import app, db, models
+from nyaa import app, models
+from nyaa.extensions import db
 
 DEFAULT_MAX_SEARCH_RESULT = 1000
 DEFAULT_PER_PAGE = 75

nyaa/template_utils.py

@@ -1,17 +1,40 @@
 import os.path
+from base64 import b32encode
 from datetime import datetime
 from email.utils import formatdate
+from urllib.parse import urlencode
 
 import flask
 from werkzeug.urls import url_encode
 
 from nyaa import app
 from nyaa.backend import get_category_id_map
+from nyaa.torrents import get_default_trackers
 
 bp = flask.Blueprint('template-utils', __name__)
 _static_cache = {}  # For static_cachebuster
 
 
+# ######################## CONTEXT PROCESSORS ########################
+
+# For processing ES links
+@bp.app_context_processor
+def create_magnet_from_es_info():
+    def _create_magnet_from_es_info(display_name, info_hash, max_trackers=5, trackers=None):
+        if trackers is None:
+            trackers = get_default_trackers()
+
+        magnet_parts = [
+            ('dn', display_name)
+        ]
+        for tracker in trackers[:max_trackers]:
+            magnet_parts.append(('tr', tracker))
+
+        b32_info_hash = b32encode(bytes.fromhex(info_hash)).decode('utf-8')
+        return 'magnet:?xt=urn:btih:' + b32_info_hash + '&' + urlencode(magnet_parts)
+    return dict(create_magnet_from_es_info=_create_magnet_from_es_info)
+
+
 # ######################### TEMPLATE GLOBALS #########################
 
 @bp.app_template_global()

nyaa/torrents.py

@@ -92,24 +92,6 @@ def create_magnet(torrent, max_trackers=5, trackers=None):
     return 'magnet:?xt=urn:btih:' + b32_info_hash + '&' + urlencode(magnet_parts)
 
 
-# For processing ES links
-@app.context_processor
-def create_magnet_from_es_info():
-    def _create_magnet_from_es_info(display_name, info_hash, max_trackers=5, trackers=None):
-        if trackers is None:
-            trackers = get_default_trackers()
-
-        magnet_parts = [
-            ('dn', display_name)
-        ]
-        for tracker in trackers[:max_trackers]:
-            magnet_parts.append(('tr', tracker))
-
-        b32_info_hash = base64.b32encode(bytes.fromhex(info_hash)).decode('utf-8')
-        return 'magnet:?xt=urn:btih:' + b32_info_hash + '&' + urlencode(magnet_parts)
-    return dict(create_magnet_from_es_info=_create_magnet_from_es_info)
-
-
 def create_default_metadata_base(torrent, trackers=None, webseeds=None):
     if trackers is None or webseeds is None:
         db_trackers, db_webseeds = get_trackers_and_webseeds(torrent)

nyaa/views/account.py

@@ -6,7 +6,8 @@ from ipaddress import ip_address
 
 import flask
 
-from nyaa import app, db, forms, models
+from nyaa import app, forms, models
+from nyaa.extensions import db
 from nyaa.views.users import get_activation_link
 
 bp = flask.Blueprint('account', __name__)

nyaa/views/admin.py

@@ -1,6 +1,7 @@
 import flask
 
-from nyaa import db, forms, models
+from nyaa import forms, models
+from nyaa.extensions import db
 
 bp = flask.Blueprint('admin', __name__)
 

nyaa/views/torrents.py

@@ -7,7 +7,8 @@ from werkzeug.datastructures import CombinedMultiDict
 
 from sqlalchemy.orm import joinedload
 
-from nyaa import app, backend, db, forms, models, torrents
+from nyaa import app, backend, forms, models, torrents
+from nyaa.extensions import db
 from nyaa.utils import cached_function
 
 bp = flask.Blueprint('torrents', __name__)

nyaa/views/users.py

@@ -5,7 +5,8 @@ from flask_paginate import Pagination
 
 from itsdangerous import BadSignature, URLSafeSerializer
 
-from nyaa import app, db, forms, models
+from nyaa import app, forms, models
+from nyaa.extensions import db
 from nyaa.search import (DEFAULT_MAX_SEARCH_RESULT, DEFAULT_PER_PAGE, SERACH_PAGINATE_DISPLAY_MSG,
                          _generate_query_string, search_db, search_elastic)
 from nyaa.utils import chain_get