Rewrite site-specific database models into declarative bases

This allows us to easily extend the bases into Nyaa and Sukebei models, with working FKs and relationships.

Note: the schema will (should!) otherwise match, however; the following index on Torrent was renamed:
uploader_flag_idx will be <prefix>_uploader_flag_idx, as we will be always creating both flavors.
The index rename should not affect SQA, but will potentially break migrations.

nyaa/models.py

@@ -3,10 +3,13 @@ from enum import Enum, IntEnum
 from datetime import datetime, timezone
 from nyaa import app, db
 from nyaa.torrents import create_magnet
+
 from sqlalchemy import func, ForeignKeyConstraint, Index
+from sqlalchemy.ext import declarative
 from sqlalchemy_utils import ChoiceType, EmailType, PasswordType
-from werkzeug.security import generate_password_hash, check_password_hash
 from sqlalchemy_fulltext import FullText
+
+from werkzeug.security import generate_password_hash, check_password_hash
 from ipaddress import ip_address
 
 import re
@@ -36,6 +39,31 @@ else:
 UTC_EPOCH = datetime.utcfromtimestamp(0)
 
 
+class DeclarativeHelperBase(object):
+    ''' This class eases our nyaa-sukebei shenanigans by automatically adjusting
+        __tablename__ and providing class methods for renaming references. '''
+    # See http://docs.sqlalchemy.org/en/latest/orm/extensions/declarative/api.html
+
+    __tablename_base__ = None
+    __flavor__ = None
+
+    @classmethod
+    def _table_prefix_string(cls):
+        return cls.__flavor__.lower() + '_'
+
+    @classmethod
+    def _table_prefix(cls, table_name):
+        return cls._table_prefix_string() + table_name
+
+    @classmethod
+    def _flavor_prefix(cls, table_name):
+        return cls.__flavor__ + table_name
+
+    @declarative.declared_attr
+    def __tablename__(cls):
+        return cls._table_prefix(cls.__tablename_base__)
+
+
 class TorrentFlags(IntEnum):
     NONE = 0
     ANONYMOUS = 1
@@ -46,16 +74,13 @@ class TorrentFlags(IntEnum):
     DELETED = 32
 
 
-DB_TABLE_PREFIX = app.config['TABLE_PREFIX']
+class TorrentBase(DeclarativeHelperBase):
-
+    __tablename_base__ = 'torrents'
-
-class Torrent(db.Model):
-    __tablename__ = DB_TABLE_PREFIX + 'torrents'
 
     id = db.Column(db.Integer, primary_key=True)
     info_hash = db.Column(BinaryType(length=20), unique=True, nullable=False, index=True)
-    display_name = db.Column(
+    display_name = db.Column(db.String(length=255, collation=COL_UTF8_GENERAL_CI),
-        db.String(length=255, collation=COL_UTF8_GENERAL_CI), nullable=False, index=True)
+                             nullable=False, index=True)
     torrent_name = db.Column(db.String(length=255), nullable=False)
     information = db.Column(db.String(length=255), nullable=False)
     description = db.Column(DescriptionTextType(collation=COL_UTF8MB4_BIN), nullable=False)
@@ -63,45 +88,84 @@ class Torrent(db.Model):
     filesize = db.Column(db.BIGINT, default=0, nullable=False, index=True)
     encoding = db.Column(db.String(length=32), nullable=False)
     flags = db.Column(db.Integer, default=0, nullable=False, index=True)
-    uploader_id = db.Column(db.Integer, db.ForeignKey('users.id'), nullable=True)
+
+    @declarative.declared_attr
+    def uploader_id(cls):
+        # Even though this is same for both tables, declarative requires this
+        return db.Column(db.Integer, db.ForeignKey('users.id'), nullable=True)
+
     uploader_ip = db.Column(db.Binary(length=16), default=None, nullable=True)
     has_torrent = db.Column(db.Boolean, nullable=False, default=False)
 
     created_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow, nullable=False)
-    updated_time = db.Column(db.DateTime(timezone=False),
+    updated_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow,
-                             default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
+                             onupdate=datetime.utcnow, nullable=False)
+
+    @declarative.declared_attr
+    def main_category_id(cls):
+        fk = db.ForeignKey(cls._table_prefix('main_categories.id'))
+        return db.Column(db.Integer, fk, nullable=False)
 
-    main_category_id = db.Column(db.Integer, db.ForeignKey(
-        DB_TABLE_PREFIX + 'main_categories.id'), nullable=False)
     sub_category_id = db.Column(db.Integer, nullable=False)
-    redirect = db.Column(db.Integer, db.ForeignKey(
-        DB_TABLE_PREFIX + 'torrents.id'), nullable=True)
 
-    __table_args__ = (
+    @declarative.declared_attr
-        Index('uploader_flag_idx', 'uploader_id', 'flags'),
+    def redirect(cls):
-        ForeignKeyConstraint(
+        fk = db.ForeignKey(cls._table_prefix('torrents.id'))
-            ['main_category_id', 'sub_category_id'],
+        return db.Column(db.Integer, fk, nullable=True)
-            [DB_TABLE_PREFIX + 'sub_categories.main_category_id',
-                DB_TABLE_PREFIX + 'sub_categories.id']
-        ), {}
-    )
 
-    user = db.relationship('User', uselist=False, back_populates='torrents')
+    @declarative.declared_attr
-    main_category = db.relationship('MainCategory', uselist=False,
+    def __table_args__(cls):
-                                    back_populates='torrents', lazy="joined")
+        return (
-    sub_category = db.relationship('SubCategory', uselist=False, backref='torrents', lazy="joined",
+                Index(cls._table_prefix('uploader_flag_idx'), 'uploader_id', 'flags'),
-                                   primaryjoin=(
+                ForeignKeyConstraint(
-                                       "and_(SubCategory.id == foreign(Torrent.sub_category_id), "
+                    ['main_category_id', 'sub_category_id'],
-                                       "SubCategory.main_category_id == Torrent.main_category_id)"))
+                    [cls._table_prefix('sub_categories.main_category_id'),
-    info = db.relationship('TorrentInfo', uselist=False,
+                        cls._table_prefix('sub_categories.id')]
-                           cascade="all, delete-orphan", back_populates='torrent')
+                ), {}
-    filelist = db.relationship('TorrentFilelist', uselist=False,
+            )
+
+    @declarative.declared_attr
+    def user(cls):
+        return db.relationship('User', uselist=False, back_populates=cls._table_prefix('torrents'))
+
+    @declarative.declared_attr
+    def main_category(cls):
+        return db.relationship(cls._flavor_prefix('MainCategory'), uselist=False,
+                               back_populates='torrents', lazy="joined")
+
+    @declarative.declared_attr
+    def sub_category(cls):
+        join_sql = ("and_({0}SubCategory.id == foreign({0}Torrent.sub_category_id), "
+                    "{0}SubCategory.main_category_id == {0}Torrent.main_category_id)")
+        return db.relationship(cls._flavor_prefix('SubCategory'), uselist=False,
+                               backref='torrents', lazy="joined",
+                               primaryjoin=join_sql.format(cls.__flavor__))
+
+    @declarative.declared_attr
+    def info(cls):
+        return db.relationship(cls._flavor_prefix('TorrentInfo'), uselist=False,
                                cascade="all, delete-orphan", back_populates='torrent')
-    stats = db.relationship('Statistic', uselist=False,
+
-                            cascade="all, delete-orphan", back_populates='torrent', lazy='joined')
+    @declarative.declared_attr
-    trackers = db.relationship('TorrentTrackers', uselist=True, cascade="all, delete-orphan",
+    def filelist(cls):
-                               lazy='joined', order_by='TorrentTrackers.order')
+        return db.relationship(cls._flavor_prefix('TorrentFilelist'), uselist=False,
-    comments = db.relationship('Comment', uselist=True,
+                               cascade="all, delete-orphan", back_populates='torrent')
+
+    @declarative.declared_attr
+    def stats(cls):
+        return db.relationship(cls._flavor_prefix('Statistic'), uselist=False,
+                               cascade="all, delete-orphan", back_populates='torrent',
+                               lazy='joined')
+
+    @declarative.declared_attr
+    def trackers(cls):
+        return db.relationship(cls._flavor_prefix('TorrentTrackers'), uselist=True,
+                               cascade="all, delete-orphan", lazy='joined',
+                               order_by=cls._flavor_prefix('TorrentTrackers.order'))
+
+    @declarative.declared_attr
+    def comments(cls):
+        return db.relationship(cls._flavor_prefix('Comment'), uselist=True,
                                cascade="all, delete-orphan")
 
     def __repr__(self):
@@ -211,44 +275,57 @@ class Torrent(db.Model):
         return cls.by_info_hash(info_hash_bytes)
 
 
-class TorrentNameSearch(FullText, Torrent):
+class TorrentFilelistBase(DeclarativeHelperBase):
-    __fulltext_columns__ = ('display_name',)
+    __tablename_base__ = 'torrents_filelist'
 
-
-class TorrentFilelist(db.Model):
-    __tablename__ = DB_TABLE_PREFIX + 'torrents_filelist'
     __table_args__ = {'mysql_row_format': 'COMPRESSED'}
 
-    torrent_id = db.Column(db.Integer, db.ForeignKey(
+    @declarative.declared_attr
-        DB_TABLE_PREFIX + 'torrents.id', ondelete="CASCADE"), primary_key=True)
+    def torrent_id(cls):
+        fk = db.ForeignKey(cls._table_prefix('torrents.id'), ondelete="CASCADE")
+        return db.Column(db.Integer, fk, primary_key=True)
+
     filelist_blob = db.Column(MediumBlobType, nullable=True)
 
-    torrent = db.relationship('Torrent', uselist=False, back_populates='filelist')
+    @declarative.declared_attr
+    def torrent(cls):
+        return db.relationship(cls._flavor_prefix('Torrent'), uselist=False,
+                               back_populates='filelist')
 
 
-class TorrentInfo(db.Model):
+class TorrentInfoBase(DeclarativeHelperBase):
-    __tablename__ = DB_TABLE_PREFIX + 'torrents_info'
+    __tablename_base__ = 'torrents_info'
+
     __table_args__ = {'mysql_row_format': 'COMPRESSED'}
 
-    torrent_id = db.Column(db.Integer, db.ForeignKey(
+    @declarative.declared_attr
-        DB_TABLE_PREFIX + 'torrents.id', ondelete="CASCADE"), primary_key=True)
+    def torrent_id(cls):
+        return db.Column(db.Integer, db.ForeignKey(
+            cls._table_prefix('torrents.id'), ondelete="CASCADE"), primary_key=True)
     info_dict = db.Column(MediumBlobType, nullable=True)
 
-    torrent = db.relationship('Torrent', uselist=False, back_populates='info')
+    @declarative.declared_attr
+    def torrent(cls):
+        return db.relationship(cls._flavor_prefix('Torrent'), uselist=False, back_populates='info')
 
 
-class Statistic(db.Model):
+class StatisticBase(DeclarativeHelperBase):
-    __tablename__ = DB_TABLE_PREFIX + 'statistics'
+    __tablename_base__ = 'statistics'
 
-    torrent_id = db.Column(db.Integer, db.ForeignKey(
+    @declarative.declared_attr
-        DB_TABLE_PREFIX + 'torrents.id', ondelete="CASCADE"), primary_key=True)
+    def torrent_id(cls):
+        fk = db.ForeignKey(cls._table_prefix('torrents.id'), ondelete="CASCADE")
+        return db.Column(db.Integer, fk, primary_key=True)
 
     seed_count = db.Column(db.Integer, default=0, nullable=False, index=True)
     leech_count = db.Column(db.Integer, default=0, nullable=False, index=True)
     download_count = db.Column(db.Integer, default=0, nullable=False, index=True)
     last_updated = db.Column(db.DateTime(timezone=False))
 
-    torrent = db.relationship('Torrent', uselist=False, back_populates='stats')
+    @declarative.declared_attr
+    def torrent(cls):
+        return db.relationship(cls._flavor_prefix('Torrent'), uselist=False,
+                               back_populates='stats')
 
 
 class Trackers(db.Model):
@@ -264,30 +341,43 @@ class Trackers(db.Model):
         return cls.query.filter_by(uri=uri).first()
 
 
-class TorrentTrackers(db.Model):
+class TorrentTrackersBase(DeclarativeHelperBase):
-    __tablename__ = DB_TABLE_PREFIX + 'torrent_trackers'
+    __tablename_base__ = 'torrent_trackers'
+
+    @declarative.declared_attr
+    def torrent_id(cls):
+        fk = db.ForeignKey(cls._table_prefix('torrents.id'), ondelete="CASCADE")
+        return db.Column(db.Integer, fk, primary_key=True)
+
+    @declarative.declared_attr
+    def tracker_id(cls):
+        fk = db.ForeignKey('trackers.id', ondelete="CASCADE")
+        return db.Column(db.Integer, fk, primary_key=True)
 
-    torrent_id = db.Column(db.Integer, db.ForeignKey(
-        DB_TABLE_PREFIX + 'torrents.id', ondelete="CASCADE"), primary_key=True)
-    tracker_id = db.Column(db.Integer, db.ForeignKey(
-        'trackers.id', ondelete="CASCADE"), primary_key=True)
     order = db.Column(db.Integer, nullable=False, index=True)
 
-    tracker = db.relationship('Trackers', uselist=False, lazy='joined')
+    @declarative.declared_attr
+    def tracker(cls):
+        return db.relationship('Trackers', uselist=False, lazy='joined')
 
     @classmethod
     def by_torrent_id(cls, torrent_id):
         return cls.query.filter_by(torrent_id=torrent_id).order_by(cls.order.desc())
 
 
-class MainCategory(db.Model):
+class MainCategoryBase(DeclarativeHelperBase):
-    __tablename__ = DB_TABLE_PREFIX + 'main_categories'
+    __tablename_base__ = 'main_categories'
 
     id = db.Column(db.Integer, primary_key=True)
     name = db.Column(db.String(length=64), nullable=False)
 
-    sub_categories = db.relationship('SubCategory', back_populates='main_category')
+    @declarative.declared_attr
-    torrents = db.relationship('Torrent', back_populates='main_category')
+    def sub_categories(cls):
+        return db.relationship(cls._flavor_prefix('SubCategory'), back_populates='main_category')
+
+    @declarative.declared_attr
+    def torrents(cls):
+        return db.relationship(cls._flavor_prefix('Torrent'), back_populates='main_category')
 
     def get_category_ids(self):
         return (self.id, 0)
@@ -301,18 +391,22 @@ class MainCategory(db.Model):
         return cls.query.get(id)
 
 
-class SubCategory(db.Model):
+class SubCategoryBase(DeclarativeHelperBase):
-    __tablename__ = DB_TABLE_PREFIX + 'sub_categories'
+    __tablename_base__ = 'sub_categories'
 
     id = db.Column(db.Integer, primary_key=True)
-    main_category_id = db.Column(db.Integer, db.ForeignKey(
+
-        DB_TABLE_PREFIX + 'main_categories.id'), primary_key=True)
+    @declarative.declared_attr
+    def main_category_id(cls):
+        fk = db.ForeignKey(cls._table_prefix('main_categories.id'))
+        return db.Column(db.Integer, fk, primary_key=True)
+
     name = db.Column(db.String(length=64), nullable=False)
 
-    main_category = db.relationship('MainCategory', uselist=False, back_populates='sub_categories')
+    @declarative.declared_attr
-#    torrents = db.relationship('Torrent', back_populates='sub_category'),
+    def main_category(cls):
-#        primaryjoin="and_(Torrent.sub_category_id == foreign(SubCategory.id), "
+        return db.relationship(cls._flavor_prefix('MainCategory'), uselist=False,
-#                    "Torrent.main_category_id == SubCategory.main_category_id)")
+                               back_populates='sub_categories')
 
     def get_category_ids(self):
         return (self.main_category_id, self.id)
@@ -326,17 +420,27 @@ class SubCategory(db.Model):
         return cls.query.get((sub_cat_id, main_cat_id))
 
 
-class Comment(db.Model):
+class CommentBase(DeclarativeHelperBase):
-    __tablename__ = DB_TABLE_PREFIX + 'comments'
+    __tablename_base__ = 'comments'
 
     id = db.Column(db.Integer, primary_key=True)
-    torrent_id = db.Column(db.Integer, db.ForeignKey(
+
-        DB_TABLE_PREFIX + 'torrents.id', ondelete='CASCADE'), nullable=False)
+    @declarative.declared_attr
-    user_id = db.Column(db.Integer, db.ForeignKey('users.id', ondelete='CASCADE'))
+    def torrent_id(cls):
+        return db.Column(db.Integer, db.ForeignKey(
+                cls._table_prefix('torrents.id'), ondelete='CASCADE'), nullable=False)
+
+    @declarative.declared_attr
+    def user_id(cls):
+        return db.Column(db.Integer, db.ForeignKey('users.id', ondelete='CASCADE'))
+
     created_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow)
     text = db.Column(db.String(length=255, collation=COL_UTF8MB4_BIN), nullable=False)
 
-    user = db.relationship('User', uselist=False, back_populates='comments', lazy="joined")
+    @declarative.declared_attr
+    def user(cls):
+        return db.relationship('User', uselist=False,
+                               back_populates=cls._table_prefix('comments'), lazy="joined")
 
     def __repr__(self):
         return '<Comment %r>' % self.id
@@ -376,9 +480,11 @@ class User(db.Model):
     last_login_date = db.Column(db.DateTime(timezone=False), default=None, nullable=True)
     last_login_ip = db.Column(db.Binary(length=16), default=None, nullable=True)
 
-    torrents = db.relationship('Torrent', back_populates='user', lazy='dynamic')
+    nyaa_torrents = db.relationship('NyaaTorrent', back_populates='user', lazy='dynamic')
-    comments = db.relationship('Comment', back_populates='user', lazy='dynamic')
+    nyaa_comments = db.relationship('NyaaComment', back_populates='user', lazy='dynamic')
-    # session = db.relationship('Session', uselist=False, back_populates='user')
+
+    sukebei_torrents = db.relationship('SukebeiTorrent', back_populates='user', lazy='dynamic')
+    sukebei_comments = db.relationship('SukebeiComment', back_populates='user', lazy='dynamic')
 
     def __init__(self, username, email, password):
         self.username = username
@@ -464,12 +570,118 @@ class User(db.Model):
         return self.level >= UserLevelType.TRUSTED
 
 
-# class Session(db.Model):
+# Actually declare our site-specific classes
-#    __tablename__ = 'sessions'
+
-#
+# Torrent
-#    session_id = db.Column(db.Integer, primary_key=True)
+class NyaaTorrent(TorrentBase, db.Model):
-#    user_id = db.Column(db.Integer, db.ForeignKey('users.id'), nullable=False)
+    __flavor__ = 'Nyaa'
-#    login_ip = db.Column(db.Binary(length=16), nullable=True)
+
-#    login_date = db.Column(db.DateTime(timezone=False), nullable=True)
+
-#
+class SukebeiTorrent(TorrentBase, db.Model):
-#    user = db.relationship('User', back_populates='session')
+    __flavor__ = 'Sukebei'
+
+
+# Fulltext models for MySQL
+if app.config['USE_MYSQL']:
+    class NyaaTorrentNameSearch(FullText, NyaaTorrent):
+        __fulltext_columns__ = ('display_name',)
+        __table_args__ = {'extend_existing': True}
+
+    class SukebeiTorrentNameSearch(FullText, SukebeiTorrent):
+        __fulltext_columns__ = ('display_name',)
+        __table_args__ = {'extend_existing': True}
+else:
+    # Bogus classes for Sqlite
+    class NyaaTorrentNameSearch(object):
+        pass
+
+    class SukebeiTorrentNameSearch(object):
+        pass
+
+
+# TorrentFilelist
+class NyaaTorrentFilelist(TorrentFilelistBase, db.Model):
+    __flavor__ = 'Nyaa'
+
+
+class SukebeiTorrentFilelist(TorrentFilelistBase, db.Model):
+    __flavor__ = 'Sukebei'
+
+
+# TorrentInfo
+class NyaaTorrentInfo(TorrentInfoBase, db.Model):
+    __flavor__ = 'Nyaa'
+
+
+class SukebeiTorrentInfo(TorrentInfoBase, db.Model):
+    __flavor__ = 'Sukebei'
+
+
+# Statistic
+class NyaaStatistic(StatisticBase, db.Model):
+    __flavor__ = 'Nyaa'
+
+
+class SukebeiStatistic(StatisticBase, db.Model):
+    __flavor__ = 'Sukebei'
+
+
+# TorrentTrackers
+class NyaaTorrentTrackers(TorrentTrackersBase, db.Model):
+    __flavor__ = 'Nyaa'
+
+
+class SukebeiTorrentTrackers(TorrentTrackersBase, db.Model):
+    __flavor__ = 'Sukebei'
+
+
+# MainCategory
+class NyaaMainCategory(MainCategoryBase, db.Model):
+    __flavor__ = 'Nyaa'
+
+
+class SukebeiMainCategory(MainCategoryBase, db.Model):
+    __flavor__ = 'Sukebei'
+
+
+# SubCategory
+class NyaaSubCategory(SubCategoryBase, db.Model):
+    __flavor__ = 'Nyaa'
+
+
+class SukebeiSubCategory(SubCategoryBase, db.Model):
+    __flavor__ = 'Sukebei'
+
+
+# Comment
+class NyaaComment(CommentBase, db.Model):
+    __flavor__ = 'Nyaa'
+
+
+class SukebeiComment(CommentBase, db.Model):
+    __flavor__ = 'Sukebei'
+
+
+# Choose our defaults for models.Torrent etc
+if app.config['SITE_FLAVOR'] == 'nyaa':
+    Torrent = NyaaTorrent
+    TorrentFilelist = NyaaTorrentFilelist
+    TorrentInfo = NyaaTorrentInfo
+    Statistic = NyaaStatistic
+    TorrentTrackers = NyaaTorrentTrackers
+    MainCategory = NyaaMainCategory
+    SubCategory = NyaaSubCategory
+    Comment = NyaaComment
+
+    TorrentNameSearch = NyaaTorrentNameSearch
+elif app.config['SITE_FLAVOR'] == 'sukebei':
+    Torrent = SukebeiTorrent
+    TorrentFilelist = SukebeiTorrentFilelist
+    TorrentInfo = SukebeiTorrentInfo
+    Statistic = SukebeiStatistic
+    TorrentTrackers = SukebeiTorrentTrackers
+    MainCategory = SukebeiMainCategory
+    SubCategory = SukebeiSubCategory
+    Comment = SukebeiComment
+
+    TorrentNameSearch = SukebeiTorrentNameSearch