import flask from enum import Enum, IntEnum from datetime import datetime, timezone from nyaa import app, db from nyaa.torrents import create_magnet from sqlalchemy import func, ForeignKeyConstraint, Index from sqlalchemy.ext import declarative from sqlalchemy_utils import ChoiceType, EmailType, PasswordType from sqlalchemy_fulltext import FullText from werkzeug.security import generate_password_hash, check_password_hash from ipaddress import ip_address import re import base64 from markupsafe import escape as escape_markup from urllib.parse import urlencode, unquote as unquote_url from hashlib import md5 if app.config['USE_MYSQL']: from sqlalchemy.dialects import mysql BinaryType = mysql.BINARY DescriptionTextType = mysql.TEXT MediumBlobType = mysql.MEDIUMBLOB COL_UTF8_GENERAL_CI = 'utf8_general_ci' COL_UTF8MB4_BIN = 'utf8mb4_bin' COL_ASCII_GENERAL_CI = 'ascii_general_ci' else: BinaryType = db.Binary DescriptionTextType = db.String MediumBlobType = db.BLOB COL_UTF8_GENERAL_CI = 'NOCASE' COL_UTF8MB4_BIN = None COL_ASCII_GENERAL_CI = 'NOCASE' # For property timestamps UTC_EPOCH = datetime.utcfromtimestamp(0) class DeclarativeHelperBase(object): ''' This class eases our nyaa-sukebei shenanigans by automatically adjusting __tablename__ and providing class methods for renaming references. ''' # See http://docs.sqlalchemy.org/en/latest/orm/extensions/declarative/api.html __tablename_base__ = None __flavor__ = None @classmethod def _table_prefix_string(cls): return cls.__flavor__.lower() + '_' @classmethod def _table_prefix(cls, table_name): return cls._table_prefix_string() + table_name @classmethod def _flavor_prefix(cls, table_name): return cls.__flavor__ + table_name @declarative.declared_attr def __tablename__(cls): return cls._table_prefix(cls.__tablename_base__) class TorrentFlags(IntEnum): NONE = 0 ANONYMOUS = 1 HIDDEN = 2 TRUSTED = 4 REMAKE = 8 COMPLETE = 16 DELETED = 32 class TorrentBase(DeclarativeHelperBase): __tablename_base__ = 'torrents' id = db.Column(db.Integer, primary_key=True) info_hash = db.Column(BinaryType(length=20), unique=True, nullable=False, index=True) display_name = db.Column(db.String(length=255, collation=COL_UTF8_GENERAL_CI), nullable=False, index=True) torrent_name = db.Column(db.String(length=255), nullable=False) information = db.Column(db.String(length=255), nullable=False) description = db.Column(DescriptionTextType(collation=COL_UTF8MB4_BIN), nullable=False) filesize = db.Column(db.BIGINT, default=0, nullable=False, index=True) encoding = db.Column(db.String(length=32), nullable=False) flags = db.Column(db.Integer, default=0, nullable=False, index=True) @declarative.declared_attr def uploader_id(cls): # Even though this is same for both tables, declarative requires this return db.Column(db.Integer, db.ForeignKey('users.id'), nullable=True) uploader_ip = db.Column(db.Binary(length=16), default=None, nullable=True) has_torrent = db.Column(db.Boolean, nullable=False, default=False) comment_count = db.Column(db.Integer, default=0, nullable=False, index=True) created_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow, nullable=False) updated_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False) @declarative.declared_attr def main_category_id(cls): fk = db.ForeignKey(cls._table_prefix('main_categories.id')) return db.Column(db.Integer, fk, nullable=False) sub_category_id = db.Column(db.Integer, nullable=False) @declarative.declared_attr def redirect(cls): fk = db.ForeignKey(cls._table_prefix('torrents.id')) return db.Column(db.Integer, fk, nullable=True) @declarative.declared_attr def __table_args__(cls): return ( Index(cls._table_prefix('uploader_flag_idx'), 'uploader_id', 'flags'), ForeignKeyConstraint( ['main_category_id', 'sub_category_id'], [cls._table_prefix('sub_categories.main_category_id'), cls._table_prefix('sub_categories.id')] ), {} ) @declarative.declared_attr def user(cls): return db.relationship('User', uselist=False, back_populates=cls._table_prefix('torrents')) @declarative.declared_attr def main_category(cls): return db.relationship(cls._flavor_prefix('MainCategory'), uselist=False, back_populates='torrents', lazy="joined") @declarative.declared_attr def sub_category(cls): join_sql = ("and_({0}SubCategory.id == foreign({0}Torrent.sub_category_id), " "{0}SubCategory.main_category_id == {0}Torrent.main_category_id)") return db.relationship(cls._flavor_prefix('SubCategory'), uselist=False, backref='torrents', lazy="joined", primaryjoin=join_sql.format(cls.__flavor__)) @declarative.declared_attr def info(cls): return db.relationship(cls._flavor_prefix('TorrentInfo'), uselist=False, cascade="all, delete-orphan", back_populates='torrent') @declarative.declared_attr def filelist(cls): return db.relationship(cls._flavor_prefix('TorrentFilelist'), uselist=False, cascade="all, delete-orphan", back_populates='torrent') @declarative.declared_attr def stats(cls): return db.relationship(cls._flavor_prefix('Statistic'), uselist=False, cascade="all, delete-orphan", back_populates='torrent', lazy='joined') @declarative.declared_attr def trackers(cls): return db.relationship(cls._flavor_prefix('TorrentTrackers'), uselist=True, cascade="all, delete-orphan", lazy='select', order_by=cls._flavor_prefix('TorrentTrackers.order')) @declarative.declared_attr def comments(cls): return db.relationship(cls._flavor_prefix('Comment'), uselist=True, cascade="all, delete-orphan") def __repr__(self): return '<{0} #{1.id} \'{1.display_name}\' {1.filesize}b>'.format(type(self).__name__, self) def update_comment_count(self): self.comment_count = Comment.query.filter_by(torrent_id=self.id).count() return self.comment_count @property def created_utc_timestamp(self): ''' Returns a UTC POSIX timestamp, as seconds ''' return (self.created_time - UTC_EPOCH).total_seconds() @property def information_as_link(self): ''' Formats the .information into an IRC or HTTP(S) if possible, otherwise escapes it. ''' irc_match = re.match(r'^#([a-zA-Z0-9-_]+)@([a-zA-Z0-9-_.:]+)$', self.information) if irc_match: # Return a formatted IRC uri return '#{0}@{1}'.format(*irc_match.groups()) url_match = re.match(r'^(https?:\/\/.+?)$', self.information) if url_match: url = url_match.group(1) invalid_url_characters = '<>"' # Check if url contains invalid characters if not any(c in url for c in invalid_url_characters): return '{1}'.format(url, escape_markup(unquote_url(url))) # Escaped return escape_markup(self.information) @property def info_hash_as_b32(self): return base64.b32encode(self.info_hash).decode('utf-8') @property def info_hash_as_hex(self): return self.info_hash.hex() @property def magnet_uri(self): return create_magnet(self) @property def uploader_ip_string(self): if self.uploader_ip: return str(ip_address(self.uploader_ip)) # Flag getters and setters below @property def anonymous(self): return self.flags & TorrentFlags.ANONYMOUS @anonymous.setter def anonymous(self, value): self.flags = (self.flags & ~TorrentFlags.ANONYMOUS) | (value and TorrentFlags.ANONYMOUS) @property def hidden(self): return self.flags & TorrentFlags.HIDDEN @hidden.setter def hidden(self, value): self.flags = (self.flags & ~TorrentFlags.HIDDEN) | (value and TorrentFlags.HIDDEN) @property def deleted(self): return self.flags & TorrentFlags.DELETED @deleted.setter def deleted(self, value): self.flags = (self.flags & ~TorrentFlags.DELETED) | (value and TorrentFlags.DELETED) @property def trusted(self): return self.flags & TorrentFlags.TRUSTED @trusted.setter def trusted(self, value): self.flags = (self.flags & ~TorrentFlags.TRUSTED) | (value and TorrentFlags.TRUSTED) @property def remake(self): return self.flags & TorrentFlags.REMAKE @remake.setter def remake(self, value): self.flags = (self.flags & ~TorrentFlags.REMAKE) | (value and TorrentFlags.REMAKE) @property def complete(self): return self.flags & TorrentFlags.COMPLETE @complete.setter def complete(self, value): self.flags = (self.flags & ~TorrentFlags.COMPLETE) | (value and TorrentFlags.COMPLETE) # Class methods @classmethod def by_id(cls, id): return cls.query.get(id) @classmethod def by_info_hash(cls, info_hash): return cls.query.filter_by(info_hash=info_hash).first() @classmethod def by_info_hash_hex(cls, info_hash_hex): info_hash_bytes = bytearray.fromhex(info_hash_hex) return cls.by_info_hash(info_hash_bytes) class TorrentFilelistBase(DeclarativeHelperBase): __tablename_base__ = 'torrents_filelist' __table_args__ = {'mysql_row_format': 'COMPRESSED'} @declarative.declared_attr def torrent_id(cls): fk = db.ForeignKey(cls._table_prefix('torrents.id'), ondelete="CASCADE") return db.Column(db.Integer, fk, primary_key=True) filelist_blob = db.Column(MediumBlobType, nullable=True) @declarative.declared_attr def torrent(cls): return db.relationship(cls._flavor_prefix('Torrent'), uselist=False, back_populates='filelist') class TorrentInfoBase(DeclarativeHelperBase): __tablename_base__ = 'torrents_info' __table_args__ = {'mysql_row_format': 'COMPRESSED'} @declarative.declared_attr def torrent_id(cls): return db.Column(db.Integer, db.ForeignKey( cls._table_prefix('torrents.id'), ondelete="CASCADE"), primary_key=True) info_dict = db.Column(MediumBlobType, nullable=True) @declarative.declared_attr def torrent(cls): return db.relationship(cls._flavor_prefix('Torrent'), uselist=False, back_populates='info') class StatisticBase(DeclarativeHelperBase): __tablename_base__ = 'statistics' @declarative.declared_attr def torrent_id(cls): fk = db.ForeignKey(cls._table_prefix('torrents.id'), ondelete="CASCADE") return db.Column(db.Integer, fk, primary_key=True) seed_count = db.Column(db.Integer, default=0, nullable=False, index=True) leech_count = db.Column(db.Integer, default=0, nullable=False, index=True) download_count = db.Column(db.Integer, default=0, nullable=False, index=True) last_updated = db.Column(db.DateTime(timezone=False)) @declarative.declared_attr def torrent(cls): return db.relationship(cls._flavor_prefix('Torrent'), uselist=False, back_populates='stats') class Trackers(db.Model): __tablename__ = 'trackers' id = db.Column(db.Integer, primary_key=True) uri = db.Column(db.String(length=255, collation=COL_UTF8_GENERAL_CI), nullable=False, unique=True) disabled = db.Column(db.Boolean, nullable=False, default=False) @classmethod def by_uri(cls, uri): return cls.query.filter_by(uri=uri).first() class TorrentTrackersBase(DeclarativeHelperBase): __tablename_base__ = 'torrent_trackers' @declarative.declared_attr def torrent_id(cls): fk = db.ForeignKey(cls._table_prefix('torrents.id'), ondelete="CASCADE") return db.Column(db.Integer, fk, primary_key=True) @declarative.declared_attr def tracker_id(cls): fk = db.ForeignKey('trackers.id', ondelete="CASCADE") return db.Column(db.Integer, fk, primary_key=True) order = db.Column(db.Integer, nullable=False, index=True) @declarative.declared_attr def tracker(cls): return db.relationship('Trackers', uselist=False, lazy='joined') @classmethod def by_torrent_id(cls, torrent_id): return cls.query.filter_by(torrent_id=torrent_id).order_by(cls.order.desc()) class MainCategoryBase(DeclarativeHelperBase): __tablename_base__ = 'main_categories' id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(length=64), nullable=False) @declarative.declared_attr def sub_categories(cls): return db.relationship(cls._flavor_prefix('SubCategory'), back_populates='main_category') @declarative.declared_attr def torrents(cls): return db.relationship(cls._flavor_prefix('Torrent'), back_populates='main_category') def get_category_ids(self): return (self.id, 0) @property def id_as_string(self): return '_'.join(str(x) for x in self.get_category_ids()) @classmethod def by_id(cls, id): return cls.query.get(id) class SubCategoryBase(DeclarativeHelperBase): __tablename_base__ = 'sub_categories' id = db.Column(db.Integer, primary_key=True) @declarative.declared_attr def main_category_id(cls): fk = db.ForeignKey(cls._table_prefix('main_categories.id')) return db.Column(db.Integer, fk, primary_key=True) name = db.Column(db.String(length=64), nullable=False) @declarative.declared_attr def main_category(cls): return db.relationship(cls._flavor_prefix('MainCategory'), uselist=False, back_populates='sub_categories') def get_category_ids(self): return (self.main_category_id, self.id) @property def id_as_string(self): return '_'.join(str(x) for x in self.get_category_ids()) @classmethod def by_category_ids(cls, main_cat_id, sub_cat_id): return cls.query.get((sub_cat_id, main_cat_id)) class CommentBase(DeclarativeHelperBase): __tablename_base__ = 'comments' id = db.Column(db.Integer, primary_key=True) @declarative.declared_attr def torrent_id(cls): return db.Column(db.Integer, db.ForeignKey( cls._table_prefix('torrents.id'), ondelete='CASCADE'), nullable=False) @declarative.declared_attr def user_id(cls): return db.Column(db.Integer, db.ForeignKey('users.id', ondelete='CASCADE')) created_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow) text = db.Column(db.String(length=255, collation=COL_UTF8MB4_BIN), nullable=False) @declarative.declared_attr def user(cls): return db.relationship('User', uselist=False, back_populates=cls._table_prefix('comments'), lazy="joined") def __repr__(self): return '' % self.id @property def created_utc_timestamp(self): ''' Returns a UTC POSIX timestamp, as seconds ''' return (self.created_time - UTC_EPOCH).total_seconds() class UserLevelType(IntEnum): REGULAR = 0 TRUSTED = 1 MODERATOR = 2 SUPERADMIN = 3 class UserStatusType(Enum): INACTIVE = 0 ACTIVE = 1 BANNED = 2 class User(db.Model): __tablename__ = 'users' id = db.Column(db.Integer, primary_key=True) username = db.Column(db.String(length=32, collation=COL_ASCII_GENERAL_CI), unique=True, nullable=False) email = db.Column(EmailType(length=255, collation=COL_ASCII_GENERAL_CI), unique=True, nullable=True) password_hash = db.Column(PasswordType(max_length=255, schemes=['argon2']), nullable=False) status = db.Column(ChoiceType(UserStatusType, impl=db.Integer()), nullable=False) level = db.Column(ChoiceType(UserLevelType, impl=db.Integer()), nullable=False) created_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow) last_login_date = db.Column(db.DateTime(timezone=False), default=None, nullable=True) last_login_ip = db.Column(db.Binary(length=16), default=None, nullable=True) nyaa_torrents = db.relationship('NyaaTorrent', back_populates='user', lazy='dynamic') nyaa_comments = db.relationship('NyaaComment', back_populates='user', lazy='dynamic') sukebei_torrents = db.relationship('SukebeiTorrent', back_populates='user', lazy='dynamic') sukebei_comments = db.relationship('SukebeiComment', back_populates='user', lazy='dynamic') def __init__(self, username, email, password): self.username = username self.email = email self.password_hash = password self.status = UserStatusType.INACTIVE self.level = UserLevelType.REGULAR def __repr__(self): return '' % self.username def validate_authorization(self, password): ''' Returns a boolean for whether the user can be logged in ''' checks = [ # Password must match password == self.password_hash, # Reject inactive and banned users self.status == UserStatusType.ACTIVE ] return all(checks) def gravatar_url(self): # from http://en.gravatar.com/site/implement/images/python/ size = 120 # construct the url default_avatar = flask.url_for('static', filename='img/avatar/default.png', _external=True) gravatar_url = 'https://www.gravatar.com/avatar/{}?{}'.format( md5(self.email.encode('utf-8').lower()).hexdigest(), urlencode({'d': default_avatar, 's': str(size)})) return gravatar_url @property def userlevel_str(self): if self.level == UserLevelType.REGULAR: return 'User' elif self.level == UserLevelType.TRUSTED: return 'Trusted' elif self.level >= UserLevelType.MODERATOR: return 'Moderator' @property def userlevel_color(self): if self.level == UserLevelType.REGULAR: return 'default' elif self.level == UserLevelType.TRUSTED: return 'success' elif self.level >= UserLevelType.MODERATOR: return 'purple' @property def ip_string(self): if self.last_login_ip: return str(ip_address(self.last_login_ip)) @classmethod def by_id(cls, id): return cls.query.get(id) @classmethod def by_username(cls, username): user = cls.query.filter_by(username=username).first() return user @classmethod def by_email(cls, email): user = cls.query.filter_by(email=email).first() return user @classmethod def by_username_or_email(cls, username_or_email): return cls.by_username(username_or_email) or cls.by_email(username_or_email) @property def is_moderator(self): return self.level >= UserLevelType.MODERATOR @property def is_superadmin(self): return self.level == UserLevelType.SUPERADMIN @property def is_trusted(self): return self.level >= UserLevelType.TRUSTED class ReportStatus(IntEnum): IN_REVIEW = 0 VALID = 1 INVALID = 2 class ReportBase(DeclarativeHelperBase): __tablename_base__ = 'reports' id = db.Column(db.Integer, primary_key=True) created_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow) reason = db.Column(db.String(length=255), nullable=False) status = db.Column(ChoiceType(ReportStatus, impl=db.Integer()), nullable=False) @declarative.declared_attr def torrent_id(cls): return db.Column(db.Integer, db.ForeignKey( cls._table_prefix('torrents.id'), ondelete='CASCADE'), nullable=False) @declarative.declared_attr def user_id(cls): return db.Column(db.Integer, db.ForeignKey('users.id')) @declarative.declared_attr def user(cls): return db.relationship('User', uselist=False, lazy="joined") @declarative.declared_attr def torrent(cls): return db.relationship(cls._flavor_prefix('Torrent'), uselist=False, lazy="joined") def __init__(self, torrent_id, user_id, reason): self.torrent_id = torrent_id self.user_id = user_id self.reason = reason self.status = ReportStatus.IN_REVIEW def __repr__(self): return '' % self.id @property def created_utc_timestamp(self): ''' Returns a UTC POSIX timestamp, as seconds ''' return (self.created_time - UTC_EPOCH).total_seconds() @classmethod def by_id(cls, id): return cls.query.get(id) @classmethod def not_reviewed(cls, page): reports = cls.query.filter_by(status=0).paginate(page=page, per_page=20) return reports @classmethod def remove_reviewed(cls, id): return cls.query.filter(cls.torrent_id == id, cls.status == 0).delete() # Actually declare our site-specific classes # Torrent class NyaaTorrent(TorrentBase, db.Model): __flavor__ = 'Nyaa' class SukebeiTorrent(TorrentBase, db.Model): __flavor__ = 'Sukebei' # Fulltext models for MySQL if app.config['USE_MYSQL']: class NyaaTorrentNameSearch(FullText, NyaaTorrent): __fulltext_columns__ = ('display_name',) __table_args__ = {'extend_existing': True} class SukebeiTorrentNameSearch(FullText, SukebeiTorrent): __fulltext_columns__ = ('display_name',) __table_args__ = {'extend_existing': True} else: # Bogus classes for Sqlite class NyaaTorrentNameSearch(object): pass class SukebeiTorrentNameSearch(object): pass # TorrentFilelist class NyaaTorrentFilelist(TorrentFilelistBase, db.Model): __flavor__ = 'Nyaa' class SukebeiTorrentFilelist(TorrentFilelistBase, db.Model): __flavor__ = 'Sukebei' # TorrentInfo class NyaaTorrentInfo(TorrentInfoBase, db.Model): __flavor__ = 'Nyaa' class SukebeiTorrentInfo(TorrentInfoBase, db.Model): __flavor__ = 'Sukebei' # Statistic class NyaaStatistic(StatisticBase, db.Model): __flavor__ = 'Nyaa' class SukebeiStatistic(StatisticBase, db.Model): __flavor__ = 'Sukebei' # TorrentTrackers class NyaaTorrentTrackers(TorrentTrackersBase, db.Model): __flavor__ = 'Nyaa' class SukebeiTorrentTrackers(TorrentTrackersBase, db.Model): __flavor__ = 'Sukebei' # MainCategory class NyaaMainCategory(MainCategoryBase, db.Model): __flavor__ = 'Nyaa' class SukebeiMainCategory(MainCategoryBase, db.Model): __flavor__ = 'Sukebei' # SubCategory class NyaaSubCategory(SubCategoryBase, db.Model): __flavor__ = 'Nyaa' class SukebeiSubCategory(SubCategoryBase, db.Model): __flavor__ = 'Sukebei' # Comment class NyaaComment(CommentBase, db.Model): __flavor__ = 'Nyaa' class SukebeiComment(CommentBase, db.Model): __flavor__ = 'Sukebei' # Report class NyaaReport(ReportBase, db.Model): __flavor__ = 'Nyaa' class SukebeiReport(ReportBase, db.Model): __flavor__ = 'Sukebei' # Choose our defaults for models.Torrent etc if app.config['SITE_FLAVOR'] == 'nyaa': Torrent = NyaaTorrent TorrentFilelist = NyaaTorrentFilelist TorrentInfo = NyaaTorrentInfo Statistic = NyaaStatistic TorrentTrackers = NyaaTorrentTrackers MainCategory = NyaaMainCategory SubCategory = NyaaSubCategory Comment = NyaaComment Report = NyaaReport TorrentNameSearch = NyaaTorrentNameSearch elif app.config['SITE_FLAVOR'] == 'sukebei': Torrent = SukebeiTorrent TorrentFilelist = SukebeiTorrentFilelist TorrentInfo = SukebeiTorrentInfo Statistic = SukebeiStatistic TorrentTrackers = SukebeiTorrentTrackers MainCategory = SukebeiMainCategory SubCategory = SukebeiSubCategory Comment = SukebeiComment Report = SukebeiReport TorrentNameSearch = SukebeiTorrentNameSearch