diff --git a/migrations/versions/ecb0b3b88142_flags2columns.py b/migrations/versions/ecb0b3b88142_flags2columns.py deleted file mode 100644 index 9f20aaf..0000000 --- a/migrations/versions/ecb0b3b88142_flags2columns.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Convert bitflags to seperate indexed columns - -Revision ID: ecb0b3b88142 -Revises: 6cc823948c5a -Create Date: 2018-04-08 02:52:44.178958 - -""" -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import mysql -import sys - -# revision identifiers, used by Alembic. -revision = 'ecb0b3b88142' -down_revision = '6cc823948c5a' -branch_labels = None -depends_on = None - - -def upgrade(): - op.execute( - "ALTER TABLE nyaa_torrents " - "ADD COLUMN anonymous BOOL NOT NULL, " - "ADD COLUMN banned BOOL NOT NULL, " - "ADD COLUMN comment_locked BOOL NOT NULL, " - "ADD COLUMN complete BOOL NOT NULL, " - "ADD COLUMN deleted BOOL NOT NULL, " - "ADD COLUMN hidden BOOL NOT NULL, " - "ADD COLUMN remake BOOL NOT NULL, " - "ADD COLUMN trusted BOOL NOT NULL;" - ) - - op.create_index(op.f('ix_nyaa_torrents_anonymous'), 'nyaa_torrents', ['anonymous'], unique=False) - op.create_index(op.f('ix_nyaa_torrents_banned'), 'nyaa_torrents', ['banned'], unique=False) - op.create_index(op.f('ix_nyaa_torrents_comment_locked'), 'nyaa_torrents', ['comment_locked'], unique=False) - op.create_index(op.f('ix_nyaa_torrents_complete'), 'nyaa_torrents', ['complete'], unique=False) - op.create_index(op.f('ix_nyaa_torrents_deleted'), 'nyaa_torrents', ['deleted'], unique=False) - op.create_index(op.f('ix_nyaa_torrents_hidden'), 'nyaa_torrents', ['hidden'], unique=False) - op.create_index(op.f('ix_nyaa_torrents_remake'), 'nyaa_torrents', ['remake'], unique=False) - op.create_index(op.f('ix_nyaa_torrents_trusted'), 'nyaa_torrents', ['trusted'], unique=False) - - op.drop_index('ix_nyaa_torrents_flags', table_name='nyaa_torrents') - op.create_index(op.f('ix_nyaa_torrents_uploader_id'), 'nyaa_torrents', ['uploader_id'], unique=False) - op.drop_index('uploader_flag_idx', table_name='nyaa_torrents') - op.create_index('ix_nyaa_super', 'nyaa_torrents', ['id', 'uploader_id', 'main_category_id', 'sub_category_id', 'anonymous', 'hidden', 'deleted', 'banned', 'trusted', 'remake', 'complete'], unique=False) - - op.execute('UPDATE nyaa_torrents SET anonymous = TRUE WHERE flags & 1 IS TRUE;') - op.execute('UPDATE nyaa_torrents SET hidden = TRUE WHERE flags & 2 IS TRUE;') - op.execute('UPDATE nyaa_torrents SET trusted = TRUE WHERE flags & 4 IS TRUE;') - op.execute('UPDATE nyaa_torrents SET remake = TRUE WHERE flags & 8 IS TRUE;') - op.execute('UPDATE nyaa_torrents SET complete = TRUE WHERE flags & 16 IS TRUE;') - op.execute('UPDATE nyaa_torrents SET deleted = TRUE WHERE flags & 32 IS TRUE;') - op.execute('UPDATE nyaa_torrents SET banned = TRUE WHERE flags & 64 IS TRUE;') - op.execute('UPDATE nyaa_torrents SET comment_locked = TRUE WHERE flags & 128 IS TRUE;') - - #op.drop_column('nyaa_torrents', 'flags') - - op.execute( - "ALTER TABLE sukebei_torrents " - "ADD COLUMN anonymous BOOL NOT NULL, " - "ADD COLUMN banned BOOL NOT NULL, " - "ADD COLUMN comment_locked BOOL NOT NULL, " - "ADD COLUMN complete BOOL NOT NULL, " - "ADD COLUMN deleted BOOL NOT NULL, " - "ADD COLUMN hidden BOOL NOT NULL, " - "ADD COLUMN remake BOOL NOT NULL, " - "ADD COLUMN trusted BOOL NOT NULL;" - ) - - op.create_index(op.f('ix_sukebei_torrents_anonymous'), 'sukebei_torrents', ['anonymous'], unique=False) - op.create_index(op.f('ix_sukebei_torrents_banned'), 'sukebei_torrents', ['banned'], unique=False) - op.create_index(op.f('ix_sukebei_torrents_comment_locked'), 'sukebei_torrents', ['comment_locked'], unique=False) - op.create_index(op.f('ix_sukebei_torrents_complete'), 'sukebei_torrents', ['complete'], unique=False) - op.create_index(op.f('ix_sukebei_torrents_deleted'), 'sukebei_torrents', ['deleted'], unique=False) - op.create_index(op.f('ix_sukebei_torrents_hidden'), 'sukebei_torrents', ['hidden'], unique=False) - op.create_index(op.f('ix_sukebei_torrents_remake'), 'sukebei_torrents', ['remake'], unique=False) - op.create_index(op.f('ix_sukebei_torrents_trusted'), 'sukebei_torrents', ['trusted'], unique=False) - - op.drop_index('ix_sukebei_torrents_flags', table_name='sukebei_torrents') - op.create_index(op.f('ix_sukebei_torrents_uploader_id'), 'sukebei_torrents', ['uploader_id'], unique=False) - op.drop_index('uploader_flag_idx', table_name='sukebei_torrents') - op.create_index('ix_sukebei_super', 'sukebei_torrents', ['id', 'uploader_id', 'main_category_id', 'sub_category_id', 'anonymous', 'hidden', 'deleted', 'banned', 'trusted', 'remake', 'complete'], unique=False) - - op.execute('UPDATE sukebei_torrents SET anonymous = TRUE WHERE flags & 1 IS TRUE;') - op.execute('UPDATE sukebei_torrents SET hidden = TRUE WHERE flags & 2 IS TRUE;') - op.execute('UPDATE sukebei_torrents SET trusted = TRUE WHERE flags & 4 IS TRUE;') - op.execute('UPDATE sukebei_torrents SET remake = TRUE WHERE flags & 8 IS TRUE;') - op.execute('UPDATE sukebei_torrents SET complete = TRUE WHERE flags & 16 IS TRUE;') - op.execute('UPDATE sukebei_torrents SET deleted = TRUE WHERE flags & 32 IS TRUE;') - op.execute('UPDATE sukebei_torrents SET banned = TRUE WHERE flags & 64 IS TRUE;') - op.execute('UPDATE sukebei_torrents SET comment_locked = TRUE WHERE flags & 128 IS TRUE;') - - #op.drop_column('sukebei_torrents', 'flags') - - -def downgrade(): - print("downgrade not supported") - sys.exit(1) diff --git a/nyaa/models.py b/nyaa/models.py index 7e52647..5e846d9 100644 --- a/nyaa/models.py +++ b/nyaa/models.py @@ -67,6 +67,42 @@ class DeclarativeHelperBase(object): return cls._table_prefix(cls.__tablename_base__) +class FlagProperty(object): + ''' This class will act as a wrapper between the given flag and the class's + flag collection. ''' + + def __init__(self, flag, flags_attr='flags'): + self._flag = flag + self._flags_attr_name = flags_attr + + def _get_flags(self, instance): + return getattr(instance, self._flags_attr_name) + + def _set_flags(self, instance, value): + return setattr(instance, self._flags_attr_name, value) + + def __get__(self, instance, owner_class): + if instance is None: + raise AttributeError() + return bool(self._get_flags(instance) & self._flag) + + def __set__(self, instance, value): + new_flags = (self._get_flags(instance) & ~self._flag) | (bool(value) and self._flag) + self._set_flags(instance, new_flags) + + +class TorrentFlags(IntEnum): + NONE = 0 + ANONYMOUS = 1 + HIDDEN = 2 + TRUSTED = 4 + REMAKE = 8 + COMPLETE = 16 + DELETED = 32 + BANNED = 64 + COMMENT_LOCKED = 128 + + class TorrentBase(DeclarativeHelperBase): __tablename_base__ = 'torrents' @@ -80,11 +116,12 @@ class TorrentBase(DeclarativeHelperBase): filesize = db.Column(db.BIGINT, default=0, nullable=False, index=True) encoding = db.Column(db.String(length=32), nullable=False) + flags = db.Column(db.Integer, default=0, nullable=False, index=True) @declarative.declared_attr def uploader_id(cls): # Even though this is same for both tables, declarative requires this - return db.Column(db.Integer, db.ForeignKey('users.id'), nullable=True, index=True) + return db.Column(db.Integer, db.ForeignKey('users.id'), nullable=True) uploader_ip = db.Column(db.Binary(length=16), default=None, nullable=True) has_torrent = db.Column(db.Boolean, nullable=False, default=False) @@ -95,15 +132,6 @@ class TorrentBase(DeclarativeHelperBase): updated_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False) - anonymous = db.Column(db.Boolean, nullable=False, default=False, index=True) - hidden = db.Column(db.Boolean, nullable=False, default=False, index=True) - deleted = db.Column(db.Boolean, nullable=False, default=False, index=True) - banned = db.Column(db.Boolean, nullable=False, default=False, index=True) - trusted = db.Column(db.Boolean, nullable=False, default=False, index=True) - remake = db.Column(db.Boolean, nullable=False, default=False, index=True) - complete = db.Column(db.Boolean, nullable=False, default=False, index=True) - comment_locked = db.Column(db.Boolean, nullable=False, default=False, index=True) - @declarative.declared_attr def main_category_id(cls): fk = db.ForeignKey(cls._table_prefix('main_categories.id')) @@ -119,9 +147,7 @@ class TorrentBase(DeclarativeHelperBase): @declarative.declared_attr def __table_args__(cls): return ( - Index('ix_' + cls._table_prefix('super'), 'id', 'uploader_id', - 'main_category_id', 'sub_category_id', - 'anonymous', 'hidden', 'deleted', 'banned', 'trusted', 'remake', 'complete'), + Index(cls._table_prefix('uploader_flag_idx'), 'uploader_id', 'flags'), ForeignKeyConstraint( ['main_category_id', 'sub_category_id'], [cls._table_prefix('sub_categories.main_category_id'), @@ -225,6 +251,17 @@ class TorrentBase(DeclarativeHelperBase): if self.uploader_ip: return str(ip_address(self.uploader_ip)) + # Flag properties below + + anonymous = FlagProperty(TorrentFlags.ANONYMOUS) + hidden = FlagProperty(TorrentFlags.HIDDEN) + deleted = FlagProperty(TorrentFlags.DELETED) + banned = FlagProperty(TorrentFlags.BANNED) + trusted = FlagProperty(TorrentFlags.TRUSTED) + remake = FlagProperty(TorrentFlags.REMAKE) + complete = FlagProperty(TorrentFlags.COMPLETE) + comment_locked = FlagProperty(TorrentFlags.COMMENT_LOCKED) + # Class methods @classmethod diff --git a/nyaa/search.py b/nyaa/search.py index 59d3a72..364aeb6 100644 --- a/nyaa/search.py +++ b/nyaa/search.py @@ -8,7 +8,6 @@ import sqlalchemy import sqlalchemy_fulltext.modes as FullTextMode from elasticsearch import Elasticsearch from elasticsearch_dsl import Q, Search -from sqlalchemy.sql.expression import false from sqlalchemy_fulltext import FullTextSearch from nyaa import models @@ -295,9 +294,9 @@ def search_db(term='', user=None, sort='id', order='desc', category='0_0', filter_keys = { '0': None, - '1': (models.Torrent.remake, False), - '2': (models.Torrent.trusted, True), - '3': (models.Torrent.complete, True) + '1': (models.TorrentFlags.REMAKE, False), + '2': (models.TorrentFlags.TRUSTED, True), + '3': (models.TorrentFlags.COMPLETE, True) } sentinel = object() @@ -357,7 +356,8 @@ def search_db(term='', user=None, sort='id', order='desc', category='0_0', if not admin: # Hide all DELETED torrents if regular user - qpc.filter(models.Torrent.deleted == false()) + qpc.filter(models.Torrent.flags.op('&')( + int(models.TorrentFlags.DELETED)).is_(False)) # If logged in user is not the same as the user being viewed, # show only torrents that aren't hidden or anonymous # @@ -367,21 +367,24 @@ def search_db(term='', user=None, sort='id', order='desc', category='0_0', # On RSS pages in user view, # show only torrents that aren't hidden or anonymous no matter what if not same_user or rss: - qpc.filter((models.Torrent.hidden == false()) & - (models.Torrent.anonymous == false())) + qpc.filter(models.Torrent.flags.op('&')( + int(models.TorrentFlags.HIDDEN | models.TorrentFlags.ANONYMOUS)).is_(False)) # General view (homepage, general search view) else: if not admin: # Hide all DELETED torrents if regular user - qpc.filter(models.Torrent.deleted == false()) + qpc.filter(models.Torrent.flags.op('&')( + int(models.TorrentFlags.DELETED)).is_(False)) # If logged in, show all torrents that aren't hidden unless they belong to you # On RSS pages, show all public torrents and nothing more. if logged_in_user and not rss: - qpc.filter((models.Torrent.hidden == false()) | - (models.Torrent.uploader_id == logged_in_user.id)) + qpc.filter( + (models.Torrent.flags.op('&')(int(models.TorrentFlags.HIDDEN)).is_(False)) | + (models.Torrent.uploader_id == logged_in_user.id)) # Otherwise, show all torrents that aren't hidden else: - qpc.filter(models.Torrent.hidden == false()) + qpc.filter(models.Torrent.flags.op('&')( + int(models.TorrentFlags.HIDDEN)).is_(False)) if main_category: qpc.filter(models.Torrent.main_category_id == main_cat_id) @@ -390,7 +393,8 @@ def search_db(term='', user=None, sort='id', order='desc', category='0_0', (models.Torrent.sub_category_id == sub_cat_id)) if filter_tuple: - qpc.filter(filter_tuple[0] == filter_tuple[1]) + qpc.filter(models.Torrent.flags.op('&')( + int(filter_tuple[0])).is_(filter_tuple[1])) if term: for item in shlex.split(term, posix=False): @@ -399,16 +403,11 @@ def search_db(term='', user=None, sort='id', order='desc', category='0_0', item, models.TorrentNameSearch, FullTextMode.NATURAL)) query, count_query = qpc.items - super_index = 'ix_' + models.Torrent._table_prefix('super') # Sort and order if sort_column.class_ != models.Torrent: index_name = _get_index_name(sort_column) query = query.join(sort_column.class_) query = query.with_hint(sort_column.class_, 'USE INDEX ({0})'.format(index_name)) - else: - query = query.with_hint(models.Torrent, 'USE INDEX ({0})'.format(super_index)) - - count_query = count_query.with_hint(models.Torrent, 'USE INDEX ({0})'.format(super_index)) query = query.order_by(getattr(sort_column, order)()) diff --git a/sync_es.py b/sync_es.py index 6077194..c4a9025 100755 --- a/sync_es.py +++ b/sync_es.py @@ -34,6 +34,7 @@ from elasticsearch.helpers import bulk, BulkIndexError from pymysqlreplication import BinLogStreamReader from pymysqlreplication.row_event import UpdateRowsEvent, DeleteRowsEvent, WriteRowsEvent from datetime import datetime +from nyaa.models import TorrentFlags import sys import json import time @@ -78,6 +79,7 @@ def pad_bytes(in_bytes, size): def reindex_torrent(t, index_name): # XXX annoyingly different from import_to_es, and # you need to keep them in sync manually. + f = t['flags'] doc = { "id": t['id'], "display_name": t['display_name'], @@ -92,15 +94,16 @@ def reindex_torrent(t, index_name): "main_category_id": t['main_category_id'], "sub_category_id": t['sub_category_id'], "comment_count": t['comment_count'], - "anonymous": bool(t['anonymous']), - "trusted": bool(t['trusted']), - "remake": bool(t['remake']), - "complete": bool(t['complete']), + # XXX all the bitflags are numbers + "anonymous": bool(f & TorrentFlags.ANONYMOUS), + "trusted": bool(f & TorrentFlags.TRUSTED), + "remake": bool(f & TorrentFlags.REMAKE), + "complete": bool(f & TorrentFlags.COMPLETE), # TODO instead of indexing and filtering later # could delete from es entirely. Probably won't matter # for at least a few months. - "hidden": bool(t['hidden']), - "deleted": bool(t['deleted']), + "hidden": bool(f & TorrentFlags.HIDDEN), + "deleted": bool(f & TorrentFlags.DELETED), "has_torrent": bool(t['has_torrent']), } # update, so we don't delete the stats if present