Initial commit.

README.md

@@ -0,0 +1,48 @@
+# NyaaV2
+
+## Setup:
+
+- Create your virtualenv, for example with `pyvenv venv`
+- Enter your virtualenv with `source venv/bin/activate`
+- Install dependencies with `pip install -r requirements.txt`
+- Run `python db_create.py` to create the database
+- Start the dev server with `python run.py`
+
+## Updated Setup (python 3.6.1):
+
+- Install dependencies https://github.com/pyenv/pyenv/wiki/Common-build-problems
+- Install `pyenv` https://github.com/pyenv/pyenv/blob/master/README.md#installation
+- Install `pyenv-virtualenv` https://github.com/pyenv/pyenv-virtualenv/blob/master/README.md
+- `pyenv install 3.6.1`
+- `pyenv virtualenv 3.6.1 nyaa`
+- `pyenv activate nyaa`
+- Install dependencies with `pip install -r requirements.txt`
+- Copy `config.example.py` into `config.py`
+- Change TALBE_PREFIX to `nyaa_` or `sukebei_` depending on the site
+
+## Setting up MySQL/MariaDB database for advanced functionality
+- Enable `USE_MYSQL` flag in config.py
+- Install latest mariadb by following instructions here https://downloads.mariadb.org/mariadb/repositories/
+    - Tested versions: `mysql  Ver 15.1 Distrib 10.0.30-MariaDB, for debian-linux-gnu (x86_64) using readline 5.2`
+- Run the following commands logged in as your root db user:
+    - `CREATE USER 'test'@'localhost' IDENTIFIED BY 'test123';`
+    - `GRANT ALL PRIVILEGES ON * . * TO 'test'@'localhost';`
+    - `FLUSH PRIVILEGES;`
+    - `CREATE DATABASE nyaav2 DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;`
+- To setup and import nyaa_maria_vx.sql:
+    - `mysql -u <user> -p nyaav2`
+    - `DROP DATABASE nyaav2;`
+    - `CREATE DATABASE nyaav2 DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;`
+    - `SOURCE ~/path/to/database/nyaa_maria_vx.sql`
+
+## Finishing up
+- Run `python db_create.py` to create the database
+- Load the .sql file
+    - `mysql -u user -p nyaav2`
+    - `SOURCE cocks.sql`
+    - Remember to change the default user password to an empty string to disable logging in
+- Start the dev server with `python run.py`
+- Deactivate `source deactivate`
+
+## Code Quality:
+- Remember to follow PEP8 style guidelines and run `./lint.sh` before committing.

WSGI.py

@@ -0,0 +1,15 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import gevent.monkey
+gevent.monkey.patch_all()
+
+from nyaa import app
+
+if app.config["DEBUG"]:
+	from werkzeug.debug import DebuggedApplication
+	app.wsgi_app = DebuggedApplication(app.wsgi_app, True)
+
+if __name__ == '__main__':
+	import gevent.pywsgi
+	gevent_server = gevent.pywsgi.WSGIServer(("localhost", 5000), app.wsgi_app)
+	gevent_server.serve_forever()

api_uploader.py

@@ -0,0 +1,55 @@
+# api_uploader.py
+
+
+# Uploads a single file
+# I will create another script for batch uploading
+
+import json
+import requests
+
+url = "http://127.0.0.1:5500/api/upload"
+
+# Required for Auth
+username = ""
+password = ""
+
+# Required
+torrent_name = ""
+
+# Required
+main_cat = ""
+# Required
+sub_cat = ""
+
+# May be blank
+information = ""
+# May be blank
+description = ""
+# flags = [Hidden, Remake, Complete, Anonymous]
+# 0 for NOT SET / 1 for SET
+# Required
+flags = [0, 0, 0, 0]
+
+metadata={
+    "username": username,
+    "password": password,
+    "display_name": torrent_name,
+    "main_cat": main_cat,
+    "sub_cat": sub_cat,
+    "information": information,
+    "description": description,
+    "flags": flags
+    }
+
+# Required
+file_name = ""
+
+files = {
+    'json': (json.dumps(metadata)),
+    'torrent': ('{0}'.format(file_name), open(file_name, 'rb'), 'application/octet-stream')}
+
+response = requests.post(url, files=files)
+
+json_response = response.json()
+
+print(json_response)

batch_upload_torrent.sh

@@ -0,0 +1,2 @@
+up_t() { curl -F "category=1_2" -F "torrent_file=@$1" 'http://localhost:5500/upload'; }
+for x in test_torrent_batch/*; do up_t "$x"; done

config.example.py

@@ -0,0 +1,51 @@
+import os
+
+
+DEBUG = True
+USE_RECAPTCHA = False
+USE_EMAIL_VERIFICATION = False
+USE_MYSQL = True
+
+# Enable this once stat integration is done
+ENABLE_SHOW_STATS = False
+
+BASE_DIR = os.path.abspath(os.path.dirname(__file__))
+if USE_MYSQL:
+    SQLALCHEMY_DATABASE_URI = ('mysql://test:test123@localhost/nyaav2')
+else:
+    SQLALCHEMY_DATABASE_URI = (
+        'sqlite:///' + os.path.join(BASE_DIR, 'test.db') + '?check_same_thread=False')
+
+CSRF_SESSION_KEY = '***'
+SECRET_KEY = '***'
+
+# Prefix for running multiple sites, user table will not be prefixed.
+# For sukebei, change 'nyaa_' to 'sukebei_'
+TABLE_PREFIX = 'nyaa_'
+
+# for recaptcha and email verification:
+# keys for localhost. Change as appropriate when actual domain is registered.
+RECAPTCHA_PUBLIC_KEY = '***'
+RECAPTCHA_PRIVATE_KEY = '***'
+SMTP_SERVER = '***'
+SMTP_PORT = 587
+MAIL_FROM_ADDRESS = '***'
+SMTP_USERNAME = '***'
+SMTP_PASSWORD = '***'
+
+RESULTS_PER_PAGE = 75
+
+# What the site identifies itself as.
+SITE_NAME = 'Nyaa'
+
+# The maximum number of files a torrent can contain
+# until the site says "Too many files to display."
+MAX_FILES_VIEW = 1000
+
+# """
+# Setting to make sure main announce url is present in torrent
+# """
+ENFORCE_MAIN_ANNOUNCE_URL = False
+MAIN_ANNOUNCE_URL = ''
+
+BACKUP_TORRENT_FOLDER = 'torrents'

db_create.py

@@ -0,0 +1,40 @@
+import sys
+from nyaa import app, db, models
+
+# Create tables
+
+db.create_all()
+
+# Insert categories
+
+if app.config['TABLE_PREFIX'] == 'nyaa_':
+    CATEGORIES = [
+        ('Anime', ['Anime Music Video', 'English-translated', 'Non-English-translated', 'Raw']),
+        ('Audio', ['Lossless', 'Lossy']),
+        ('Literature', ['English-translated', 'Non-English-translated', 'Raw']),
+        ('Live Action', ['English-translated', 'Idol/Promotional Video', 'Non-English-translated', 'Raw']),
+        ('Pictures', ['Graphics', 'Photos']),
+        ('Software', ['Applications', 'Games']),
+    ]
+elif app.config['TABLE_PREFIX'] == 'sukebei_':
+    CATEGORIES = [
+        ('Art', ['Anime', 'Doujinshi', 'Games', 'Manga', 'Pictures']),
+        ('Real Life', ['Photobooks / Pictures', 'Videos']),
+    ]
+else:
+    CATEGORIES = []
+
+for main_cat_name, sub_cat_names in CATEGORIES:
+    main_cat = models.MainCategory(name=main_cat_name)
+    for i, sub_cat_name in enumerate(sub_cat_names):
+        # Composite keys can't autoincrement, set sub_cat id manually (1-index)
+        sub_cat = models.SubCategory(id=i+1, name=sub_cat_name, main_category=main_cat)
+    db.session.add(main_cat)
+
+db.session.commit()
+
+# Create fulltext index
+
+if app.config['USE_MYSQL']:
+    db.engine.execute('ALTER TABLE ' + app.config['TABLE_PREFIX'] + 'torrents ADD FULLTEXT KEY (display_name)')
+

lint.sh

@@ -0,0 +1,6 @@
+autopep8 nyaa/ \
+	--recursive \
+	--in-place \
+	--pep8-passes 2000 \
+	--max-line-length 100 \
+	--verbose

my.cnf

@@ -0,0 +1,6 @@
+[mysqld]
+innodb_ft_min_token_size=2
+ft_min_word_len=2
+innodb_ft_cache_size = 80000000
+innodb_ft_total_cache_size = 1600000000
+max_allowed_packet = 100M

nyaa/__init__.py

@@ -0,0 +1,53 @@
+import os
+import logging
+import flask
+from flask_sqlalchemy import SQLAlchemy
+from flask_assets import Environment, Bundle
+from flask_debugtoolbar import DebugToolbarExtension
+from nyaa import fix_paginate
+
+
+app = flask.Flask(__name__)
+app.config.from_object('config')
+
+# Database
+app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
+
+# Debugging
+if app.config['DEBUG']:
+    app.config['DEBUG_TB_INTERCEPT_REDIRECTS'] = False
+    toolbar = DebugToolbarExtension(app)
+    app.logger.setLevel(logging.DEBUG)
+else:
+    app.logger.setLevel(logging.WARNING)
+
+# Logging
+if 'LOG_FILE' in app.config:
+    from logging.handlers import RotatingFileHandler
+    app.log_handler = RotatingFileHandler(
+        app.config['LOG_FILE'], maxBytes=10000, backupCount=1)
+    app.logger.addHandler(app.log_handler)
+
+# Log errors and display a message to the user in production mdode
+if not app.config['DEBUG']:
+    @app.errorhandler(500)
+    def internal_error(exception):
+        app.logger.error(exception)
+        flask.flash(flask.Markup(
+            '<strong>An error occured!</strong> Debugging information has been logged.'), 'danger')
+        return flask.redirect('/')
+
+# Enable the jinja2 do extension.
+app.jinja_env.add_extension('jinja2.ext.do')
+app.jinja_env.lstrip_blocks = True
+app.jinja_env.trim_blocks = True
+
+db = SQLAlchemy(app)
+
+assets = Environment(app)
+
+# css = Bundle('style.scss', filters='libsass',
+#             output='style.css', depends='**/*.scss')
+# assets.register('style_all', css)
+
+from nyaa import routes

nyaa/api_handler.py

@@ -0,0 +1,317 @@
+import flask
+from nyaa import app, db
+from nyaa import models, forms
+from nyaa import bencode, utils
+from nyaa import torrents
+
+import json
+import os.path
+from orderedset import OrderedSet
+from werkzeug import secure_filename
+
+DEBUG_API = False
+#################################### API ROUTES ####################################
+CATEGORIES = [
+    ('Anime', ['Anime Music Video', 'English-translated', 'Non-English-translated', 'Raw']),
+    ('Audio', ['Lossless', 'Lossy']),
+    ('Literature', ['English-translated', 'Non-English-translated', 'Raw']),
+    ('Live Action', ['English-translated',
+                     'Idol/Promotional Video', 'Non-English-translated', 'Raw']),
+    ('Pictures', ['Graphics', 'Photos']),
+    ('Software', ['Applications', 'Games']),
+]
+
+
+def validate_main_sub_cat(main_cat_name, sub_cat_name):
+    for main_cat in models.MainCategory.query.order_by(models.MainCategory.id):
+        if main_cat_name == main_cat.name:
+            for sub_cat in main_cat.sub_categories:
+                if sub_cat_name == sub_cat.name:
+                    cat_id = main_cat.id_as_string
+                    sub_cat_id = sub_cat.id_as_string
+                    cat_sub_cat = sub_cat_id.split('_')
+                    #print('cat: {0} sub_cat: {1}'.format(cat_sub_cat[0], cat_sub_cat[1]))
+
+                    return True, cat_sub_cat[0], cat_sub_cat[1]
+
+        return False, 0, 0
+
+
+def _replace_utf8_values(dict_or_list):
+    ''' Will replace 'property' with 'property.utf-8' and remove latter if it exists.
+        Thanks, bitcomet! :/ '''
+    did_change = False
+    if isinstance(dict_or_list, dict):
+        for key in [key for key in dict_or_list.keys() if key.endswith('.utf-8')]:
+            dict_or_list[key.replace('.utf-8', '')] = dict_or_list.pop(key)
+            did_change = True
+        for value in dict_or_list.values():
+            did_change = _replace_utf8_values(value) or did_change
+    elif isinstance(dict_or_list, list):
+        for item in dict_or_list:
+            did_change = _replace_utf8_values(item) or did_change
+    return did_change
+
+
+def validate_torrent_flags(torrent_flags):
+    _torrent_flags = ['hidden', 'remake', 'complete', 'anonymous']
+
+    if len(torrent_flags) != 4:
+        return False
+
+    for flag in torrent_flags:
+        if int(flag) not in [0, 1]:
+            return False
+    return True
+
+# It might be good to factor this out of forms UploadForm because the same code is
+# used in both files.
+
+
+def validate_torrent_file(torrent_file_name, torrent_file):
+    # Decode and ensure data is bencoded data
+    try:
+        torrent_dict = bencode.decode(torrent_file)
+    except (bencode.MalformedBencodeException, UnicodeError):
+        return False, 'Malformed torrent file'
+
+    # Uncomment for debug print of the torrent
+    # forms._debug_print_torrent_metadata(torrent_dict)
+
+    try:
+        forms._validate_torrent_metadata(torrent_dict)
+    except AssertionError as e:
+        return False, 'Malformed torrent metadata ({})'.format(e.args[0])
+
+    # Note! bencode will sort dict keys, as per the spec
+    # This may result in a different hash if the uploaded torrent does not match the
+    # spec, but it's their own fault for using broken software! Right?
+    bencoded_info_dict = bencode.encode(torrent_dict['info'])
+    info_hash = utils.sha1_hash(bencoded_info_dict)
+
+    # Check if the info_hash exists already in the database
+    existing_torrent = models.Torrent.by_info_hash(info_hash)
+    if existing_torrent:
+        return False, 'That torrent already exists (#{})'.format(existing_torrent.id)
+
+    # Torrent is legit, pass original filename and dict along
+    return True, forms.TorrentFileData(filename=os.path.basename(torrent_file_name),
+                                       torrent_dict=torrent_dict,
+                                       info_hash=info_hash,
+                                       bencoded_info_dict=bencoded_info_dict)
+
+
+def api_upload(upload_request):
+    if upload_request.method == 'POST':
+        j = None
+        torrent_file = None
+        try:
+            if 'json' in upload_request.files:
+                f = upload_request.files['json']
+                j = json.loads(f.read().decode('utf-8'))
+                if DEBUG_API:
+                    print(json.dumps(j, indent=4))
+
+                _json_keys = ['username', 'password',
+                              'display_name', 'main_cat', 'sub_cat', 'flags']  # 'information' and 'description' are not required
+                # Check that required fields are present
+                for _k in _json_keys:
+                    if _k not in j.keys():
+                        return flask.make_response(flask.jsonify({"Error": "Missing JSON field: {0}.".format(_k)}), 400)
+                # Check that no extra fields are present
+                for k in j.keys():
+                    if k not in ['username', 'password',
+                                 'display_name', 'main_cat', 'sub_cat', 'information', 'description', 'flags']:
+                        return flask.make_response(flask.jsonify({"Error": "Incorrect JSON field(s)."}), 400)
+            else:
+                return flask.make_response(flask.jsonify({"Error": "No metadata."}), 400)
+            if 'torrent' in upload_request.files:
+                f = upload_request.files['torrent']
+                if DEBUG_API:
+                    print(f.filename)
+                torrent_file = f
+                # print(f.read())
+            else:
+                return flask.make_response(flask.jsonify({"Error": "No torrent file."}), 400)
+
+            # 'username' and 'password' must have been provided as they are part of j.keys()
+            username = j['username']
+            password = j['password']
+            # Validate that the provided username and password belong to a valid user
+            user = models.User.by_username(username)
+
+            if not user:
+                user = models.User.by_email(username)
+
+            if not user or password != user.password_hash or user.status == models.UserStatusType.INACTIVE:
+                return flask.make_response(flask.jsonify({"Error": "Incorrect username or password"}), 403)
+
+            current_user = user
+
+            display_name = j['display_name']
+            if (len(display_name) < 3) or (len(display_name) > 1024):
+                return flask.make_response(flask.jsonify({"Error": "Torrent name must be between 3 and 1024 characters."}), 400)
+
+            main_cat_name = j['main_cat']
+            sub_cat_name = j['sub_cat']
+
+            cat_subcat_status, cat_id, sub_cat_id = validate_main_sub_cat(
+                main_cat_name, sub_cat_name)
+            if not cat_subcat_status:
+                return flask.make_response(flask.jsonify({"Error": "Incorrect Category / Sub-Category."}), 400)
+
+            # TODO Sanitize information
+            information = None
+            try:
+                information = j['information']
+                if len(information) > 255:
+                    return flask.make_response(flask.jsonify({"Error": "Information is limited to 255 characters."}), 400)
+            except Exception as e:
+                information = ''
+
+            # TODO Sanitize description
+            description = None
+            try:
+                description = j['description']
+                if len(description) > (10 * 1024):
+                    return flask.make_response(flask.jsonify({"Error": "Description is limited to {0} characters.".format(10 * 1024)}), 403)
+            except Exception as e:
+                description = ''
+
+            v_flags = validate_torrent_flags(j['flags'])
+            if v_flags:
+                torrent_flags = j['flags']
+            else:
+                return flask.make_response(flask.jsonify({"Error": "Incorrect torrent flags."}), 400)
+
+            torrent_status, torrent_data = validate_torrent_file(
+                torrent_file.filename, torrent_file.read())  # Needs validation
+
+            if not torrent_status:
+                return flask.make_response(flask.jsonify({"Error": "Invalid or Duplicate torrent file."}), 400)
+
+            # The torrent has been  validated and is safe to access with ['foo'] etc - all relevant
+            # keys and values have been checked for (see UploadForm in forms.py for details)
+            info_dict = torrent_data.torrent_dict['info']
+
+            changed_to_utf8 = _replace_utf8_values(torrent_data.torrent_dict)
+
+            torrent_filesize = info_dict.get('length') or sum(
+                f['length'] for f in info_dict.get('files'))
+
+            # In case no encoding, assume UTF-8.
+            torrent_encoding = torrent_data.torrent_dict.get('encoding', b'utf-8').decode('utf-8')
+
+            torrent = models.Torrent(info_hash=torrent_data.info_hash,
+                                     display_name=display_name,
+                                     torrent_name=torrent_data.filename,
+                                     information=information,
+                                     description=description,
+                                     encoding=torrent_encoding,
+                                     filesize=torrent_filesize,
+                                     user=current_user)
+
+            # Store bencoded info_dict
+            torrent.info = models.TorrentInfo(info_dict=torrent_data.bencoded_info_dict)
+            torrent.stats = models.Statistic()
+            torrent.has_torrent = True
+
+            # Fields with default value will be None before first commit, so set .flags
+            torrent.flags = 0
+
+            torrent.anonymous = True if torrent_flags[0] else False
+            torrent.hidden = True if torrent_flags[1] else False
+            torrent.remake = True if torrent_flags[2] else False
+            torrent.complete = True if torrent_flags[3] else False
+            # Copy trusted status from user if possible
+            torrent.trusted = (current_user.level >=
+                               models.UserLevelType.TRUSTED) if current_user else False
+
+            # Set category ids
+            torrent.main_category_id = cat_id
+            torrent.sub_category_id = sub_cat_id
+            # To simplify parsing the filelist, turn single-file torrent into a list
+            torrent_filelist = info_dict.get('files')
+
+            used_path_encoding = changed_to_utf8 and 'utf-8' or torrent_encoding
+
+            parsed_file_tree = dict()
+            if not torrent_filelist:
+                # If single-file, the root will be the file-tree (no directory)
+                file_tree_root = parsed_file_tree
+                torrent_filelist = [{'length': torrent_filesize, 'path': [info_dict['name']]}]
+            else:
+                # If multi-file, use the directory name as root for files
+                file_tree_root = parsed_file_tree.setdefault(
+                    info_dict['name'].decode(used_path_encoding), {})
+
+            # Parse file dicts into a tree
+            for file_dict in torrent_filelist:
+                # Decode path parts from utf8-bytes
+                path_parts = [path_part.decode(used_path_encoding)
+                              for path_part in file_dict['path']]
+
+                filename = path_parts.pop()
+                current_directory = file_tree_root
+
+                for directory in path_parts:
+                    current_directory = current_directory.setdefault(directory, {})
+
+                current_directory[filename] = file_dict['length']
+
+            parsed_file_tree = utils.sorted_pathdict(parsed_file_tree)
+
+            json_bytes = json.dumps(parsed_file_tree, separators=(',', ':')).encode('utf8')
+            torrent.filelist = models.TorrentFilelist(filelist_blob=json_bytes)
+
+            db.session.add(torrent)
+            db.session.flush()
+
+            # Store the users trackers
+            trackers = OrderedSet()
+            announce = torrent_data.torrent_dict.get('announce', b'').decode('ascii')
+            if announce:
+                trackers.add(announce)
+
+            # List of lists with single item
+            announce_list = torrent_data.torrent_dict.get('announce-list', [])
+            for announce in announce_list:
+                trackers.add(announce[0].decode('ascii'))
+
+            # Remove our trackers, maybe? TODO ?
+
+            # Search for/Add trackers in DB
+            db_trackers = OrderedSet()
+            for announce in trackers:
+                tracker = models.Trackers.by_uri(announce)
+
+                # Insert new tracker if not found
+                if not tracker:
+                    tracker = models.Trackers(uri=announce)
+                    db.session.add(tracker)
+
+                db_trackers.add(tracker)
+
+            db.session.flush()
+
+            # Store tracker refs in DB
+            for order, tracker in enumerate(db_trackers):
+                torrent_tracker = models.TorrentTrackers(torrent_id=torrent.id,
+                    tracker_id=tracker.id, order=order)
+                db.session.add(torrent_tracker)
+
+            db.session.commit()
+
+            if app.config.get('BACKUP_TORRENT_FOLDER'):
+                torrent_file.seek(0, 0)
+                torrent_path = os.path.join(app.config['BACKUP_TORRENT_FOLDER'], '{}.{}'.format(torrent.id, secure_filename(torrent_file.filename)))
+                torrent_file.save(torrent_path)
+            torrent_file.close()
+
+            #print('Success? {0}'.format(torrent.id))
+            return flask.make_response(flask.jsonify({"Success": "Request was processed {0}".format(torrent.id)}), 200)
+        except Exception as e:
+            print('Exception: {0}'.format(e))
+            return flask.make_response(flask.jsonify({"Error": "Incorrect JSON. Please see HELP page for examples."}), 400)
+    else:
+        return flask.make_response(flask.jsonify({"Error": "Bad request"}), 400)

nyaa/bencode.py

@@ -0,0 +1,163 @@
+from io import BytesIO
+
+
+def _pairwise(iterable):
+    """ Returns items from an iterable two at a time, ala
+        [0, 1, 2, 3, ...] -> [(0, 1), (2, 3), ...] """
+    iterable = iter(iterable)
+    return zip(iterable, iterable)
+
+
+__all__ = ['encode', 'decode', 'BencodeException', 'MalformedBencodeException']
+
+# https://wiki.theory.org/BitTorrentSpecification#Bencoding
+
+
+class BencodeException(Exception):
+    pass
+
+
+class MalformedBencodeException(BencodeException):
+    pass
+
+
+# bencode types
+_DIGITS = b'0123456789'
+_B_INT = b'i'
+_B_LIST = b'l'
+_B_DICT = b'd'
+_B_END = b'e'
+
+
+# Decoding of bencoded data
+
+def _bencode_decode(file_object, decode_keys_as_utf8=True):
+    """ Decodes a bencoded value, raising a MalformedBencodeException on errors.
+        decode_keys_as_utf8 controls decoding dict keys as utf8 (which they
+        almost always are) """
+    if isinstance(file_object, str):
+        file_object = file_object.encode('utf8')
+    if isinstance(file_object, bytes):
+        file_object = BytesIO(file_object)
+
+    def create_ex(msg):
+        return MalformedBencodeException(
+            '{0} at position {1} (0x{1:02X} hex)'.format(msg, file_object.tell()))
+
+    def _read_list():
+        """ Decodes values from stream until a None is returned ('e') """
+        items = []
+        while True:
+            value = _bencode_decode(file_object, decode_keys_as_utf8=decode_keys_as_utf8)
+            if value is None:
+                break
+            items.append(value)
+        return items
+
+    kind = file_object.read(1)
+
+    if kind == _B_INT:  # Integer
+        int_bytes = b''
+        while True:
+            c = file_object.read(1)
+            if not c:
+                raise create_ex('Unexpected end while reading an integer')
+            elif c == _B_END:
+                try:
+                    return int(int_bytes.decode('utf8'))
+                except Exception as e:
+                    raise create_ex('Unable to parse int')
+
+            # not a digit OR '-' in the middle of the int
+            if (c not in _DIGITS + b'-') or (c == b'-' and int_bytes):
+                raise create_ex('Unexpected input while reading an integer: ' + repr(c))
+            else:
+                int_bytes += c
+
+    elif kind == _B_LIST:  # List
+        return _read_list()
+
+    elif kind == _B_DICT:  # Dictionary
+        keys_and_values = _read_list()
+        if len(keys_and_values) % 2 != 0:
+            raise MalformedBencodeException('Uneven amount of key/value pairs')
+
+        # "Technically" the bencode dictionary keys are bytestrings,
+        # but real-world they're always(?) UTF-8.
+        decoded_dict = dict((decode_keys_as_utf8 and k.decode('utf8') or k, v)
+                            for k, v in _pairwise(keys_and_values))
+        return decoded_dict
+
+    # List/dict end, but make sure input is not just 'e'
+    elif kind == _B_END and file_object.tell() > 0:
+        return None
+
+    elif kind in _DIGITS:  # Bytestring
+        str_len_bytes = kind  # keep first digit
+        # Read string length until a ':'
+        while True:
+            c = file_object.read(1)
+            if c in _DIGITS:
+                str_len_bytes += c
+            elif c == b':':
+                break
+            else:
+                raise create_ex('Unexpected input while reading string length: ' + repr(c))
+        try:
+            str_len = int(str_len_bytes.decode())
+        except Exception as e:
+            raise create_ex('Unable to parse bytestring length')
+
+        bytestring = file_object.read(str_len)
+        if len(bytestring) != str_len:
+            raise create_ex('Read only {} bytes, {} wanted'.format(len(bytestring), str_len))
+
+        return bytestring
+    else:
+        raise create_ex('Unexpected data type ({})'.format(repr(kind)))
+
+
+# Bencoding
+
+def _bencode_int(value):
+    """ Encode an integer, eg 64 -> i64e """
+    return _B_INT + str(value).encode('utf8') + _B_END
+
+
+def _bencode_bytes(value):
+    """ Encode a bytestring (strings as UTF-8), eg 'hello' -> 5:hello """
+    if isinstance(value, str):
+        value = value.encode('utf8')
+    return str(len(value)).encode('utf8') + b':' + value
+
+
+def _bencode_list(value):
+    """ Encode a list, eg [64, "hello"] -> li64e5:helloe """
+    return _B_LIST + b''.join(_bencode(item) for item in value) + _B_END
+
+
+def _bencode_dict(value):
+    """ Encode a dict, which is keys and values interleaved as a list,
+        eg {"hello":123}-> d5:helloi123ee """
+    dict_keys = sorted(value.keys())  # Sort keys as per spec
+    return _B_DICT + b''.join(
+        _bencode_bytes(key) + _bencode(value[key]) for key in dict_keys) + _B_END
+
+
+def _bencode(value):
+    """ Bencode any supported value (int, bytes, str, list, dict) """
+    if isinstance(value, int):
+        return _bencode_int(value)
+    elif isinstance(value, (str, bytes)):
+        return _bencode_bytes(value)
+    elif isinstance(value, list):
+        return _bencode_list(value)
+    elif isinstance(value, dict):
+        return _bencode_dict(value)
+
+    raise BencodeException('Unsupported type ' + str(type(value)))
+
+
+# The functions call themselves
+encode = _bencode
+decode = _bencode_decode

nyaa/fix_paginate.py

@@ -0,0 +1,28 @@
+from flask_sqlalchemy import Pagination, BaseQuery
+from flask import abort
+
+def paginate_faste(self, page=1, per_page=50, max_page=None, step=5):
+    if page < 1:
+        abort(404)
+
+    if max_page and page > max_page:
+        abort(404)
+
+    items = self.limit(per_page).offset((page - 1) * per_page).all()
+
+    if not items and page != 1:
+        abort(404)
+
+    # No need to count if we're on the first page and there are fewer
+    # items than we expected.
+    if page == 1 and len(items) < per_page:
+        total = len(items)
+    else:
+        if max_page:
+            total = self.order_by(None).limit(per_page * min((page + step), max_page)).count()
+        else:
+            total = self.order_by(None).limit(per_page * (page + step)).count()
+
+    return Pagination(self, page, per_page, total, items)
+
+BaseQuery.paginate_faste = paginate_faste

nyaa/forms.py

@@ -0,0 +1,352 @@
+from nyaa import db, app
+from nyaa.models import User
+from nyaa import bencode, utils, models
+
+import os
+import re
+from flask_wtf import FlaskForm
+from flask_wtf.file import FileField, FileRequired
+from wtforms import TextField, PasswordField, BooleanField, TextAreaField, SelectField
+from wtforms.validators import Required, Optional, Email, Length, EqualTo, ValidationError, Regexp
+
+# For DisabledSelectField
+from wtforms.widgets import Select as SelectWidget
+from wtforms.widgets import html_params, HTMLString
+
+from flask_wtf.recaptcha import RecaptchaField
+
+
+class Unique(object):
+
+    """ validator that checks field uniqueness """
+
+    def __init__(self, model, field, message=None):
+        self.model = model
+        self.field = field
+        if not message:
+            message = 'This element already exists'
+        self.message = message
+
+    def __call__(self, form, field):
+        check = self.model.query.filter(self.field == field.data).first()
+        if check:
+            raise ValidationError(self.message)
+
+
+_username_validator = Regexp(
+    r'[a-zA-Z0-9_\-]+',
+    message='Your username must only consist of alphanumerics and _- (a-zA-Z0-9_-)')
+
+
+class LoginForm(FlaskForm):
+    username = TextField('Username or email address', [Required(), _username_validator])
+    password = PasswordField('Password', [Required()])
+
+
+class RegisterForm(FlaskForm):
+    username = TextField('Username', [
+        Required(),
+        Length(min=3, max=32),
+        _username_validator,
+        Unique(User, User.username, 'Username not availiable')
+    ])
+
+    email = TextField('Email address', [
+        Email(),
+        Required(),
+        Length(min=5, max=128),
+        Unique(User, User.email, 'Email already in use by another account')
+    ])
+
+    password = PasswordField('Password', [
+        Required(),
+        EqualTo('password_confirm', message='Passwords must match'),
+        Length(min=6, max=1024,
+               message='Password must be at least %(min)d characters long.')
+    ])
+
+    password_confirm = PasswordField('Password (confirm)')
+
+    if app.config['USE_RECAPTCHA']:
+        recaptcha = RecaptchaField()
+
+
+class ProfileForm(FlaskForm):
+    email = TextField('New email address', [
+        Email(),
+        Optional(),
+        Length(min=5, max=128),
+        Unique(User, User.email, 'Email is taken')
+    ])
+
+    current_password = PasswordField('Current password', [Optional()])
+
+    new_password = PasswordField('New password (confirm)', [
+        Optional(),
+        EqualTo('password_confirm', message='Passwords must match'),
+        Length(min=6, max=1024,
+               message='Password must be at least %(min)d characters long.')
+    ])
+
+    password_confirm = PasswordField('Repeat Password')
+
+
+# Classes for a SelectField that can be set to disable options (id, name, disabled)
+# TODO: Move to another file for cleaner look
+class DisabledSelectWidget(SelectWidget):
+    def __call__(self, field, **kwargs):
+        kwargs.setdefault('id', field.id)
+        if self.multiple:
+            kwargs['multiple'] = True
+        html = ['<select %s>' % html_params(name=field.name, **kwargs)]
+        for val, label, selected, disabled in field.iter_choices():
+            extra = disabled and {'disabled': ''} or {}
+            html.append(self.render_option(val, label, selected, **extra))
+        html.append('</select>')
+        return HTMLString(''.join(html))
+
+
+class DisabledSelectField(SelectField):
+    widget = DisabledSelectWidget()
+
+    def iter_choices(self):
+        for choice_tuple in self.choices:
+            value, label = choice_tuple[:2]
+            disabled = len(choice_tuple) == 3 and choice_tuple[2] or False
+            yield (value, label, self.coerce(value) == self.data, disabled)
+
+    def pre_validate(self, form):
+        for v in self.choices:
+            if self.data == v[0]:
+                break
+        else:
+            raise ValueError(self.gettext('Not a valid choice'))
+
+
+class EditForm(FlaskForm):
+    display_name = TextField('Display name', [
+        Length(min=3, max=255,
+               message='Torrent name must be at least %(min)d characters and %(max)d at most.')
+    ])
+
+    category = DisabledSelectField('Category')
+
+    def validate_category(form, field):
+        cat_match = re.match(r'^(\d+)_(\d+)$', field.data)
+        if not cat_match:
+            raise ValidationError('You must select a category')
+
+        main_cat_id = int(cat_match.group(1))
+        sub_cat_id = int(cat_match.group(2))
+
+        cat = models.SubCategory.by_category_ids(main_cat_id, sub_cat_id)
+
+        if not cat:
+            raise ValidationError('You must select a proper category')
+
+        field.parsed_data = cat
+
+    is_hidden = BooleanField('Hidden')
+    is_deleted = BooleanField('Deleted')
+    is_remake = BooleanField('Remake')
+    is_anonymous = BooleanField('Anonymous')
+    is_complete = BooleanField('Complete')
+
+    information = TextField('Information', [
+        Length(max=255, message='Information must be at most %(max)d characters long.')
+    ])
+    description = TextAreaField('Description (markdown supported)', [
+        Length(max=10 * 1024, message='Description must be at most %(max)d characters long.')
+    ])
+
+
+class UploadForm(FlaskForm):
+
+    class Meta:
+        csrf = False
+
+    torrent_file = FileField('Torrent file', [
+        FileRequired()
+    ])
+
+    display_name = TextField('Display name (optional)', [
+        Optional(),
+        Length(min=3, max=255,
+               message='Torrent name must be at least %(min)d characters long and %(max)d at most.')
+    ])
+
+    # category = SelectField('Category')
+    category = DisabledSelectField('Category')
+
+    def validate_category(form, field):
+        cat_match = re.match(r'^(\d+)_(\d+)$', field.data)
+        if not cat_match:
+            raise ValidationError('You must select a category')
+
+        main_cat_id = int(cat_match.group(1))
+        sub_cat_id = int(cat_match.group(2))
+
+        cat = models.SubCategory.by_category_ids(main_cat_id, sub_cat_id)
+
+        if not cat:
+            raise ValidationError('You must select a proper category')
+
+        field.parsed_data = cat
+
+    is_hidden = BooleanField('Hidden')
+    is_remake = BooleanField('Remake')
+    is_anonymous = BooleanField('Anonymous')
+    is_complete = BooleanField('Complete')
+
+    information = TextField('Information', [
+        Length(max=255, message='Information must be at most %(max)d characters long.')
+    ])
+    description = TextAreaField('Description (markdown supported)', [
+        Length(max=10 * 1024, message='Description must be at most %(max)d characters long.')
+    ])
+
+    def validate_torrent_file(form, field):
+        # Decode and ensure data is bencoded data
+        try:
+            torrent_dict = bencode.decode(field.data)
+            #field.data.close()
+        except (bencode.MalformedBencodeException, UnicodeError):
+            raise ValidationError('Malformed torrent file')
+
+        # Uncomment for debug print of the torrent
+        # _debug_print_torrent_metadata(torrent_dict)
+
+        try:
+            _validate_torrent_metadata(torrent_dict)
+        except AssertionError as e:
+            raise ValidationError('Malformed torrent metadata ({})'.format(e.args[0]))
+
+        try:
+            _validate_trackers(torrent_dict)
+        except AssertionError as e:
+            raise ValidationError('Malformed torrent trackers ({})'.format(e.args[0]))
+
+        if app.config.get('ENFORCE_MAIN_ANNOUNCE_URL'):
+            main_announce_url = app.config.get('MAIN_ANNOUNCE_URL')
+            if not main_announce_url:
+                raise Exception('Config MAIN_ANNOUNCE_URL not set!')
+
+            announce = torrent_dict.get('announce', b'').decode('utf-8')
+            if announce != main_announce_url:
+                raise ValidationError('Please set {} as the first tracker in the torrent'.format(main_announce_url))
+
+        # Note! bencode will sort dict keys, as per the spec
+        # This may result in a different hash if the uploaded torrent does not match the
+        # spec, but it's their own fault for using broken software! Right?
+        bencoded_info_dict = bencode.encode(torrent_dict['info'])
+        info_hash = utils.sha1_hash(bencoded_info_dict)
+
+        # Check if the info_hash exists already in the database
+        existing_torrent = models.Torrent.by_info_hash(info_hash)
+        if existing_torrent:
+            raise ValidationError('That torrent already exists (#{})'.format(existing_torrent.id))
+
+        # Torrent is legit, pass original filename and dict along
+        field.parsed_data = TorrentFileData(filename=os.path.basename(field.data.filename),
+                                            torrent_dict=torrent_dict,
+                                            info_hash=info_hash,
+                                            bencoded_info_dict=bencoded_info_dict)
+
+
+class TorrentFileData(object):
+    """Quick and dirty class to pass data from the validator"""
+
+    def __init__(self, **kwargs):
+        for k, v in kwargs.items():
+            setattr(self, k, v)
+
+# https://wiki.theory.org/BitTorrentSpecification#Metainfo_File_Structure
+
+def _validate_trackers(torrent_dict):
+    announce = torrent_dict.get('announce')
+    _validate_bytes(announce)
+
+    announce_list = torrent_dict.get('announce-list')
+    if announce_list is not None:
+        _validate_list(announce_list, 'announce-list')
+
+        for announce in announce_list:
+            _validate_list(announce, 'announce-list item')
+
+            _validate_bytes(announce[0], 'announce-list item item')
+
+
+def _validate_torrent_metadata(torrent_dict):
+    ''' Validates a torrent metadata dict, raising AssertionError on errors '''
+    assert isinstance(torrent_dict, dict), 'torrent metadata is not a dict'
+
+    info_dict = torrent_dict.get('info')
+    assert info_dict is not None, 'no info_dict in torrent'
+    assert isinstance(info_dict, dict), 'info is not a dict'
+
+    encoding_bytes = torrent_dict.get('encoding', b'utf-8')
+    encoding = _validate_bytes(encoding_bytes, 'encoding', 'utf-8').lower()
+
+    name = info_dict.get('name')
+    _validate_bytes(name, 'name', test_decode=encoding)
+
+    piece_length = info_dict.get('piece length')
+    _validate_number(piece_length, 'piece length', check_positive=True)
+
+    pieces = info_dict.get('pieces')
+    _validate_bytes(pieces, 'pieces')
+    assert len(pieces) % 20 == 0, 'pieces length is not a multiple of 20'
+
+    files = info_dict.get('files')
+    if files is not None:
+        _validate_list(files, 'filelist')
+
+        for file_dict in files:
+            file_length = file_dict.get('length')
+            _validate_number(file_length, 'file length', check_positive_or_zero=True)
+
+            path_list = file_dict.get('path')
+            _validate_list(path_list, 'path')
+            for path_part in path_list:
+                _validate_bytes(path_part, 'path part', test_decode=encoding)
+
+    else:
+        length = info_dict.get('length')
+        _validate_number(length, 'length', check_positive=True)
+
+
+def _validate_bytes(value, name='value', test_decode=None):
+    assert isinstance(value, bytes), name + ' is not bytes'
+    assert len(value) > 0, name + ' is empty'
+    if test_decode:
+        try:
+            return value.decode(test_decode)
+        except UnicodeError:
+            raise AssertionError(name + ' could not be decoded from ' + repr(test_decode))
+
+
+def _validate_number(value, name='value', check_positive=False, check_positive_or_zero=False):
+    assert isinstance(value, int), name + ' is not an int'
+    if check_positive_or_zero:
+        assert value >= 0, name + ' is less than 0'
+    elif check_positive:
+        assert value > 0, name + ' is not positive'
+
+
+def _validate_list(value, name='value', check_empty=False):
+    assert isinstance(value, list), name + ' is not a list'
+    if check_empty:
+        assert len(value) > 0, name + ' is empty'
+
+
+def _debug_print_torrent_metadata(torrent_dict):
+    from pprint import pprint
+
+    # Temporarily remove 'pieces' from infodict for clean debug prints
+    info_dict = torrent_dict.get('info', {})
+    orig_pieces = info_dict.get('pieces')
+
+    info_dict['pieces'] = '<piece data>'
+    pprint(torrent_dict)
+
+    info_dict['pieces'] = orig_pieces

nyaa/models.py

@@ -0,0 +1,332 @@
+from enum import Enum, IntEnum
+from datetime import datetime
+from nyaa import app, db
+from nyaa.torrents import create_magnet
+from sqlalchemy import func, ForeignKeyConstraint, Index
+from sqlalchemy_utils import ChoiceType, EmailType, PasswordType
+from werkzeug.security import generate_password_hash, check_password_hash
+from sqlalchemy_fulltext import FullText
+
+if app.config['USE_MYSQL']:
+    from sqlalchemy.dialects import mysql
+    BinaryType = mysql.BINARY
+    DescriptionTextType = mysql.TEXT
+    MediumBlobType = mysql.MEDIUMBLOB
+    COL_UTF8_GENERAL_CI = 'utf8_general_ci'
+    COL_UTF8MB4_BIN = 'utf8mb4_bin'
+    COL_ASCII_GENERAL_CI = 'ascii_general_ci'
+else:
+    BinaryType = db.Binary
+    DescriptionTextType = db.String
+    MediumBlobType = db.BLOB
+    COL_UTF8_GENERAL_CI = 'NOCASE'
+    COL_UTF8MB4_BIN = None
+    COL_ASCII_GENERAL_CI = 'NOCASE'
+
+
+class TorrentFlags(IntEnum):
+    NONE = 0
+    ANONYMOUS = 1
+    HIDDEN = 2
+    TRUSTED = 4
+    REMAKE = 8
+    COMPLETE = 16
+    DELETED = 32
+
+
+class Torrent(db.Model):
+    __tablename__ = app.config['TABLE_PREFIX'] + 'torrents'
+
+    id = db.Column(db.Integer, primary_key=True)
+    info_hash = db.Column(BinaryType(length=20), unique=True, nullable=False, index=True)
+    display_name = db.Column(
+        db.String(length=255, collation=COL_UTF8_GENERAL_CI), nullable=False, index=True)
+    torrent_name = db.Column(db.String(length=255), nullable=False)
+    information = db.Column(db.String(length=255), nullable=False)
+    description = db.Column(DescriptionTextType(collation=COL_UTF8MB4_BIN), nullable=False)
+
+    filesize = db.Column(db.BIGINT, default=0, nullable=False, index=True)
+    encoding = db.Column(db.String(length=32), nullable=False)
+    flags = db.Column(db.Integer, default=0, nullable=False, index=True)
+    uploader_id = db.Column(db.Integer, db.ForeignKey('users.id'), nullable=True)
+    has_torrent = db.Column(db.Boolean, nullable=False, default=False)
+
+    created_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow, nullable=False)
+    updated_time = db.Column(db.DateTime(timezone=False),
+                             default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
+
+    main_category_id = db.Column(db.Integer, db.ForeignKey(
+        app.config['TABLE_PREFIX'] + 'main_categories.id'), nullable=False)
+    sub_category_id = db.Column(db.Integer, nullable=False)
+    redirect = db.Column(db.Integer, db.ForeignKey(
+        app.config['TABLE_PREFIX'] + 'torrents.id'), nullable=True)
+
+    __table_args__ = (
+        Index('uploader_flag_idx', 'uploader_id', 'flags'),
+        ForeignKeyConstraint(
+            ['main_category_id', 'sub_category_id'],
+            [app.config['TABLE_PREFIX'] + 'sub_categories.main_category_id',
+                app.config['TABLE_PREFIX'] + 'sub_categories.id']
+        ), {}
+    )
+
+    user = db.relationship('User', uselist=False, back_populates='torrents')
+    main_category = db.relationship('MainCategory', uselist=False,
+                                    back_populates='torrents', lazy="joined")
+    sub_category = db.relationship('SubCategory', uselist=False, backref='torrents', lazy="joined",
+                                   primaryjoin="and_(SubCategory.id == foreign(Torrent.sub_category_id), "
+                                   "SubCategory.main_category_id == Torrent.main_category_id)")
+    info = db.relationship('TorrentInfo', uselist=False, back_populates='torrent')
+    filelist = db.relationship('TorrentFilelist', uselist=False, back_populates='torrent')
+    stats = db.relationship('Statistic', uselist=False, back_populates='torrent', lazy='joined')
+    trackers = db.relationship('TorrentTrackers', uselist=True, lazy='joined')
+
+    def __repr__(self):
+        return '<{0} #{1.id} \'{1.display_name}\' {1.filesize}b>'.format(type(self).__name__, self)
+
+    @property
+    def magnet_uri(self):
+        return create_magnet(self)
+
+    @property
+    def anonymous(self):
+        return self.flags & TorrentFlags.ANONYMOUS
+
+    @anonymous.setter
+    def anonymous(self, value):
+        self.flags = (self.flags & ~TorrentFlags.ANONYMOUS) | (value and TorrentFlags.ANONYMOUS)
+
+    @property
+    def hidden(self):
+        return self.flags & TorrentFlags.HIDDEN
+
+    @hidden.setter
+    def hidden(self, value):
+        self.flags = (self.flags & ~TorrentFlags.HIDDEN) | (value and TorrentFlags.HIDDEN)
+
+    @property
+    def deleted(self):
+        return self.flags & TorrentFlags.DELETED
+
+    @deleted.setter
+    def deleted(self, value):
+        self.flags = (self.flags & ~TorrentFlags.DELETED) | (value and TorrentFlags.DELETED)
+
+    @property
+    def trusted(self):
+        return self.flags & TorrentFlags.TRUSTED
+
+    @trusted.setter
+    def trusted(self, value):
+        self.flags = (self.flags & ~TorrentFlags.TRUSTED) | (value and TorrentFlags.TRUSTED)
+
+    @property
+    def remake(self):
+        return self.flags & TorrentFlags.REMAKE
+
+    @remake.setter
+    def remake(self, value):
+        self.flags = (self.flags & ~TorrentFlags.REMAKE) | (value and TorrentFlags.REMAKE)
+
+    @property
+    def complete(self):
+        return self.flags & TorrentFlags.COMPLETE
+
+    @complete.setter
+    def complete(self, value):
+        self.flags = (self.flags & ~TorrentFlags.COMPLETE) | (value and TorrentFlags.COMPLETE)
+
+    @classmethod
+    def by_id(cls, id):
+        return cls.query.get(id)
+
+    @classmethod
+    def by_info_hash(cls, info_hash):
+        return cls.query.filter_by(info_hash=info_hash).first()
+
+
+class TorrentNameSearch(FullText, Torrent):
+    __fulltext_columns__ = ('display_name',)
+
+
+class TorrentFilelist(db.Model):
+    __tablename__ = app.config['TABLE_PREFIX'] + 'torrents_filelist'
+    __table_args__ = {'mysql_row_format': 'COMPRESSED'}
+
+    torrent_id = db.Column(db.Integer, db.ForeignKey(
+        app.config['TABLE_PREFIX'] + 'torrents.id', ondelete="CASCADE"), primary_key=True)
+    filelist_blob = db.Column(MediumBlobType, nullable=True)
+
+    torrent = db.relationship('Torrent', uselist=False, back_populates='filelist')
+
+
+class TorrentInfo(db.Model):
+    __tablename__ = app.config['TABLE_PREFIX'] + 'torrents_info'
+    __table_args__ = {'mysql_row_format': 'COMPRESSED'}
+
+    torrent_id = db.Column(db.Integer, db.ForeignKey(
+        app.config['TABLE_PREFIX'] + 'torrents.id', ondelete="CASCADE"), primary_key=True)
+    info_dict = db.Column(MediumBlobType, nullable=True)
+
+    torrent = db.relationship('Torrent', uselist=False, back_populates='info')
+
+
+class Statistic(db.Model):
+    __tablename__ = app.config['TABLE_PREFIX'] + 'statistics'
+
+    torrent_id = db.Column(db.Integer, db.ForeignKey(
+        app.config['TABLE_PREFIX'] + 'torrents.id', ondelete="CASCADE"), primary_key=True)
+
+    seed_count = db.Column(db.Integer, default=0, nullable=False, index=True)
+    leech_count = db.Column(db.Integer, default=0, nullable=False, index=True)
+    download_count = db.Column(db.Integer, default=0, nullable=False, index=True)
+    last_updated = db.Column(db.DateTime(timezone=False))
+
+    torrent = db.relationship('Torrent', uselist=False, back_populates='stats')
+
+
+class Trackers(db.Model):
+    __tablename__ = 'trackers'
+
+    id = db.Column(db.Integer, primary_key=True)
+    uri = db.Column(db.String(length=255, collation=COL_UTF8_GENERAL_CI), nullable=False, unique=True)
+    disabled = db.Column(db.Boolean, nullable=False, default=False)
+
+    @classmethod
+    def by_uri(cls, uri):
+        return cls.query.filter_by(uri=uri).first()
+
+
+class TorrentTrackers(db.Model):
+    __tablename__ = app.config['TABLE_PREFIX'] + 'torrent_trackers'
+
+    torrent_id = db.Column(db.Integer, db.ForeignKey(app.config['TABLE_PREFIX'] + 'torrents.id', ondelete="CASCADE"), primary_key=True)
+    tracker_id = db.Column(db.Integer, db.ForeignKey('trackers.id', ondelete="CASCADE"), primary_key=True)
+    order = db.Column(db.Integer, nullable=False, index=True)
+
+    tracker = db.relationship('Trackers', uselist=False, lazy='joined')
+
+    @classmethod
+    def by_torrent_id(cls, torrent_id):
+        return cls.query.filter_by(torrent_id=torrent_id).order_by(cls.order.desc())
+
+
+class MainCategory(db.Model):
+    __tablename__ = app.config['TABLE_PREFIX'] + 'main_categories'
+
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String(length=64), nullable=False)
+
+    sub_categories = db.relationship('SubCategory', back_populates='main_category')
+    torrents = db.relationship('Torrent', back_populates='main_category')
+
+    def get_category_ids(self):
+        return (self.id, 0)
+
+    @property
+    def id_as_string(self):
+        return '_'.join(str(x) for x in self.get_category_ids())
+
+    @classmethod
+    def by_id(cls, id):
+        return cls.query.get(id)
+
+
+class SubCategory(db.Model):
+    __tablename__ = app.config['TABLE_PREFIX'] + 'sub_categories'
+
+    id = db.Column(db.Integer, primary_key=True)
+    main_category_id = db.Column(db.Integer, db.ForeignKey(
+        app.config['TABLE_PREFIX'] + 'main_categories.id'), primary_key=True)
+    name = db.Column(db.String(length=64), nullable=False)
+
+    main_category = db.relationship('MainCategory', uselist=False, back_populates='sub_categories')
+#    torrents = db.relationship('Torrent', back_populates='sub_category'),
+#        primaryjoin="and_(Torrent.sub_category_id == foreign(SubCategory.id), "