1
0
Fork 0
mirror of https://gitlab.com/SIGBUS/nyaa.git synced 2024-10-07 04:01:53 +00:00

Initial commit.

This commit is contained in:
nyaadev 2017-05-12 20:51:49 +02:00
commit 00d65e312c
89 changed files with 5733 additions and 0 deletions

48
README.md Normal file
View file

@ -0,0 +1,48 @@
# NyaaV2
## Setup:
- Create your virtualenv, for example with `pyvenv venv`
- Enter your virtualenv with `source venv/bin/activate`
- Install dependencies with `pip install -r requirements.txt`
- Run `python db_create.py` to create the database
- Start the dev server with `python run.py`
## Updated Setup (python 3.6.1):
- Install dependencies https://github.com/pyenv/pyenv/wiki/Common-build-problems
- Install `pyenv` https://github.com/pyenv/pyenv/blob/master/README.md#installation
- Install `pyenv-virtualenv` https://github.com/pyenv/pyenv-virtualenv/blob/master/README.md
- `pyenv install 3.6.1`
- `pyenv virtualenv 3.6.1 nyaa`
- `pyenv activate nyaa`
- Install dependencies with `pip install -r requirements.txt`
- Copy `config.example.py` into `config.py`
- Change TALBE_PREFIX to `nyaa_` or `sukebei_` depending on the site
## Setting up MySQL/MariaDB database for advanced functionality
- Enable `USE_MYSQL` flag in config.py
- Install latest mariadb by following instructions here https://downloads.mariadb.org/mariadb/repositories/
- Tested versions: `mysql Ver 15.1 Distrib 10.0.30-MariaDB, for debian-linux-gnu (x86_64) using readline 5.2`
- Run the following commands logged in as your root db user:
- `CREATE USER 'test'@'localhost' IDENTIFIED BY 'test123';`
- `GRANT ALL PRIVILEGES ON * . * TO 'test'@'localhost';`
- `FLUSH PRIVILEGES;`
- `CREATE DATABASE nyaav2 DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;`
- To setup and import nyaa_maria_vx.sql:
- `mysql -u <user> -p nyaav2`
- `DROP DATABASE nyaav2;`
- `CREATE DATABASE nyaav2 DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;`
- `SOURCE ~/path/to/database/nyaa_maria_vx.sql`
## Finishing up
- Run `python db_create.py` to create the database
- Load the .sql file
- `mysql -u user -p nyaav2`
- `SOURCE cocks.sql`
- Remember to change the default user password to an empty string to disable logging in
- Start the dev server with `python run.py`
- Deactivate `source deactivate`
## Code Quality:
- Remember to follow PEP8 style guidelines and run `./lint.sh` before committing.

15
WSGI.py Normal file
View file

@ -0,0 +1,15 @@
#!/usr/bin/python3
# -*- coding: utf-8 -*-
import gevent.monkey
gevent.monkey.patch_all()
from nyaa import app
if app.config["DEBUG"]:
from werkzeug.debug import DebuggedApplication
app.wsgi_app = DebuggedApplication(app.wsgi_app, True)
if __name__ == '__main__':
import gevent.pywsgi
gevent_server = gevent.pywsgi.WSGIServer(("localhost", 5000), app.wsgi_app)
gevent_server.serve_forever()

55
api_uploader.py Normal file
View file

@ -0,0 +1,55 @@
# api_uploader.py
# Uploads a single file
# I will create another script for batch uploading
import json
import requests
url = "http://127.0.0.1:5500/api/upload"
# Required for Auth
username = ""
password = ""
# Required
torrent_name = ""
# Required
main_cat = ""
# Required
sub_cat = ""
# May be blank
information = ""
# May be blank
description = ""
# flags = [Hidden, Remake, Complete, Anonymous]
# 0 for NOT SET / 1 for SET
# Required
flags = [0, 0, 0, 0]
metadata={
"username": username,
"password": password,
"display_name": torrent_name,
"main_cat": main_cat,
"sub_cat": sub_cat,
"information": information,
"description": description,
"flags": flags
}
# Required
file_name = ""
files = {
'json': (json.dumps(metadata)),
'torrent': ('{0}'.format(file_name), open(file_name, 'rb'), 'application/octet-stream')}
response = requests.post(url, files=files)
json_response = response.json()
print(json_response)

2
batch_upload_torrent.sh Executable file
View file

@ -0,0 +1,2 @@
up_t() { curl -F "category=1_2" -F "torrent_file=@$1" 'http://localhost:5500/upload'; }
for x in test_torrent_batch/*; do up_t "$x"; done

51
config.example.py Normal file
View file

@ -0,0 +1,51 @@
import os
DEBUG = True
USE_RECAPTCHA = False
USE_EMAIL_VERIFICATION = False
USE_MYSQL = True
# Enable this once stat integration is done
ENABLE_SHOW_STATS = False
BASE_DIR = os.path.abspath(os.path.dirname(__file__))
if USE_MYSQL:
SQLALCHEMY_DATABASE_URI = ('mysql://test:test123@localhost/nyaav2')
else:
SQLALCHEMY_DATABASE_URI = (
'sqlite:///' + os.path.join(BASE_DIR, 'test.db') + '?check_same_thread=False')
CSRF_SESSION_KEY = '***'
SECRET_KEY = '***'
# Prefix for running multiple sites, user table will not be prefixed.
# For sukebei, change 'nyaa_' to 'sukebei_'
TABLE_PREFIX = 'nyaa_'
# for recaptcha and email verification:
# keys for localhost. Change as appropriate when actual domain is registered.
RECAPTCHA_PUBLIC_KEY = '***'
RECAPTCHA_PRIVATE_KEY = '***'
SMTP_SERVER = '***'
SMTP_PORT = 587
MAIL_FROM_ADDRESS = '***'
SMTP_USERNAME = '***'
SMTP_PASSWORD = '***'
RESULTS_PER_PAGE = 75
# What the site identifies itself as.
SITE_NAME = 'Nyaa'
# The maximum number of files a torrent can contain
# until the site says "Too many files to display."
MAX_FILES_VIEW = 1000
# """
# Setting to make sure main announce url is present in torrent
# """
ENFORCE_MAIN_ANNOUNCE_URL = False
MAIN_ANNOUNCE_URL = ''
BACKUP_TORRENT_FOLDER = 'torrents'

40
db_create.py Normal file
View file

@ -0,0 +1,40 @@
import sys
from nyaa import app, db, models
# Create tables
db.create_all()
# Insert categories
if app.config['TABLE_PREFIX'] == 'nyaa_':
CATEGORIES = [
('Anime', ['Anime Music Video', 'English-translated', 'Non-English-translated', 'Raw']),
('Audio', ['Lossless', 'Lossy']),
('Literature', ['English-translated', 'Non-English-translated', 'Raw']),
('Live Action', ['English-translated', 'Idol/Promotional Video', 'Non-English-translated', 'Raw']),
('Pictures', ['Graphics', 'Photos']),
('Software', ['Applications', 'Games']),
]
elif app.config['TABLE_PREFIX'] == 'sukebei_':
CATEGORIES = [
('Art', ['Anime', 'Doujinshi', 'Games', 'Manga', 'Pictures']),
('Real Life', ['Photobooks / Pictures', 'Videos']),
]
else:
CATEGORIES = []
for main_cat_name, sub_cat_names in CATEGORIES:
main_cat = models.MainCategory(name=main_cat_name)
for i, sub_cat_name in enumerate(sub_cat_names):
# Composite keys can't autoincrement, set sub_cat id manually (1-index)
sub_cat = models.SubCategory(id=i+1, name=sub_cat_name, main_category=main_cat)
db.session.add(main_cat)
db.session.commit()
# Create fulltext index
if app.config['USE_MYSQL']:
db.engine.execute('ALTER TABLE ' + app.config['TABLE_PREFIX'] + 'torrents ADD FULLTEXT KEY (display_name)')

6
lint.sh Executable file
View file

@ -0,0 +1,6 @@
autopep8 nyaa/ \
--recursive \
--in-place \
--pep8-passes 2000 \
--max-line-length 100 \
--verbose

6
my.cnf Normal file
View file

@ -0,0 +1,6 @@
[mysqld]
innodb_ft_min_token_size=2
ft_min_word_len=2
innodb_ft_cache_size = 80000000
innodb_ft_total_cache_size = 1600000000
max_allowed_packet = 100M

53
nyaa/__init__.py Normal file
View file

@ -0,0 +1,53 @@
import os
import logging
import flask
from flask_sqlalchemy import SQLAlchemy
from flask_assets import Environment, Bundle
from flask_debugtoolbar import DebugToolbarExtension
from nyaa import fix_paginate
app = flask.Flask(__name__)
app.config.from_object('config')
# Database
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
# Debugging
if app.config['DEBUG']:
app.config['DEBUG_TB_INTERCEPT_REDIRECTS'] = False
toolbar = DebugToolbarExtension(app)
app.logger.setLevel(logging.DEBUG)
else:
app.logger.setLevel(logging.WARNING)
# Logging
if 'LOG_FILE' in app.config:
from logging.handlers import RotatingFileHandler
app.log_handler = RotatingFileHandler(
app.config['LOG_FILE'], maxBytes=10000, backupCount=1)
app.logger.addHandler(app.log_handler)
# Log errors and display a message to the user in production mdode
if not app.config['DEBUG']:
@app.errorhandler(500)
def internal_error(exception):
app.logger.error(exception)
flask.flash(flask.Markup(
'<strong>An error occured!</strong> Debugging information has been logged.'), 'danger')
return flask.redirect('/')
# Enable the jinja2 do extension.
app.jinja_env.add_extension('jinja2.ext.do')
app.jinja_env.lstrip_blocks = True
app.jinja_env.trim_blocks = True
db = SQLAlchemy(app)
assets = Environment(app)
# css = Bundle('style.scss', filters='libsass',
# output='style.css', depends='**/*.scss')
# assets.register('style_all', css)
from nyaa import routes

317
nyaa/api_handler.py Normal file
View file

@ -0,0 +1,317 @@
import flask
from nyaa import app, db
from nyaa import models, forms
from nyaa import bencode, utils
from nyaa import torrents
import json
import os.path
from orderedset import OrderedSet
from werkzeug import secure_filename
DEBUG_API = False
#################################### API ROUTES ####################################
CATEGORIES = [
('Anime', ['Anime Music Video', 'English-translated', 'Non-English-translated', 'Raw']),
('Audio', ['Lossless', 'Lossy']),
('Literature', ['English-translated', 'Non-English-translated', 'Raw']),
('Live Action', ['English-translated',
'Idol/Promotional Video', 'Non-English-translated', 'Raw']),
('Pictures', ['Graphics', 'Photos']),
('Software', ['Applications', 'Games']),
]
def validate_main_sub_cat(main_cat_name, sub_cat_name):
for main_cat in models.MainCategory.query.order_by(models.MainCategory.id):
if main_cat_name == main_cat.name:
for sub_cat in main_cat.sub_categories:
if sub_cat_name == sub_cat.name:
cat_id = main_cat.id_as_string
sub_cat_id = sub_cat.id_as_string
cat_sub_cat = sub_cat_id.split('_')
#print('cat: {0} sub_cat: {1}'.format(cat_sub_cat[0], cat_sub_cat[1]))
return True, cat_sub_cat[0], cat_sub_cat[1]
return False, 0, 0
def _replace_utf8_values(dict_or_list):
''' Will replace 'property' with 'property.utf-8' and remove latter if it exists.
Thanks, bitcomet! :/ '''
did_change = False
if isinstance(dict_or_list, dict):
for key in [key for key in dict_or_list.keys() if key.endswith('.utf-8')]:
dict_or_list[key.replace('.utf-8', '')] = dict_or_list.pop(key)
did_change = True
for value in dict_or_list.values():
did_change = _replace_utf8_values(value) or did_change
elif isinstance(dict_or_list, list):
for item in dict_or_list:
did_change = _replace_utf8_values(item) or did_change
return did_change
def validate_torrent_flags(torrent_flags):
_torrent_flags = ['hidden', 'remake', 'complete', 'anonymous']
if len(torrent_flags) != 4:
return False
for flag in torrent_flags:
if int(flag) not in [0, 1]:
return False
return True
# It might be good to factor this out of forms UploadForm because the same code is
# used in both files.
def validate_torrent_file(torrent_file_name, torrent_file):
# Decode and ensure data is bencoded data
try:
torrent_dict = bencode.decode(torrent_file)
except (bencode.MalformedBencodeException, UnicodeError):
return False, 'Malformed torrent file'
# Uncomment for debug print of the torrent
# forms._debug_print_torrent_metadata(torrent_dict)
try:
forms._validate_torrent_metadata(torrent_dict)
except AssertionError as e:
return False, 'Malformed torrent metadata ({})'.format(e.args[0])
# Note! bencode will sort dict keys, as per the spec
# This may result in a different hash if the uploaded torrent does not match the
# spec, but it's their own fault for using broken software! Right?
bencoded_info_dict = bencode.encode(torrent_dict['info'])
info_hash = utils.sha1_hash(bencoded_info_dict)
# Check if the info_hash exists already in the database
existing_torrent = models.Torrent.by_info_hash(info_hash)
if existing_torrent:
return False, 'That torrent already exists (#{})'.format(existing_torrent.id)
# Torrent is legit, pass original filename and dict along
return True, forms.TorrentFileData(filename=os.path.basename(torrent_file_name),
torrent_dict=torrent_dict,
info_hash=info_hash,
bencoded_info_dict=bencoded_info_dict)
def api_upload(upload_request):
if upload_request.method == 'POST':
j = None
torrent_file = None
try:
if 'json' in upload_request.files:
f = upload_request.files['json']
j = json.loads(f.read().decode('utf-8'))
if DEBUG_API:
print(json.dumps(j, indent=4))
_json_keys = ['username', 'password',
'display_name', 'main_cat', 'sub_cat', 'flags'] # 'information' and 'description' are not required
# Check that required fields are present
for _k in _json_keys:
if _k not in j.keys():
return flask.make_response(flask.jsonify({"Error": "Missing JSON field: {0}.".format(_k)}), 400)
# Check that no extra fields are present
for k in j.keys():
if k not in ['username', 'password',
'display_name', 'main_cat', 'sub_cat', 'information', 'description', 'flags']:
return flask.make_response(flask.jsonify({"Error": "Incorrect JSON field(s)."}), 400)
else:
return flask.make_response(flask.jsonify({"Error": "No metadata."}), 400)
if 'torrent' in upload_request.files:
f = upload_request.files['torrent']
if DEBUG_API:
print(f.filename)
torrent_file = f
# print(f.read())
else:
return flask.make_response(flask.jsonify({"Error": "No torrent file."}), 400)
# 'username' and 'password' must have been provided as they are part of j.keys()
username = j['username']
password = j['password']
# Validate that the provided username and password belong to a valid user
user = models.User.by_username(username)
if not user:
user = models.User.by_email(username)
if not user or password != user.password_hash or user.status == models.UserStatusType.INACTIVE:
return flask.make_response(flask.jsonify({"Error": "Incorrect username or password"}), 403)
current_user = user
display_name = j['display_name']
if (len(display_name) < 3) or (len(display_name) > 1024):
return flask.make_response(flask.jsonify({"Error": "Torrent name must be between 3 and 1024 characters."}), 400)
main_cat_name = j['main_cat']
sub_cat_name = j['sub_cat']
cat_subcat_status, cat_id, sub_cat_id = validate_main_sub_cat(
main_cat_name, sub_cat_name)
if not cat_subcat_status:
return flask.make_response(flask.jsonify({"Error": "Incorrect Category / Sub-Category."}), 400)
# TODO Sanitize information
information = None
try:
information = j['information']
if len(information) > 255:
return flask.make_response(flask.jsonify({"Error": "Information is limited to 255 characters."}), 400)
except Exception as e:
information = ''
# TODO Sanitize description
description = None
try:
description = j['description']
if len(description) > (10 * 1024):
return flask.make_response(flask.jsonify({"Error": "Description is limited to {0} characters.".format(10 * 1024)}), 403)
except Exception as e:
description = ''
v_flags = validate_torrent_flags(j['flags'])
if v_flags:
torrent_flags = j['flags']
else:
return flask.make_response(flask.jsonify({"Error": "Incorrect torrent flags."}), 400)
torrent_status, torrent_data = validate_torrent_file(
torrent_file.filename, torrent_file.read()) # Needs validation
if not torrent_status:
return flask.make_response(flask.jsonify({"Error": "Invalid or Duplicate torrent file."}), 400)
# The torrent has been validated and is safe to access with ['foo'] etc - all relevant
# keys and values have been checked for (see UploadForm in forms.py for details)
info_dict = torrent_data.torrent_dict['info']
changed_to_utf8 = _replace_utf8_values(torrent_data.torrent_dict)
torrent_filesize = info_dict.get('length') or sum(
f['length'] for f in info_dict.get('files'))
# In case no encoding, assume UTF-8.
torrent_encoding = torrent_data.torrent_dict.get('encoding', b'utf-8').decode('utf-8')
torrent = models.Torrent(info_hash=torrent_data.info_hash,
display_name=display_name,
torrent_name=torrent_data.filename,
information=information,
description=description,
encoding=torrent_encoding,
filesize=torrent_filesize,
user=current_user)
# Store bencoded info_dict
torrent.info = models.TorrentInfo(info_dict=torrent_data.bencoded_info_dict)
torrent.stats = models.Statistic()
torrent.has_torrent = True
# Fields with default value will be None before first commit, so set .flags
torrent.flags = 0
torrent.anonymous = True if torrent_flags[0] else False
torrent.hidden = True if torrent_flags[1] else False
torrent.remake = True if torrent_flags[2] else False
torrent.complete = True if torrent_flags[3] else False
# Copy trusted status from user if possible
torrent.trusted = (current_user.level >=
models.UserLevelType.TRUSTED) if current_user else False
# Set category ids
torrent.main_category_id = cat_id
torrent.sub_category_id = sub_cat_id
# To simplify parsing the filelist, turn single-file torrent into a list
torrent_filelist = info_dict.get('files')
used_path_encoding = changed_to_utf8 and 'utf-8' or torrent_encoding
parsed_file_tree = dict()
if not torrent_filelist:
# If single-file, the root will be the file-tree (no directory)
file_tree_root = parsed_file_tree
torrent_filelist = [{'length': torrent_filesize, 'path': [info_dict['name']]}]
else:
# If multi-file, use the directory name as root for files
file_tree_root = parsed_file_tree.setdefault(
info_dict['name'].decode(used_path_encoding), {})
# Parse file dicts into a tree
for file_dict in torrent_filelist:
# Decode path parts from utf8-bytes
path_parts = [path_part.decode(used_path_encoding)
for path_part in file_dict['path']]
filename = path_parts.pop()
current_directory = file_tree_root
for directory in path_parts:
current_directory = current_directory.setdefault(directory, {})
current_directory[filename] = file_dict['length']
parsed_file_tree = utils.sorted_pathdict(parsed_file_tree)
json_bytes = json.dumps(parsed_file_tree, separators=(',', ':')).encode('utf8')
torrent.filelist = models.TorrentFilelist(filelist_blob=json_bytes)
db.session.add(torrent)
db.session.flush()
# Store the users trackers
trackers = OrderedSet()
announce = torrent_data.torrent_dict.get('announce', b'').decode('ascii')
if announce:
trackers.add(announce)
# List of lists with single item
announce_list = torrent_data.torrent_dict.get('announce-list', [])
for announce in announce_list:
trackers.add(announce[0].decode('ascii'))
# Remove our trackers, maybe? TODO ?
# Search for/Add trackers in DB
db_trackers = OrderedSet()
for announce in trackers:
tracker = models.Trackers.by_uri(announce)
# Insert new tracker if not found
if not tracker:
tracker = models.Trackers(uri=announce)
db.session.add(tracker)
db_trackers.add(tracker)
db.session.flush()
# Store tracker refs in DB
for order, tracker in enumerate(db_trackers):
torrent_tracker = models.TorrentTrackers(torrent_id=torrent.id,
tracker_id=tracker.id, order=order)
db.session.add(torrent_tracker)
db.session.commit()
if app.config.get('BACKUP_TORRENT_FOLDER'):
torrent_file.seek(0, 0)
torrent_path = os.path.join(app.config['BACKUP_TORRENT_FOLDER'], '{}.{}'.format(torrent.id, secure_filename(torrent_file.filename)))
torrent_file.save(torrent_path)
torrent_file.close()
#print('Success? {0}'.format(torrent.id))
return flask.make_response(flask.jsonify({"Success": "Request was processed {0}".format(torrent.id)}), 200)
except Exception as e:
print('Exception: {0}'.format(e))
return flask.make_response(flask.jsonify({"Error": "Incorrect JSON. Please see HELP page for examples."}), 400)
else:
return flask.make_response(flask.jsonify({"Error": "Bad request"}), 400)

163
nyaa/bencode.py Normal file
View file

@ -0,0 +1,163 @@
from io import BytesIO
def _pairwise(iterable):
""" Returns items from an iterable two at a time, ala
[0, 1, 2, 3, ...] -> [(0, 1), (2, 3), ...] """
iterable = iter(iterable)
return zip(iterable, iterable)
__all__ = ['encode', 'decode', 'BencodeException', 'MalformedBencodeException']
# https://wiki.theory.org/BitTorrentSpecification#Bencoding
class BencodeException(Exception):
pass
class MalformedBencodeException(BencodeException):
pass
# bencode types
_DIGITS = b'0123456789'
_B_INT = b'i'
_B_LIST = b'l'
_B_DICT = b'd'
_B_END = b'e'
# Decoding of bencoded data
def _bencode_decode(file_object, decode_keys_as_utf8=True):
""" Decodes a bencoded value, raising a MalformedBencodeException on errors.
decode_keys_as_utf8 controls decoding dict keys as utf8 (which they
almost always are) """
if isinstance(file_object, str):
file_object = file_object.encode('utf8')
if isinstance(file_object, bytes):
file_object = BytesIO(file_object)
def create_ex(msg):
return MalformedBencodeException(
'{0} at position {1} (0x{1:02X} hex)'.format(msg, file_object.tell()))
def _read_list():
""" Decodes values from stream until a None is returned ('e') """
items = []
while True:
value = _bencode_decode(file_object, decode_keys_as_utf8=decode_keys_as_utf8)
if value is None:
break
items.append(value)
return items
kind = file_object.read(1)
if kind == _B_INT: # Integer
int_bytes = b''
while True:
c = file_object.read(1)
if not c:
raise create_ex('Unexpected end while reading an integer')
elif c == _B_END:
try:
return int(int_bytes.decode('utf8'))
except Exception as e:
raise create_ex('Unable to parse int')
# not a digit OR '-' in the middle of the int
if (c not in _DIGITS + b'-') or (c == b'-' and int_bytes):
raise create_ex('Unexpected input while reading an integer: ' + repr(c))
else:
int_bytes += c
elif kind == _B_LIST: # List
return _read_list()
elif kind == _B_DICT: # Dictionary
keys_and_values = _read_list()
if len(keys_and_values) % 2 != 0:
raise MalformedBencodeException('Uneven amount of key/value pairs')
# "Technically" the bencode dictionary keys are bytestrings,
# but real-world they're always(?) UTF-8.
decoded_dict = dict((decode_keys_as_utf8 and k.decode('utf8') or k, v)
for k, v in _pairwise(keys_and_values))
return decoded_dict
# List/dict end, but make sure input is not just 'e'
elif kind == _B_END and file_object.tell() > 0:
return None
elif kind in _DIGITS: # Bytestring
str_len_bytes = kind # keep first digit
# Read string length until a ':'
while True:
c = file_object.read(1)
if c in _DIGITS:
str_len_bytes += c
elif c == b':':
break
else:
raise create_ex('Unexpected input while reading string length: ' + repr(c))
try:
str_len = int(str_len_bytes.decode())
except Exception as e:
raise create_ex('Unable to parse bytestring length')
bytestring = file_object.read(str_len)
if len(bytestring) != str_len:
raise create_ex('Read only {} bytes, {} wanted'.format(len(bytestring), str_len))
return bytestring
else:
raise create_ex('Unexpected data type ({})'.format(repr(kind)))
# Bencoding
def _bencode_int(value):
""" Encode an integer, eg 64 -> i64e """
return _B_INT + str(value).encode('utf8') + _B_END
def _bencode_bytes(value):
""" Encode a bytestring (strings as UTF-8), eg 'hello' -> 5:hello """
if isinstance(value, str):
value = value.encode('utf8')
return str(len(value)).encode('utf8') + b':' + value
def _bencode_list(value):
""" Encode a list, eg [64, "hello"] -> li64e5:helloe """
return _B_LIST + b''.join(_bencode(item) for item in value) + _B_END
def _bencode_dict(value):
""" Encode a dict, which is keys and values interleaved as a list,
eg {"hello":123}-> d5:helloi123ee """
dict_keys = sorted(value.keys()) # Sort keys as per spec
return _B_DICT + b''.join(
_bencode_bytes(key) + _bencode(value[key]) for key in dict_keys) + _B_END
def _bencode(value):
""" Bencode any supported value (int, bytes, str, list, dict) """
if isinstance(value, int):
return _bencode_int(value)
elif isinstance(value, (str, bytes)):
return _bencode_bytes(value)
elif isinstance(value, list):
return _bencode_list(value)
elif isinstance(value, dict):
return _bencode_dict(value)
raise BencodeException('Unsupported type ' + str(type(value)))
# The functions call themselves
encode = _bencode
decode = _bencode_decode

28
nyaa/fix_paginate.py Normal file
View file

@ -0,0 +1,28 @@
from flask_sqlalchemy import Pagination, BaseQuery
from flask import abort
def paginate_faste(self, page=1, per_page=50, max_page=None, step=5):
if page < 1:
abort(404)
if max_page and page > max_page:
abort(404)
items = self.limit(per_page).offset((page - 1) * per_page).all()
if not items and page != 1:
abort(404)
# No need to count if we're on the first page and there are fewer
# items than we expected.
if page == 1 and len(items) < per_page:
total = len(items)
else:
if max_page:
total = self.order_by(None).limit(per_page * min((page + step), max_page)).count()
else:
total = self.order_by(None).limit(per_page * (page + step)).count()
return Pagination(self, page, per_page, total, items)
BaseQuery.paginate_faste = paginate_faste

352
nyaa/forms.py Normal file
View file

@ -0,0 +1,352 @@
from nyaa import db, app
from nyaa.models import User
from nyaa import bencode, utils, models
import os
import re
from flask_wtf import FlaskForm
from flask_wtf.file import FileField, FileRequired
from wtforms import TextField, PasswordField, BooleanField, TextAreaField, SelectField
from wtforms.validators import Required, Optional, Email, Length, EqualTo, ValidationError, Regexp
# For DisabledSelectField
from wtforms.widgets import Select as SelectWidget
from wtforms.widgets import html_params, HTMLString
from flask_wtf.recaptcha import RecaptchaField
class Unique(object):
""" validator that checks field uniqueness """
def __init__(self, model, field, message=None):
self.model = model
self.field = field
if not message:
message = 'This element already exists'
self.message = message
def __call__(self, form, field):
check = self.model.query.filter(self.field == field.data).first()
if check:
raise ValidationError(self.message)
_username_validator = Regexp(
r'[a-zA-Z0-9_\-]+',
message='Your username must only consist of alphanumerics and _- (a-zA-Z0-9_-)')
class LoginForm(FlaskForm):
username = TextField('Username or email address', [Required(), _username_validator])
password = PasswordField('Password', [Required()])
class RegisterForm(FlaskForm):
username = TextField('Username', [
Required(),
Length(min=3, max=32),
_username_validator,
Unique(User, User.username, 'Username not availiable')
])
email = TextField('Email address', [
Email(),
Required(),
Length(min=5, max=128),
Unique(User, User.email, 'Email already in use by another account')
])
password = PasswordField('Password', [
Required(),
EqualTo('password_confirm', message='Passwords must match'),
Length(min=6, max=1024,
message='Password must be at least %(min)d characters long.')
])
password_confirm = PasswordField('Password (confirm)')
if app.config['USE_RECAPTCHA']:
recaptcha = RecaptchaField()
class ProfileForm(FlaskForm):
email = TextField('New email address', [
Email(),
Optional(),
Length(min=5, max=128),
Unique(User, User.email, 'Email is taken')
])
current_password = PasswordField('Current password', [Optional()])
new_password = PasswordField('New password (confirm)', [
Optional(),
EqualTo('password_confirm', message='Passwords must match'),
Length(min=6, max=1024,
message='Password must be at least %(min)d characters long.')
])
password_confirm = PasswordField('Repeat Password')
# Classes for a SelectField that can be set to disable options (id, name, disabled)
# TODO: Move to another file for cleaner look
class DisabledSelectWidget(SelectWidget):
def __call__(self, field, **kwargs):
kwargs.setdefault('id', field.id)
if self.multiple:
kwargs['multiple'] = True
html = ['<select %s>' % html_params(name=field.name, **kwargs)]
for val, label, selected, disabled in field.iter_choices():
extra = disabled and {'disabled': ''} or {}
html.append(self.render_option(val, label, selected, **extra))
html.append('</select>')
return HTMLString(''.join(html))
class DisabledSelectField(SelectField):
widget = DisabledSelectWidget()
def iter_choices(self):
for choice_tuple in self.choices:
value, label = choice_tuple[:2]
disabled = len(choice_tuple) == 3 and choice_tuple[2] or False
yield (value, label, self.coerce(value) == self.data, disabled)
def pre_validate(self, form):
for v in self.choices:
if self.data == v[0]:
break
else:
raise ValueError(self.gettext('Not a valid choice'))
class EditForm(FlaskForm):
display_name = TextField('Display name', [
Length(min=3, max=255,
message='Torrent name must be at least %(min)d characters and %(max)d at most.')
])
category = DisabledSelectField('Category')
def validate_category(form, field):
cat_match = re.match(r'^(\d+)_(\d+)$', field.data)
if not cat_match:
raise ValidationError('You must select a category')
main_cat_id = int(cat_match.group(1))
sub_cat_id = int(cat_match.group(2))
cat = models.SubCategory.by_category_ids(main_cat_id, sub_cat_id)
if not cat:
raise ValidationError('You must select a proper category')
field.parsed_data = cat
is_hidden = BooleanField('Hidden')
is_deleted = BooleanField('Deleted')
is_remake = BooleanField('Remake')
is_anonymous = BooleanField('Anonymous')
is_complete = BooleanField('Complete')
information = TextField('Information', [
Length(max=255, message='Information must be at most %(max)d characters long.')
])
description = TextAreaField('Description (markdown supported)', [
Length(max=10 * 1024, message='Description must be at most %(max)d characters long.')
])
class UploadForm(FlaskForm):
class Meta:
csrf = False
torrent_file = FileField('Torrent file', [
FileRequired()
])
display_name = TextField('Display name (optional)', [
Optional(),
Length(min=3, max=255,
message='Torrent name must be at least %(min)d characters long and %(max)d at most.')
])
# category = SelectField('Category')
category = DisabledSelectField('Category')
def validate_category(form, field):
cat_match = re.match(r'^(\d+)_(\d+)$', field.data)
if not cat_match:
raise ValidationError('You must select a category')
main_cat_id = int(cat_match.group(1))
sub_cat_id = int(cat_match.group(2))
cat = models.SubCategory.by_category_ids(main_cat_id, sub_cat_id)
if not cat:
raise ValidationError('You must select a proper category')
field.parsed_data = cat
is_hidden = BooleanField('Hidden')
is_remake = BooleanField('Remake')
is_anonymous = BooleanField('Anonymous')
is_complete = BooleanField('Complete')
information = TextField('Information', [
Length(max=255, message='Information must be at most %(max)d characters long.')
])
description = TextAreaField('Description (markdown supported)', [
Length(max=10 * 1024, message='Description must be at most %(max)d characters long.')
])
def validate_torrent_file(form, field):
# Decode and ensure data is bencoded data
try:
torrent_dict = bencode.decode(field.data)
#field.data.close()
except (bencode.MalformedBencodeException, UnicodeError):
raise ValidationError('Malformed torrent file')
# Uncomment for debug print of the torrent
# _debug_print_torrent_metadata(torrent_dict)
try:
_validate_torrent_metadata(torrent_dict)
except AssertionError as e:
raise ValidationError('Malformed torrent metadata ({})'.format(e.args[0]))
try:
_validate_trackers(torrent_dict)
except AssertionError as e:
raise ValidationError('Malformed torrent trackers ({})'.format(e.args[0]))
if app.config.get('ENFORCE_MAIN_ANNOUNCE_URL'):
main_announce_url = app.config.get('MAIN_ANNOUNCE_URL')
if not main_announce_url:
raise Exception('Config MAIN_ANNOUNCE_URL not set!')
announce = torrent_dict.get('announce', b'').decode('utf-8')
if announce != main_announce_url:
raise ValidationError('Please set {} as the first tracker in the torrent'.format(main_announce_url))
# Note! bencode will sort dict keys, as per the spec
# This may result in a different hash if the uploaded torrent does not match the
# spec, but it's their own fault for using broken software! Right?
bencoded_info_dict = bencode.encode(torrent_dict['info'])
info_hash = utils.sha1_hash(bencoded_info_dict)
# Check if the info_hash exists already in the database
existing_torrent = models.Torrent.by_info_hash(info_hash)
if existing_torrent:
raise ValidationError('That torrent already exists (#{})'.format(existing_torrent.id))
# Torrent is legit, pass original filename and dict along
field.parsed_data = TorrentFileData(filename=os.path.basename(field.data.filename),
torrent_dict=torrent_dict,
info_hash=info_hash,
bencoded_info_dict=bencoded_info_dict)
class TorrentFileData(object):
"""Quick and dirty class to pass data from the validator"""
def __init__(self, **kwargs):
for k, v in kwargs.items():
setattr(self, k, v)
# https://wiki.theory.org/BitTorrentSpecification#Metainfo_File_Structure
def _validate_trackers(torrent_dict):
announce = torrent_dict.get('announce')
_validate_bytes(announce)
announce_list = torrent_dict.get('announce-list')
if announce_list is not None:
_validate_list(announce_list, 'announce-list')
for announce in announce_list:
_validate_list(announce, 'announce-list item')
_validate_bytes(announce[0], 'announce-list item item')
def _validate_torrent_metadata(torrent_dict):
''' Validates a torrent metadata dict, raising AssertionError on errors '''
assert isinstance(torrent_dict, dict), 'torrent metadata is not a dict'
info_dict = torrent_dict.get('info')
assert info_dict is not None, 'no info_dict in torrent'
assert isinstance(info_dict, dict), 'info is not a dict'
encoding_bytes = torrent_dict.get('encoding', b'utf-8')
encoding = _validate_bytes(encoding_bytes, 'encoding', 'utf-8').lower()
name = info_dict.get('name')
_validate_bytes(name, 'name', test_decode=encoding)
piece_length = info_dict.get('piece length')
_validate_number(piece_length, 'piece length', check_positive=True)
pieces = info_dict.get('pieces')
_validate_bytes(pieces, 'pieces')
assert len(pieces) % 20 == 0, 'pieces length is not a multiple of 20'
files = info_dict.get('files')
if files is not None:
_validate_list(files, 'filelist')
for file_dict in files:
file_length = file_dict.get('length')
_validate_number(file_length, 'file length', check_positive_or_zero=True)
path_list = file_dict.get('path')
_validate_list(path_list, 'path')
for path_part in path_list:
_validate_bytes(path_part, 'path part', test_decode=encoding)
else:
length = info_dict.get('length')
_validate_number(length, 'length', check_positive=True)
def _validate_bytes(value, name='value', test_decode=None):
assert isinstance(value, bytes), name + ' is not bytes'
assert len(value) > 0, name + ' is empty'
if test_decode:
try:
return value.decode(test_decode)
except UnicodeError:
raise AssertionError(name + ' could not be decoded from ' + repr(test_decode))
def _validate_number(value, name='value', check_positive=False, check_positive_or_zero=False):
assert isinstance(value, int), name + ' is not an int'
if check_positive_or_zero:
assert value >= 0, name + ' is less than 0'
elif check_positive:
assert value > 0, name + ' is not positive'
def _validate_list(value, name='value', check_empty=False):
assert isinstance(value, list), name + ' is not a list'
if check_empty:
assert len(value) > 0, name + ' is empty'
def _debug_print_torrent_metadata(torrent_dict):
from pprint import pprint
# Temporarily remove 'pieces' from infodict for clean debug prints
info_dict = torrent_dict.get('info', {})
orig_pieces = info_dict.get('pieces')
info_dict['pieces'] = '<piece data>'
pprint(torrent_dict)
info_dict['pieces'] = orig_pieces

332
nyaa/models.py Normal file
View file

@ -0,0 +1,332 @@
from enum import Enum, IntEnum
from datetime import datetime
from nyaa import app, db
from nyaa.torrents import create_magnet
from sqlalchemy import func, ForeignKeyConstraint, Index
from sqlalchemy_utils import ChoiceType, EmailType, PasswordType
from werkzeug.security import generate_password_hash, check_password_hash
from sqlalchemy_fulltext import FullText
if app.config['USE_MYSQL']:
from sqlalchemy.dialects import mysql
BinaryType = mysql.BINARY
DescriptionTextType = mysql.TEXT
MediumBlobType = mysql.MEDIUMBLOB
COL_UTF8_GENERAL_CI = 'utf8_general_ci'
COL_UTF8MB4_BIN = 'utf8mb4_bin'
COL_ASCII_GENERAL_CI = 'ascii_general_ci'
else:
BinaryType = db.Binary
DescriptionTextType = db.String
MediumBlobType = db.BLOB
COL_UTF8_GENERAL_CI = 'NOCASE'
COL_UTF8MB4_BIN = None
COL_ASCII_GENERAL_CI = 'NOCASE'
class TorrentFlags(IntEnum):
NONE = 0
ANONYMOUS = 1
HIDDEN = 2
TRUSTED = 4
REMAKE = 8
COMPLETE = 16
DELETED = 32
class Torrent(db.Model):
__tablename__ = app.config['TABLE_PREFIX'] + 'torrents'
id = db.Column(db.Integer, primary_key=True)
info_hash = db.Column(BinaryType(length=20), unique=True, nullable=False, index=True)
display_name = db.Column(
db.String(length=255, collation=COL_UTF8_GENERAL_CI), nullable=False, index=True)
torrent_name = db.Column(db.String(length=255), nullable=False)
information = db.Column(db.String(length=255), nullable=False)
description = db.Column(DescriptionTextType(collation=COL_UTF8MB4_BIN), nullable=False)
filesize = db.Column(db.BIGINT, default=0, nullable=False, index=True)
encoding = db.Column(db.String(length=32), nullable=False)
flags = db.Column(db.Integer, default=0, nullable=False, index=True)
uploader_id = db.Column(db.Integer, db.ForeignKey('users.id'), nullable=True)
has_torrent = db.Column(db.Boolean, nullable=False, default=False)
created_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow, nullable=False)
updated_time = db.Column(db.DateTime(timezone=False),
default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
main_category_id = db.Column(db.Integer, db.ForeignKey(
app.config['TABLE_PREFIX'] + 'main_categories.id'), nullable=False)
sub_category_id = db.Column(db.Integer, nullable=False)
redirect = db.Column(db.Integer, db.ForeignKey(
app.config['TABLE_PREFIX'] + 'torrents.id'), nullable=True)
__table_args__ = (
Index('uploader_flag_idx', 'uploader_id', 'flags'),
ForeignKeyConstraint(
['main_category_id', 'sub_category_id'],
[app.config['TABLE_PREFIX'] + 'sub_categories.main_category_id',
app.config['TABLE_PREFIX'] + 'sub_categories.id']
), {}
)
user = db.relationship('User', uselist=False, back_populates='torrents')
main_category = db.relationship('MainCategory', uselist=False,
back_populates='torrents', lazy="joined")
sub_category = db.relationship('SubCategory', uselist=False, backref='torrents', lazy="joined",
primaryjoin="and_(SubCategory.id == foreign(Torrent.sub_category_id), "
"SubCategory.main_category_id == Torrent.main_category_id)")
info = db.relationship('TorrentInfo', uselist=False, back_populates='torrent')
filelist = db.relationship('TorrentFilelist', uselist=False, back_populates='torrent')
stats = db.relationship('Statistic', uselist=False, back_populates='torrent', lazy='joined')
trackers = db.relationship('TorrentTrackers', uselist=True, lazy='joined')
def __repr__(self):
return '<{0} #{1.id} \'{1.display_name}\' {1.filesize}b>'.format(type(self).__name__, self)
@property
def magnet_uri(self):
return create_magnet(self)
@property
def anonymous(self):
return self.flags & TorrentFlags.ANONYMOUS
@anonymous.setter
def anonymous(self, value):
self.flags = (self.flags & ~TorrentFlags.ANONYMOUS) | (value and TorrentFlags.ANONYMOUS)
@property
def hidden(self):
return self.flags & TorrentFlags.HIDDEN
@hidden.setter
def hidden(self, value):
self.flags = (self.flags & ~TorrentFlags.HIDDEN) | (value and TorrentFlags.HIDDEN)
@property
def deleted(self):
return self.flags & TorrentFlags.DELETED
@deleted.setter
def deleted(self, value):
self.flags = (self.flags & ~TorrentFlags.DELETED) | (value and TorrentFlags.DELETED)
@property
def trusted(self):
return self.flags & TorrentFlags.TRUSTED
@trusted.setter
def trusted(self, value):
self.flags = (self.flags & ~TorrentFlags.TRUSTED) | (value and TorrentFlags.TRUSTED)
@property
def remake(self):
return self.flags & TorrentFlags.REMAKE
@remake.setter
def remake(self, value):
self.flags = (self.flags & ~TorrentFlags.REMAKE) | (value and TorrentFlags.REMAKE)
@property
def complete(self):
return self.flags & TorrentFlags.COMPLETE
@complete.setter
def complete(self, value):
self.flags = (self.flags & ~TorrentFlags.COMPLETE) | (value and TorrentFlags.COMPLETE)
@classmethod
def by_id(cls, id):
return cls.query.get(id)
@classmethod
def by_info_hash(cls, info_hash):
return cls.query.filter_by(info_hash=info_hash).first()
class TorrentNameSearch(FullText, Torrent):
__fulltext_columns__ = ('display_name',)
class TorrentFilelist(db.Model):
__tablename__ = app.config['TABLE_PREFIX'] + 'torrents_filelist'
__table_args__ = {'mysql_row_format': 'COMPRESSED'}
torrent_id = db.Column(db.Integer, db.ForeignKey(
app.config['TABLE_PREFIX'] + 'torrents.id', ondelete="CASCADE"), primary_key=True)
filelist_blob = db.Column(MediumBlobType, nullable=True)
torrent = db.relationship('Torrent', uselist=False, back_populates='filelist')
class TorrentInfo(db.Model):
__tablename__ = app.config['TABLE_PREFIX'] + 'torrents_info'
__table_args__ = {'mysql_row_format': 'COMPRESSED'}
torrent_id = db.Column(db.Integer, db.ForeignKey(
app.config['TABLE_PREFIX'] + 'torrents.id', ondelete="CASCADE"), primary_key=True)
info_dict = db.Column(MediumBlobType, nullable=True)
torrent = db.relationship('Torrent', uselist=False, back_populates='info')
class Statistic(db.Model):
__tablename__ = app.config['TABLE_PREFIX'] + 'statistics'
torrent_id = db.Column(db.Integer, db.ForeignKey(
app.config['TABLE_PREFIX'] + 'torrents.id', ondelete="CASCADE"), primary_key=True)
seed_count = db.Column(db.Integer, default=0, nullable=False, index=True)
leech_count = db.Column(db.Integer, default=0, nullable=False, index=True)
download_count = db.Column(db.Integer, default=0, nullable=False, index=True)
last_updated = db.Column(db.DateTime(timezone=False))
torrent = db.relationship('Torrent', uselist=False, back_populates='stats')
class Trackers(db.Model):
__tablename__ = 'trackers'
id = db.Column(db.Integer, primary_key=True)
uri = db.Column(db.String(length=255, collation=COL_UTF8_GENERAL_CI), nullable=False, unique=True)
disabled = db.Column(db.Boolean, nullable=False, default=False)
@classmethod
def by_uri(cls, uri):
return cls.query.filter_by(uri=uri).first()
class TorrentTrackers(db.Model):
__tablename__ = app.config['TABLE_PREFIX'] + 'torrent_trackers'
torrent_id = db.Column(db.Integer, db.ForeignKey(app.config['TABLE_PREFIX'] + 'torrents.id', ondelete="CASCADE"), primary_key=True)
tracker_id = db.Column(db.Integer, db.ForeignKey('trackers.id', ondelete="CASCADE"), primary_key=True)
order = db.Column(db.Integer, nullable=False, index=True)
tracker = db.relationship('Trackers', uselist=False, lazy='joined')
@classmethod
def by_torrent_id(cls, torrent_id):
return cls.query.filter_by(torrent_id=torrent_id).order_by(cls.order.desc())
class MainCategory(db.Model):
__tablename__ = app.config['TABLE_PREFIX'] + 'main_categories'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(length=64), nullable=False)
sub_categories = db.relationship('SubCategory', back_populates='main_category')
torrents = db.relationship('Torrent', back_populates='main_category')
def get_category_ids(self):
return (self.id, 0)
@property
def id_as_string(self):
return '_'.join(str(x) for x in self.get_category_ids())
@classmethod
def by_id(cls, id):
return cls.query.get(id)
class SubCategory(db.Model):
__tablename__ = app.config['TABLE_PREFIX'] + 'sub_categories'
id = db.Column(db.Integer, primary_key=True)
main_category_id = db.Column(db.Integer, db.ForeignKey(
app.config['TABLE_PREFIX'] + 'main_categories.id'), primary_key=True)
name = db.Column(db.String(length=64), nullable=False)
main_category = db.relationship('MainCategory', uselist=False, back_populates='sub_categories')
# torrents = db.relationship('Torrent', back_populates='sub_category'),
# primaryjoin="and_(Torrent.sub_category_id == foreign(SubCategory.id), "