mirror of
https://gitlab.com/SIGBUS/nyaa.git
synced 2024-10-07 04:01:53 +00:00
Initial commit.
This commit is contained in:
commit
00d65e312c
48
README.md
Normal file
48
README.md
Normal file
|
@ -0,0 +1,48 @@
|
|||
# NyaaV2
|
||||
|
||||
## Setup:
|
||||
|
||||
- Create your virtualenv, for example with `pyvenv venv`
|
||||
- Enter your virtualenv with `source venv/bin/activate`
|
||||
- Install dependencies with `pip install -r requirements.txt`
|
||||
- Run `python db_create.py` to create the database
|
||||
- Start the dev server with `python run.py`
|
||||
|
||||
## Updated Setup (python 3.6.1):
|
||||
|
||||
- Install dependencies https://github.com/pyenv/pyenv/wiki/Common-build-problems
|
||||
- Install `pyenv` https://github.com/pyenv/pyenv/blob/master/README.md#installation
|
||||
- Install `pyenv-virtualenv` https://github.com/pyenv/pyenv-virtualenv/blob/master/README.md
|
||||
- `pyenv install 3.6.1`
|
||||
- `pyenv virtualenv 3.6.1 nyaa`
|
||||
- `pyenv activate nyaa`
|
||||
- Install dependencies with `pip install -r requirements.txt`
|
||||
- Copy `config.example.py` into `config.py`
|
||||
- Change TALBE_PREFIX to `nyaa_` or `sukebei_` depending on the site
|
||||
|
||||
## Setting up MySQL/MariaDB database for advanced functionality
|
||||
- Enable `USE_MYSQL` flag in config.py
|
||||
- Install latest mariadb by following instructions here https://downloads.mariadb.org/mariadb/repositories/
|
||||
- Tested versions: `mysql Ver 15.1 Distrib 10.0.30-MariaDB, for debian-linux-gnu (x86_64) using readline 5.2`
|
||||
- Run the following commands logged in as your root db user:
|
||||
- `CREATE USER 'test'@'localhost' IDENTIFIED BY 'test123';`
|
||||
- `GRANT ALL PRIVILEGES ON * . * TO 'test'@'localhost';`
|
||||
- `FLUSH PRIVILEGES;`
|
||||
- `CREATE DATABASE nyaav2 DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;`
|
||||
- To setup and import nyaa_maria_vx.sql:
|
||||
- `mysql -u <user> -p nyaav2`
|
||||
- `DROP DATABASE nyaav2;`
|
||||
- `CREATE DATABASE nyaav2 DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;`
|
||||
- `SOURCE ~/path/to/database/nyaa_maria_vx.sql`
|
||||
|
||||
## Finishing up
|
||||
- Run `python db_create.py` to create the database
|
||||
- Load the .sql file
|
||||
- `mysql -u user -p nyaav2`
|
||||
- `SOURCE cocks.sql`
|
||||
- Remember to change the default user password to an empty string to disable logging in
|
||||
- Start the dev server with `python run.py`
|
||||
- Deactivate `source deactivate`
|
||||
|
||||
## Code Quality:
|
||||
- Remember to follow PEP8 style guidelines and run `./lint.sh` before committing.
|
15
WSGI.py
Normal file
15
WSGI.py
Normal file
|
@ -0,0 +1,15 @@
|
|||
#!/usr/bin/python3
|
||||
# -*- coding: utf-8 -*-
|
||||
import gevent.monkey
|
||||
gevent.monkey.patch_all()
|
||||
|
||||
from nyaa import app
|
||||
|
||||
if app.config["DEBUG"]:
|
||||
from werkzeug.debug import DebuggedApplication
|
||||
app.wsgi_app = DebuggedApplication(app.wsgi_app, True)
|
||||
|
||||
if __name__ == '__main__':
|
||||
import gevent.pywsgi
|
||||
gevent_server = gevent.pywsgi.WSGIServer(("localhost", 5000), app.wsgi_app)
|
||||
gevent_server.serve_forever()
|
55
api_uploader.py
Normal file
55
api_uploader.py
Normal file
|
@ -0,0 +1,55 @@
|
|||
# api_uploader.py
|
||||
|
||||
|
||||
# Uploads a single file
|
||||
# I will create another script for batch uploading
|
||||
|
||||
import json
|
||||
import requests
|
||||
|
||||
url = "http://127.0.0.1:5500/api/upload"
|
||||
|
||||
# Required for Auth
|
||||
username = ""
|
||||
password = ""
|
||||
|
||||
# Required
|
||||
torrent_name = ""
|
||||
|
||||
# Required
|
||||
main_cat = ""
|
||||
# Required
|
||||
sub_cat = ""
|
||||
|
||||
# May be blank
|
||||
information = ""
|
||||
# May be blank
|
||||
description = ""
|
||||
# flags = [Hidden, Remake, Complete, Anonymous]
|
||||
# 0 for NOT SET / 1 for SET
|
||||
# Required
|
||||
flags = [0, 0, 0, 0]
|
||||
|
||||
metadata={
|
||||
"username": username,
|
||||
"password": password,
|
||||
"display_name": torrent_name,
|
||||
"main_cat": main_cat,
|
||||
"sub_cat": sub_cat,
|
||||
"information": information,
|
||||
"description": description,
|
||||
"flags": flags
|
||||
}
|
||||
|
||||
# Required
|
||||
file_name = ""
|
||||
|
||||
files = {
|
||||
'json': (json.dumps(metadata)),
|
||||
'torrent': ('{0}'.format(file_name), open(file_name, 'rb'), 'application/octet-stream')}
|
||||
|
||||
response = requests.post(url, files=files)
|
||||
|
||||
json_response = response.json()
|
||||
|
||||
print(json_response)
|
2
batch_upload_torrent.sh
Executable file
2
batch_upload_torrent.sh
Executable file
|
@ -0,0 +1,2 @@
|
|||
up_t() { curl -F "category=1_2" -F "torrent_file=@$1" 'http://localhost:5500/upload'; }
|
||||
for x in test_torrent_batch/*; do up_t "$x"; done
|
51
config.example.py
Normal file
51
config.example.py
Normal file
|
@ -0,0 +1,51 @@
|
|||
import os
|
||||
|
||||
|
||||
DEBUG = True
|
||||
USE_RECAPTCHA = False
|
||||
USE_EMAIL_VERIFICATION = False
|
||||
USE_MYSQL = True
|
||||
|
||||
# Enable this once stat integration is done
|
||||
ENABLE_SHOW_STATS = False
|
||||
|
||||
BASE_DIR = os.path.abspath(os.path.dirname(__file__))
|
||||
if USE_MYSQL:
|
||||
SQLALCHEMY_DATABASE_URI = ('mysql://test:test123@localhost/nyaav2')
|
||||
else:
|
||||
SQLALCHEMY_DATABASE_URI = (
|
||||
'sqlite:///' + os.path.join(BASE_DIR, 'test.db') + '?check_same_thread=False')
|
||||
|
||||
CSRF_SESSION_KEY = '***'
|
||||
SECRET_KEY = '***'
|
||||
|
||||
# Prefix for running multiple sites, user table will not be prefixed.
|
||||
# For sukebei, change 'nyaa_' to 'sukebei_'
|
||||
TABLE_PREFIX = 'nyaa_'
|
||||
|
||||
# for recaptcha and email verification:
|
||||
# keys for localhost. Change as appropriate when actual domain is registered.
|
||||
RECAPTCHA_PUBLIC_KEY = '***'
|
||||
RECAPTCHA_PRIVATE_KEY = '***'
|
||||
SMTP_SERVER = '***'
|
||||
SMTP_PORT = 587
|
||||
MAIL_FROM_ADDRESS = '***'
|
||||
SMTP_USERNAME = '***'
|
||||
SMTP_PASSWORD = '***'
|
||||
|
||||
RESULTS_PER_PAGE = 75
|
||||
|
||||
# What the site identifies itself as.
|
||||
SITE_NAME = 'Nyaa'
|
||||
|
||||
# The maximum number of files a torrent can contain
|
||||
# until the site says "Too many files to display."
|
||||
MAX_FILES_VIEW = 1000
|
||||
|
||||
# """
|
||||
# Setting to make sure main announce url is present in torrent
|
||||
# """
|
||||
ENFORCE_MAIN_ANNOUNCE_URL = False
|
||||
MAIN_ANNOUNCE_URL = ''
|
||||
|
||||
BACKUP_TORRENT_FOLDER = 'torrents'
|
40
db_create.py
Normal file
40
db_create.py
Normal file
|
@ -0,0 +1,40 @@
|
|||
import sys
|
||||
from nyaa import app, db, models
|
||||
|
||||
# Create tables
|
||||
|
||||
db.create_all()
|
||||
|
||||
# Insert categories
|
||||
|
||||
if app.config['TABLE_PREFIX'] == 'nyaa_':
|
||||
CATEGORIES = [
|
||||
('Anime', ['Anime Music Video', 'English-translated', 'Non-English-translated', 'Raw']),
|
||||
('Audio', ['Lossless', 'Lossy']),
|
||||
('Literature', ['English-translated', 'Non-English-translated', 'Raw']),
|
||||
('Live Action', ['English-translated', 'Idol/Promotional Video', 'Non-English-translated', 'Raw']),
|
||||
('Pictures', ['Graphics', 'Photos']),
|
||||
('Software', ['Applications', 'Games']),
|
||||
]
|
||||
elif app.config['TABLE_PREFIX'] == 'sukebei_':
|
||||
CATEGORIES = [
|
||||
('Art', ['Anime', 'Doujinshi', 'Games', 'Manga', 'Pictures']),
|
||||
('Real Life', ['Photobooks / Pictures', 'Videos']),
|
||||
]
|
||||
else:
|
||||
CATEGORIES = []
|
||||
|
||||
for main_cat_name, sub_cat_names in CATEGORIES:
|
||||
main_cat = models.MainCategory(name=main_cat_name)
|
||||
for i, sub_cat_name in enumerate(sub_cat_names):
|
||||
# Composite keys can't autoincrement, set sub_cat id manually (1-index)
|
||||
sub_cat = models.SubCategory(id=i+1, name=sub_cat_name, main_category=main_cat)
|
||||
db.session.add(main_cat)
|
||||
|
||||
db.session.commit()
|
||||
|
||||
# Create fulltext index
|
||||
|
||||
if app.config['USE_MYSQL']:
|
||||
db.engine.execute('ALTER TABLE ' + app.config['TABLE_PREFIX'] + 'torrents ADD FULLTEXT KEY (display_name)')
|
||||
|
6
lint.sh
Executable file
6
lint.sh
Executable file
|
@ -0,0 +1,6 @@
|
|||
autopep8 nyaa/ \
|
||||
--recursive \
|
||||
--in-place \
|
||||
--pep8-passes 2000 \
|
||||
--max-line-length 100 \
|
||||
--verbose
|
6
my.cnf
Normal file
6
my.cnf
Normal file
|
@ -0,0 +1,6 @@
|
|||
[mysqld]
|
||||
innodb_ft_min_token_size=2
|
||||
ft_min_word_len=2
|
||||
innodb_ft_cache_size = 80000000
|
||||
innodb_ft_total_cache_size = 1600000000
|
||||
max_allowed_packet = 100M
|
53
nyaa/__init__.py
Normal file
53
nyaa/__init__.py
Normal file
|
@ -0,0 +1,53 @@
|
|||
import os
|
||||
import logging
|
||||
import flask
|
||||
from flask_sqlalchemy import SQLAlchemy
|
||||
from flask_assets import Environment, Bundle
|
||||
from flask_debugtoolbar import DebugToolbarExtension
|
||||
from nyaa import fix_paginate
|
||||
|
||||
|
||||
app = flask.Flask(__name__)
|
||||
app.config.from_object('config')
|
||||
|
||||
# Database
|
||||
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
|
||||
|
||||
# Debugging
|
||||
if app.config['DEBUG']:
|
||||
app.config['DEBUG_TB_INTERCEPT_REDIRECTS'] = False
|
||||
toolbar = DebugToolbarExtension(app)
|
||||
app.logger.setLevel(logging.DEBUG)
|
||||
else:
|
||||
app.logger.setLevel(logging.WARNING)
|
||||
|
||||
# Logging
|
||||
if 'LOG_FILE' in app.config:
|
||||
from logging.handlers import RotatingFileHandler
|
||||
app.log_handler = RotatingFileHandler(
|
||||
app.config['LOG_FILE'], maxBytes=10000, backupCount=1)
|
||||
app.logger.addHandler(app.log_handler)
|
||||
|
||||
# Log errors and display a message to the user in production mdode
|
||||
if not app.config['DEBUG']:
|
||||
@app.errorhandler(500)
|
||||
def internal_error(exception):
|
||||
app.logger.error(exception)
|
||||
flask.flash(flask.Markup(
|
||||
'<strong>An error occured!</strong> Debugging information has been logged.'), 'danger')
|
||||
return flask.redirect('/')
|
||||
|
||||
# Enable the jinja2 do extension.
|
||||
app.jinja_env.add_extension('jinja2.ext.do')
|
||||
app.jinja_env.lstrip_blocks = True
|
||||
app.jinja_env.trim_blocks = True
|
||||
|
||||
db = SQLAlchemy(app)
|
||||
|
||||
assets = Environment(app)
|
||||
|
||||
# css = Bundle('style.scss', filters='libsass',
|
||||
# output='style.css', depends='**/*.scss')
|
||||
# assets.register('style_all', css)
|
||||
|
||||
from nyaa import routes
|
317
nyaa/api_handler.py
Normal file
317
nyaa/api_handler.py
Normal file
|
@ -0,0 +1,317 @@
|
|||
import flask
|
||||
from nyaa import app, db
|
||||
from nyaa import models, forms
|
||||
from nyaa import bencode, utils
|
||||
from nyaa import torrents
|
||||
|
||||
import json
|
||||
import os.path
|
||||
from orderedset import OrderedSet
|
||||
from werkzeug import secure_filename
|
||||
|
||||
DEBUG_API = False
|
||||
#################################### API ROUTES ####################################
|
||||
CATEGORIES = [
|
||||
('Anime', ['Anime Music Video', 'English-translated', 'Non-English-translated', 'Raw']),
|
||||
('Audio', ['Lossless', 'Lossy']),
|
||||
('Literature', ['English-translated', 'Non-English-translated', 'Raw']),
|
||||
('Live Action', ['English-translated',
|
||||
'Idol/Promotional Video', 'Non-English-translated', 'Raw']),
|
||||
('Pictures', ['Graphics', 'Photos']),
|
||||
('Software', ['Applications', 'Games']),
|
||||
]
|
||||
|
||||
|
||||
def validate_main_sub_cat(main_cat_name, sub_cat_name):
|
||||
for main_cat in models.MainCategory.query.order_by(models.MainCategory.id):
|
||||
if main_cat_name == main_cat.name:
|
||||
for sub_cat in main_cat.sub_categories:
|
||||
if sub_cat_name == sub_cat.name:
|
||||
cat_id = main_cat.id_as_string
|
||||
sub_cat_id = sub_cat.id_as_string
|
||||
cat_sub_cat = sub_cat_id.split('_')
|
||||
#print('cat: {0} sub_cat: {1}'.format(cat_sub_cat[0], cat_sub_cat[1]))
|
||||
|
||||
return True, cat_sub_cat[0], cat_sub_cat[1]
|
||||
|
||||
return False, 0, 0
|
||||
|
||||
|
||||
def _replace_utf8_values(dict_or_list):
|
||||
''' Will replace 'property' with 'property.utf-8' and remove latter if it exists.
|
||||
Thanks, bitcomet! :/ '''
|
||||
did_change = False
|
||||
if isinstance(dict_or_list, dict):
|
||||
for key in [key for key in dict_or_list.keys() if key.endswith('.utf-8')]:
|
||||
dict_or_list[key.replace('.utf-8', '')] = dict_or_list.pop(key)
|
||||
did_change = True
|
||||
for value in dict_or_list.values():
|
||||
did_change = _replace_utf8_values(value) or did_change
|
||||
elif isinstance(dict_or_list, list):
|
||||
for item in dict_or_list:
|
||||
did_change = _replace_utf8_values(item) or did_change
|
||||
return did_change
|
||||
|
||||
|
||||
def validate_torrent_flags(torrent_flags):
|
||||
_torrent_flags = ['hidden', 'remake', 'complete', 'anonymous']
|
||||
|
||||
if len(torrent_flags) != 4:
|
||||
return False
|
||||
|
||||
for flag in torrent_flags:
|
||||
if int(flag) not in [0, 1]:
|
||||
return False
|
||||
return True
|
||||
|
||||
# It might be good to factor this out of forms UploadForm because the same code is
|
||||
# used in both files.
|
||||
|
||||
|
||||
def validate_torrent_file(torrent_file_name, torrent_file):
|
||||
# Decode and ensure data is bencoded data
|
||||
try:
|
||||
torrent_dict = bencode.decode(torrent_file)
|
||||
except (bencode.MalformedBencodeException, UnicodeError):
|
||||
return False, 'Malformed torrent file'
|
||||
|
||||
# Uncomment for debug print of the torrent
|
||||
# forms._debug_print_torrent_metadata(torrent_dict)
|
||||
|
||||
try:
|
||||
forms._validate_torrent_metadata(torrent_dict)
|
||||
except AssertionError as e:
|
||||
return False, 'Malformed torrent metadata ({})'.format(e.args[0])
|
||||
|
||||
# Note! bencode will sort dict keys, as per the spec
|
||||
# This may result in a different hash if the uploaded torrent does not match the
|
||||
# spec, but it's their own fault for using broken software! Right?
|
||||
bencoded_info_dict = bencode.encode(torrent_dict['info'])
|
||||
info_hash = utils.sha1_hash(bencoded_info_dict)
|
||||
|
||||
# Check if the info_hash exists already in the database
|
||||
existing_torrent = models.Torrent.by_info_hash(info_hash)
|
||||
if existing_torrent:
|
||||
return False, 'That torrent already exists (#{})'.format(existing_torrent.id)
|
||||
|
||||
# Torrent is legit, pass original filename and dict along
|
||||
return True, forms.TorrentFileData(filename=os.path.basename(torrent_file_name),
|
||||
torrent_dict=torrent_dict,
|
||||
info_hash=info_hash,
|
||||
bencoded_info_dict=bencoded_info_dict)
|
||||
|
||||
|
||||
def api_upload(upload_request):
|
||||
if upload_request.method == 'POST':
|
||||
j = None
|
||||
torrent_file = None
|
||||
try:
|
||||
if 'json' in upload_request.files:
|
||||
f = upload_request.files['json']
|
||||
j = json.loads(f.read().decode('utf-8'))
|
||||
if DEBUG_API:
|
||||
print(json.dumps(j, indent=4))
|
||||
|
||||
_json_keys = ['username', 'password',
|
||||
'display_name', 'main_cat', 'sub_cat', 'flags'] # 'information' and 'description' are not required
|
||||
# Check that required fields are present
|
||||
for _k in _json_keys:
|
||||
if _k not in j.keys():
|
||||
return flask.make_response(flask.jsonify({"Error": "Missing JSON field: {0}.".format(_k)}), 400)
|
||||
# Check that no extra fields are present
|
||||
for k in j.keys():
|
||||
if k not in ['username', 'password',
|
||||
'display_name', 'main_cat', 'sub_cat', 'information', 'description', 'flags']:
|
||||
return flask.make_response(flask.jsonify({"Error": "Incorrect JSON field(s)."}), 400)
|
||||
else:
|
||||
return flask.make_response(flask.jsonify({"Error": "No metadata."}), 400)
|
||||
if 'torrent' in upload_request.files:
|
||||
f = upload_request.files['torrent']
|
||||
if DEBUG_API:
|
||||
print(f.filename)
|
||||
torrent_file = f
|
||||
# print(f.read())
|
||||
else:
|
||||
return flask.make_response(flask.jsonify({"Error": "No torrent file."}), 400)
|
||||
|
||||
# 'username' and 'password' must have been provided as they are part of j.keys()
|
||||
username = j['username']
|
||||
password = j['password']
|
||||
# Validate that the provided username and password belong to a valid user
|
||||
user = models.User.by_username(username)
|
||||
|
||||
if not user:
|
||||
user = models.User.by_email(username)
|
||||
|
||||
if not user or password != user.password_hash or user.status == models.UserStatusType.INACTIVE:
|
||||
return flask.make_response(flask.jsonify({"Error": "Incorrect username or password"}), 403)
|
||||
|
||||
current_user = user
|
||||
|
||||
display_name = j['display_name']
|
||||
if (len(display_name) < 3) or (len(display_name) > 1024):
|
||||
return flask.make_response(flask.jsonify({"Error": "Torrent name must be between 3 and 1024 characters."}), 400)
|
||||
|
||||
main_cat_name = j['main_cat']
|
||||
sub_cat_name = j['sub_cat']
|
||||
|
||||
cat_subcat_status, cat_id, sub_cat_id = validate_main_sub_cat(
|
||||
main_cat_name, sub_cat_name)
|
||||
if not cat_subcat_status:
|
||||
return flask.make_response(flask.jsonify({"Error": "Incorrect Category / Sub-Category."}), 400)
|
||||
|
||||
# TODO Sanitize information
|
||||
information = None
|
||||
try:
|
||||
information = j['information']
|
||||
if len(information) > 255:
|
||||
return flask.make_response(flask.jsonify({"Error": "Information is limited to 255 characters."}), 400)
|
||||
except Exception as e:
|
||||
information = ''
|
||||
|
||||
# TODO Sanitize description
|
||||
description = None
|
||||
try:
|
||||
description = j['description']
|
||||
if len(description) > (10 * 1024):
|
||||
return flask.make_response(flask.jsonify({"Error": "Description is limited to {0} characters.".format(10 * 1024)}), 403)
|
||||
except Exception as e:
|
||||
description = ''
|
||||
|
||||
v_flags = validate_torrent_flags(j['flags'])
|
||||
if v_flags:
|
||||
torrent_flags = j['flags']
|
||||
else:
|
||||
return flask.make_response(flask.jsonify({"Error": "Incorrect torrent flags."}), 400)
|
||||
|
||||
torrent_status, torrent_data = validate_torrent_file(
|
||||
torrent_file.filename, torrent_file.read()) # Needs validation
|
||||
|
||||
if not torrent_status:
|
||||
return flask.make_response(flask.jsonify({"Error": "Invalid or Duplicate torrent file."}), 400)
|
||||
|
||||
# The torrent has been validated and is safe to access with ['foo'] etc - all relevant
|
||||
# keys and values have been checked for (see UploadForm in forms.py for details)
|
||||
info_dict = torrent_data.torrent_dict['info']
|
||||
|
||||
changed_to_utf8 = _replace_utf8_values(torrent_data.torrent_dict)
|
||||
|
||||
torrent_filesize = info_dict.get('length') or sum(
|
||||
f['length'] for f in info_dict.get('files'))
|
||||
|
||||
# In case no encoding, assume UTF-8.
|
||||
torrent_encoding = torrent_data.torrent_dict.get('encoding', b'utf-8').decode('utf-8')
|
||||
|
||||
torrent = models.Torrent(info_hash=torrent_data.info_hash,
|
||||
display_name=display_name,
|
||||
torrent_name=torrent_data.filename,
|
||||
information=information,
|
||||
description=description,
|
||||
encoding=torrent_encoding,
|
||||
filesize=torrent_filesize,
|
||||
user=current_user)
|
||||
|
||||
# Store bencoded info_dict
|
||||
torrent.info = models.TorrentInfo(info_dict=torrent_data.bencoded_info_dict)
|
||||
torrent.stats = models.Statistic()
|
||||
torrent.has_torrent = True
|
||||
|
||||
# Fields with default value will be None before first commit, so set .flags
|
||||
torrent.flags = 0
|
||||
|
||||
torrent.anonymous = True if torrent_flags[0] else False
|
||||
torrent.hidden = True if torrent_flags[1] else False
|
||||
torrent.remake = True if torrent_flags[2] else False
|
||||
torrent.complete = True if torrent_flags[3] else False
|
||||
# Copy trusted status from user if possible
|
||||
torrent.trusted = (current_user.level >=
|
||||
models.UserLevelType.TRUSTED) if current_user else False
|
||||
|
||||
# Set category ids
|
||||
torrent.main_category_id = cat_id
|
||||
torrent.sub_category_id = sub_cat_id
|
||||
# To simplify parsing the filelist, turn single-file torrent into a list
|
||||
torrent_filelist = info_dict.get('files')
|
||||
|
||||
used_path_encoding = changed_to_utf8 and 'utf-8' or torrent_encoding
|
||||
|
||||
parsed_file_tree = dict()
|
||||
if not torrent_filelist:
|
||||
# If single-file, the root will be the file-tree (no directory)
|
||||
file_tree_root = parsed_file_tree
|
||||
torrent_filelist = [{'length': torrent_filesize, 'path': [info_dict['name']]}]
|
||||
else:
|
||||
# If multi-file, use the directory name as root for files
|
||||
file_tree_root = parsed_file_tree.setdefault(
|
||||
info_dict['name'].decode(used_path_encoding), {})
|
||||
|
||||
# Parse file dicts into a tree
|
||||
for file_dict in torrent_filelist:
|
||||
# Decode path parts from utf8-bytes
|
||||
path_parts = [path_part.decode(used_path_encoding)
|
||||
for path_part in file_dict['path']]
|
||||
|
||||
filename = path_parts.pop()
|
||||
current_directory = file_tree_root
|
||||
|
||||
for directory in path_parts:
|
||||
current_directory = current_directory.setdefault(directory, {})
|
||||
|
||||
current_directory[filename] = file_dict['length']
|
||||
|
||||
parsed_file_tree = utils.sorted_pathdict(parsed_file_tree)
|
||||
|
||||
json_bytes = json.dumps(parsed_file_tree, separators=(',', ':')).encode('utf8')
|
||||
torrent.filelist = models.TorrentFilelist(filelist_blob=json_bytes)
|
||||
|
||||
db.session.add(torrent)
|
||||
db.session.flush()
|
||||
|
||||
# Store the users trackers
|
||||
trackers = OrderedSet()
|
||||
announce = torrent_data.torrent_dict.get('announce', b'').decode('ascii')
|
||||
if announce:
|
||||
trackers.add(announce)
|
||||
|
||||
# List of lists with single item
|
||||
announce_list = torrent_data.torrent_dict.get('announce-list', [])
|
||||
for announce in announce_list:
|
||||
trackers.add(announce[0].decode('ascii'))
|
||||
|
||||
# Remove our trackers, maybe? TODO ?
|
||||
|
||||
# Search for/Add trackers in DB
|
||||
db_trackers = OrderedSet()
|
||||
for announce in trackers:
|
||||
tracker = models.Trackers.by_uri(announce)
|
||||
|
||||
# Insert new tracker if not found
|
||||
if not tracker:
|
||||
tracker = models.Trackers(uri=announce)
|
||||
db.session.add(tracker)
|
||||
|
||||
db_trackers.add(tracker)
|
||||
|
||||
db.session.flush()
|
||||
|
||||
# Store tracker refs in DB
|
||||
for order, tracker in enumerate(db_trackers):
|
||||
torrent_tracker = models.TorrentTrackers(torrent_id=torrent.id,
|
||||
tracker_id=tracker.id, order=order)
|
||||
db.session.add(torrent_tracker)
|
||||
|
||||
db.session.commit()
|
||||
|
||||
if app.config.get('BACKUP_TORRENT_FOLDER'):
|
||||
torrent_file.seek(0, 0)
|
||||
torrent_path = os.path.join(app.config['BACKUP_TORRENT_FOLDER'], '{}.{}'.format(torrent.id, secure_filename(torrent_file.filename)))
|
||||
torrent_file.save(torrent_path)
|
||||
torrent_file.close()
|
||||
|
||||
#print('Success? {0}'.format(torrent.id))
|
||||
return flask.make_response(flask.jsonify({"Success": "Request was processed {0}".format(torrent.id)}), 200)
|
||||
except Exception as e:
|
||||
print('Exception: {0}'.format(e))
|
||||
return flask.make_response(flask.jsonify({"Error": "Incorrect JSON. Please see HELP page for examples."}), 400)
|
||||
else:
|
||||
return flask.make_response(flask.jsonify({"Error": "Bad request"}), 400)
|
163
nyaa/bencode.py
Normal file
163
nyaa/bencode.py
Normal file
|
@ -0,0 +1,163 @@
|
|||
from io import BytesIO
|
||||
|
||||
|
||||
def _pairwise(iterable):
|
||||
""" Returns items from an iterable two at a time, ala
|
||||
[0, 1, 2, 3, ...] -> [(0, 1), (2, 3), ...] """
|
||||
iterable = iter(iterable)
|
||||
return zip(iterable, iterable)
|
||||
|
||||
|
||||
__all__ = ['encode', 'decode', 'BencodeException', 'MalformedBencodeException']
|
||||
|
||||
# https://wiki.theory.org/BitTorrentSpecification#Bencoding
|
||||
|
||||
|
||||
class BencodeException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class MalformedBencodeException(BencodeException):
|
||||
pass
|
||||
|
||||
|
||||
# bencode types
|
||||
_DIGITS = b'0123456789'
|
||||
_B_INT = b'i'
|
||||
_B_LIST = b'l'
|
||||
_B_DICT = b'd'
|
||||
_B_END = b'e'
|
||||
|
||||
|
||||
# Decoding of bencoded data
|
||||
|
||||
def _bencode_decode(file_object, decode_keys_as_utf8=True):
|
||||
""" Decodes a bencoded value, raising a MalformedBencodeException on errors.
|
||||
decode_keys_as_utf8 controls decoding dict keys as utf8 (which they
|
||||
almost always are) """
|
||||
if isinstance(file_object, str):
|
||||
file_object = file_object.encode('utf8')
|
||||
if isinstance(file_object, bytes):
|
||||
file_object = BytesIO(file_object)
|
||||
|
||||
def create_ex(msg):
|
||||
return MalformedBencodeException(
|
||||
'{0} at position {1} (0x{1:02X} hex)'.format(msg, file_object.tell()))
|
||||
|
||||
def _read_list():
|
||||
""" Decodes values from stream until a None is returned ('e') """
|
||||
items = []
|
||||
while True:
|
||||
value = _bencode_decode(file_object, decode_keys_as_utf8=decode_keys_as_utf8)
|
||||
if value is None:
|
||||
break
|
||||
items.append(value)
|
||||
return items
|
||||
|
||||
kind = file_object.read(1)
|
||||
|
||||
if kind == _B_INT: # Integer
|
||||
int_bytes = b''
|
||||
while True:
|
||||
c = file_object.read(1)
|
||||
if not c:
|
||||
raise create_ex('Unexpected end while reading an integer')
|
||||
elif c == _B_END:
|
||||
try:
|
||||
return int(int_bytes.decode('utf8'))
|
||||
except Exception as e:
|
||||
raise create_ex('Unable to parse int')
|
||||
|
||||
# not a digit OR '-' in the middle of the int
|
||||
if (c not in _DIGITS + b'-') or (c == b'-' and int_bytes):
|
||||
raise create_ex('Unexpected input while reading an integer: ' + repr(c))
|
||||
else:
|
||||
int_bytes += c
|
||||
|
||||
elif kind == _B_LIST: # List
|
||||
return _read_list()
|
||||
|
||||
elif kind == _B_DICT: # Dictionary
|
||||
keys_and_values = _read_list()
|
||||
if len(keys_and_values) % 2 != 0:
|
||||
raise MalformedBencodeException('Uneven amount of key/value pairs')
|
||||
|
||||
# "Technically" the bencode dictionary keys are bytestrings,
|
||||
# but real-world they're always(?) UTF-8.
|
||||
decoded_dict = dict((decode_keys_as_utf8 and k.decode('utf8') or k, v)
|
||||
for k, v in _pairwise(keys_and_values))
|
||||
return decoded_dict
|
||||
|
||||
# List/dict end, but make sure input is not just 'e'
|
||||
elif kind == _B_END and file_object.tell() > 0:
|
||||
return None
|
||||
|
||||
elif kind in _DIGITS: # Bytestring
|
||||
str_len_bytes = kind # keep first digit
|
||||
# Read string length until a ':'
|
||||
while True:
|
||||
c = file_object.read(1)
|
||||
if c in _DIGITS:
|
||||
str_len_bytes += c
|
||||
elif c == b':':
|
||||
break
|
||||
else:
|
||||
raise create_ex('Unexpected input while reading string length: ' + repr(c))
|
||||
try:
|
||||
str_len = int(str_len_bytes.decode())
|
||||
except Exception as e:
|
||||
raise create_ex('Unable to parse bytestring length')
|
||||
|
||||
bytestring = file_object.read(str_len)
|
||||
if len(bytestring) != str_len:
|
||||
raise create_ex('Read only {} bytes, {} wanted'.format(len(bytestring), str_len))
|
||||
|
||||
return bytestring
|
||||
else:
|
||||
raise create_ex('Unexpected data type ({})'.format(repr(kind)))
|
||||
|
||||
|
||||
# Bencoding
|
||||
|
||||
def _bencode_int(value):
|
||||
""" Encode an integer, eg 64 -> i64e """
|
||||
return _B_INT + str(value).encode('utf8') + _B_END
|
||||
|
||||
|
||||
def _bencode_bytes(value):
|
||||
""" Encode a bytestring (strings as UTF-8), eg 'hello' -> 5:hello """
|
||||
if isinstance(value, str):
|
||||
value = value.encode('utf8')
|
||||
return str(len(value)).encode('utf8') + b':' + value
|
||||
|
||||
|
||||
def _bencode_list(value):
|
||||
""" Encode a list, eg [64, "hello"] -> li64e5:helloe """
|
||||
return _B_LIST + b''.join(_bencode(item) for item in value) + _B_END
|
||||
|
||||
|
||||
def _bencode_dict(value):
|
||||
""" Encode a dict, which is keys and values interleaved as a list,
|
||||
eg {"hello":123}-> d5:helloi123ee """
|
||||
dict_keys = sorted(value.keys()) # Sort keys as per spec
|
||||
return _B_DICT + b''.join(
|
||||
_bencode_bytes(key) + _bencode(value[key]) for key in dict_keys) + _B_END
|
||||
|
||||
|
||||
def _bencode(value):
|
||||
""" Bencode any supported value (int, bytes, str, list, dict) """
|
||||
if isinstance(value, int):
|
||||
return _bencode_int(value)
|
||||
elif isinstance(value, (str, bytes)):
|
||||
return _bencode_bytes(value)
|
||||
elif isinstance(value, list):
|
||||
return _bencode_list(value)
|
||||
elif isinstance(value, dict):
|
||||
return _bencode_dict(value)
|
||||
|
||||
raise BencodeException('Unsupported type ' + str(type(value)))
|
||||
|
||||
|
||||
# The functions call themselves
|
||||
encode = _bencode
|
||||
decode = _bencode_decode
|
28
nyaa/fix_paginate.py
Normal file
28
nyaa/fix_paginate.py
Normal file
|
@ -0,0 +1,28 @@
|
|||
from flask_sqlalchemy import Pagination, BaseQuery
|
||||
from flask import abort
|
||||
|
||||
def paginate_faste(self, page=1, per_page=50, max_page=None, step=5):
|
||||
if page < 1:
|
||||
abort(404)
|
||||
|
||||
if max_page and page > max_page:
|
||||
abort(404)
|
||||
|
||||
items = self.limit(per_page).offset((page - 1) * per_page).all()
|
||||
|
||||
if not items and page != 1:
|
||||
abort(404)
|
||||
|
||||
# No need to count if we're on the first page and there are fewer
|
||||
# items than we expected.
|
||||
if page == 1 and len(items) < per_page:
|
||||
total = len(items)
|
||||
else:
|
||||
if max_page:
|
||||
total = self.order_by(None).limit(per_page * min((page + step), max_page)).count()
|
||||
else:
|
||||
total = self.order_by(None).limit(per_page * (page + step)).count()
|
||||
|
||||
return Pagination(self, page, per_page, total, items)
|
||||
|
||||
BaseQuery.paginate_faste = paginate_faste
|
352
nyaa/forms.py
Normal file
352
nyaa/forms.py
Normal file
|
@ -0,0 +1,352 @@
|
|||
from nyaa import db, app
|
||||
from nyaa.models import User
|
||||
from nyaa import bencode, utils, models
|
||||
|
||||
import os
|
||||
import re
|
||||
from flask_wtf import FlaskForm
|
||||
from flask_wtf.file import FileField, FileRequired
|
||||
from wtforms import TextField, PasswordField, BooleanField, TextAreaField, SelectField
|
||||
from wtforms.validators import Required, Optional, Email, Length, EqualTo, ValidationError, Regexp
|
||||
|
||||
# For DisabledSelectField
|
||||
from wtforms.widgets import Select as SelectWidget
|
||||
from wtforms.widgets import html_params, HTMLString
|
||||
|
||||
from flask_wtf.recaptcha import RecaptchaField
|
||||
|
||||
|
||||
class Unique(object):
|
||||
|
||||
""" validator that checks field uniqueness """
|
||||
|
||||
def __init__(self, model, field, message=None):
|
||||
self.model = model
|
||||
self.field = field
|
||||
if not message:
|
||||
message = 'This element already exists'
|
||||
self.message = message
|
||||
|
||||
def __call__(self, form, field):
|
||||
check = self.model.query.filter(self.field == field.data).first()
|
||||
if check:
|
||||
raise ValidationError(self.message)
|
||||
|
||||
|
||||
_username_validator = Regexp(
|
||||
r'[a-zA-Z0-9_\-]+',
|
||||
message='Your username must only consist of alphanumerics and _- (a-zA-Z0-9_-)')
|
||||
|
||||
|
||||
class LoginForm(FlaskForm):
|
||||
username = TextField('Username or email address', [Required(), _username_validator])
|
||||
password = PasswordField('Password', [Required()])
|
||||
|
||||
|
||||
class RegisterForm(FlaskForm):
|
||||
username = TextField('Username', [
|
||||
Required(),
|
||||
Length(min=3, max=32),
|
||||
_username_validator,
|
||||
Unique(User, User.username, 'Username not availiable')
|
||||
])
|
||||
|
||||
email = TextField('Email address', [
|
||||
Email(),
|
||||
Required(),
|
||||
Length(min=5, max=128),
|
||||
Unique(User, User.email, 'Email already in use by another account')
|
||||
])
|
||||
|
||||
password = PasswordField('Password', [
|
||||
Required(),
|
||||
EqualTo('password_confirm', message='Passwords must match'),
|
||||
Length(min=6, max=1024,
|
||||
message='Password must be at least %(min)d characters long.')
|
||||
])
|
||||
|
||||
password_confirm = PasswordField('Password (confirm)')
|
||||
|
||||
if app.config['USE_RECAPTCHA']:
|
||||
recaptcha = RecaptchaField()
|
||||
|
||||
|
||||
class ProfileForm(FlaskForm):
|
||||
email = TextField('New email address', [
|
||||
Email(),
|
||||
Optional(),
|
||||
Length(min=5, max=128),
|
||||
Unique(User, User.email, 'Email is taken')
|
||||
])
|
||||
|
||||
current_password = PasswordField('Current password', [Optional()])
|
||||
|
||||
new_password = PasswordField('New password (confirm)', [
|
||||
Optional(),
|
||||
EqualTo('password_confirm', message='Passwords must match'),
|
||||
Length(min=6, max=1024,
|
||||
message='Password must be at least %(min)d characters long.')
|
||||
])
|
||||
|
||||
password_confirm = PasswordField('Repeat Password')
|
||||
|
||||
|
||||
# Classes for a SelectField that can be set to disable options (id, name, disabled)
|
||||
# TODO: Move to another file for cleaner look
|
||||
class DisabledSelectWidget(SelectWidget):
|
||||
def __call__(self, field, **kwargs):
|
||||
kwargs.setdefault('id', field.id)
|
||||
if self.multiple:
|
||||
kwargs['multiple'] = True
|
||||
html = ['<select %s>' % html_params(name=field.name, **kwargs)]
|
||||
for val, label, selected, disabled in field.iter_choices():
|
||||
extra = disabled and {'disabled': ''} or {}
|
||||
html.append(self.render_option(val, label, selected, **extra))
|
||||
html.append('</select>')
|
||||
return HTMLString(''.join(html))
|
||||
|
||||
|
||||
class DisabledSelectField(SelectField):
|
||||
widget = DisabledSelectWidget()
|
||||
|
||||
def iter_choices(self):
|
||||
for choice_tuple in self.choices:
|
||||
value, label = choice_tuple[:2]
|
||||
disabled = len(choice_tuple) == 3 and choice_tuple[2] or False
|
||||
yield (value, label, self.coerce(value) == self.data, disabled)
|
||||
|
||||
def pre_validate(self, form):
|
||||
for v in self.choices:
|
||||
if self.data == v[0]:
|
||||
break
|
||||
else:
|
||||
raise ValueError(self.gettext('Not a valid choice'))
|
||||
|
||||
|
||||
class EditForm(FlaskForm):
|
||||
display_name = TextField('Display name', [
|
||||
Length(min=3, max=255,
|
||||
message='Torrent name must be at least %(min)d characters and %(max)d at most.')
|
||||
])
|
||||
|
||||
category = DisabledSelectField('Category')
|
||||
|
||||
def validate_category(form, field):
|
||||
cat_match = re.match(r'^(\d+)_(\d+)$', field.data)
|
||||
if not cat_match:
|
||||
raise ValidationError('You must select a category')
|
||||
|
||||
main_cat_id = int(cat_match.group(1))
|
||||
sub_cat_id = int(cat_match.group(2))
|
||||
|
||||
cat = models.SubCategory.by_category_ids(main_cat_id, sub_cat_id)
|
||||
|
||||
if not cat:
|
||||
raise ValidationError('You must select a proper category')
|
||||
|
||||
field.parsed_data = cat
|
||||
|
||||
is_hidden = BooleanField('Hidden')
|
||||
is_deleted = BooleanField('Deleted')
|
||||
is_remake = BooleanField('Remake')
|
||||
is_anonymous = BooleanField('Anonymous')
|
||||
is_complete = BooleanField('Complete')
|
||||
|
||||
information = TextField('Information', [
|
||||
Length(max=255, message='Information must be at most %(max)d characters long.')
|
||||
])
|
||||
description = TextAreaField('Description (markdown supported)', [
|
||||
Length(max=10 * 1024, message='Description must be at most %(max)d characters long.')
|
||||
])
|
||||
|
||||
|
||||
class UploadForm(FlaskForm):
|
||||
|
||||
class Meta:
|
||||
csrf = False
|
||||
|
||||
torrent_file = FileField('Torrent file', [
|
||||
FileRequired()
|
||||
])
|
||||
|
||||
display_name = TextField('Display name (optional)', [
|
||||
Optional(),
|
||||
Length(min=3, max=255,
|
||||
message='Torrent name must be at least %(min)d characters long and %(max)d at most.')
|
||||
])
|
||||
|
||||
# category = SelectField('Category')
|
||||
category = DisabledSelectField('Category')
|
||||
|
||||
def validate_category(form, field):
|
||||
cat_match = re.match(r'^(\d+)_(\d+)$', field.data)
|
||||
if not cat_match:
|
||||
raise ValidationError('You must select a category')
|
||||
|
||||
main_cat_id = int(cat_match.group(1))
|
||||
sub_cat_id = int(cat_match.group(2))
|
||||
|
||||
cat = models.SubCategory.by_category_ids(main_cat_id, sub_cat_id)
|
||||
|
||||
if not cat:
|
||||
raise ValidationError('You must select a proper category')
|
||||
|
||||
field.parsed_data = cat
|
||||
|
||||
is_hidden = BooleanField('Hidden')
|
||||
is_remake = BooleanField('Remake')
|
||||
is_anonymous = BooleanField('Anonymous')
|
||||
is_complete = BooleanField('Complete')
|
||||
|
||||
information = TextField('Information', [
|
||||
Length(max=255, message='Information must be at most %(max)d characters long.')
|
||||
])
|
||||
description = TextAreaField('Description (markdown supported)', [
|
||||
Length(max=10 * 1024, message='Description must be at most %(max)d characters long.')
|
||||
])
|
||||
|
||||
def validate_torrent_file(form, field):
|
||||
# Decode and ensure data is bencoded data
|
||||
try:
|
||||
torrent_dict = bencode.decode(field.data)
|
||||
#field.data.close()
|
||||
except (bencode.MalformedBencodeException, UnicodeError):
|
||||
raise ValidationError('Malformed torrent file')
|
||||
|
||||
# Uncomment for debug print of the torrent
|
||||
# _debug_print_torrent_metadata(torrent_dict)
|
||||
|
||||
try:
|
||||
_validate_torrent_metadata(torrent_dict)
|
||||
except AssertionError as e:
|
||||
raise ValidationError('Malformed torrent metadata ({})'.format(e.args[0]))
|
||||
|
||||
try:
|
||||
_validate_trackers(torrent_dict)
|
||||
except AssertionError as e:
|
||||
raise ValidationError('Malformed torrent trackers ({})'.format(e.args[0]))
|
||||
|
||||
if app.config.get('ENFORCE_MAIN_ANNOUNCE_URL'):
|
||||
main_announce_url = app.config.get('MAIN_ANNOUNCE_URL')
|
||||
if not main_announce_url:
|
||||
raise Exception('Config MAIN_ANNOUNCE_URL not set!')
|
||||
|
||||
announce = torrent_dict.get('announce', b'').decode('utf-8')
|
||||
if announce != main_announce_url:
|
||||
raise ValidationError('Please set {} as the first tracker in the torrent'.format(main_announce_url))
|
||||
|
||||
# Note! bencode will sort dict keys, as per the spec
|
||||
# This may result in a different hash if the uploaded torrent does not match the
|
||||
# spec, but it's their own fault for using broken software! Right?
|
||||
bencoded_info_dict = bencode.encode(torrent_dict['info'])
|
||||
info_hash = utils.sha1_hash(bencoded_info_dict)
|
||||
|
||||
# Check if the info_hash exists already in the database
|
||||
existing_torrent = models.Torrent.by_info_hash(info_hash)
|
||||
if existing_torrent:
|
||||
raise ValidationError('That torrent already exists (#{})'.format(existing_torrent.id))
|
||||
|
||||
# Torrent is legit, pass original filename and dict along
|
||||
field.parsed_data = TorrentFileData(filename=os.path.basename(field.data.filename),
|
||||
torrent_dict=torrent_dict,
|
||||
info_hash=info_hash,
|
||||
bencoded_info_dict=bencoded_info_dict)
|
||||
|
||||
|
||||
class TorrentFileData(object):
|
||||
"""Quick and dirty class to pass data from the validator"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
for k, v in kwargs.items():
|
||||
setattr(self, k, v)
|
||||
|
||||
# https://wiki.theory.org/BitTorrentSpecification#Metainfo_File_Structure
|
||||
|
||||
def _validate_trackers(torrent_dict):
|
||||
announce = torrent_dict.get('announce')
|
||||
_validate_bytes(announce)
|
||||
|
||||
announce_list = torrent_dict.get('announce-list')
|
||||
if announce_list is not None:
|
||||
_validate_list(announce_list, 'announce-list')
|
||||
|
||||
for announce in announce_list:
|
||||
_validate_list(announce, 'announce-list item')
|
||||
|
||||
_validate_bytes(announce[0], 'announce-list item item')
|
||||
|
||||
|
||||
def _validate_torrent_metadata(torrent_dict):
|
||||
''' Validates a torrent metadata dict, raising AssertionError on errors '''
|
||||
assert isinstance(torrent_dict, dict), 'torrent metadata is not a dict'
|
||||
|
||||
info_dict = torrent_dict.get('info')
|
||||
assert info_dict is not None, 'no info_dict in torrent'
|
||||
assert isinstance(info_dict, dict), 'info is not a dict'
|
||||
|
||||
encoding_bytes = torrent_dict.get('encoding', b'utf-8')
|
||||
encoding = _validate_bytes(encoding_bytes, 'encoding', 'utf-8').lower()
|
||||
|
||||
name = info_dict.get('name')
|
||||
_validate_bytes(name, 'name', test_decode=encoding)
|
||||
|
||||
piece_length = info_dict.get('piece length')
|
||||
_validate_number(piece_length, 'piece length', check_positive=True)
|
||||
|
||||
pieces = info_dict.get('pieces')
|
||||
_validate_bytes(pieces, 'pieces')
|
||||
assert len(pieces) % 20 == 0, 'pieces length is not a multiple of 20'
|
||||
|
||||
files = info_dict.get('files')
|
||||
if files is not None:
|
||||
_validate_list(files, 'filelist')
|
||||
|
||||
for file_dict in files:
|
||||
file_length = file_dict.get('length')
|
||||
_validate_number(file_length, 'file length', check_positive_or_zero=True)
|
||||
|
||||
path_list = file_dict.get('path')
|
||||
_validate_list(path_list, 'path')
|
||||
for path_part in path_list:
|
||||
_validate_bytes(path_part, 'path part', test_decode=encoding)
|
||||
|
||||
else:
|
||||
length = info_dict.get('length')
|
||||
_validate_number(length, 'length', check_positive=True)
|
||||
|
||||
|
||||
def _validate_bytes(value, name='value', test_decode=None):
|
||||
assert isinstance(value, bytes), name + ' is not bytes'
|
||||
assert len(value) > 0, name + ' is empty'
|
||||
if test_decode:
|
||||
try:
|
||||
return value.decode(test_decode)
|
||||
except UnicodeError:
|
||||
raise AssertionError(name + ' could not be decoded from ' + repr(test_decode))
|
||||
|
||||
|
||||
def _validate_number(value, name='value', check_positive=False, check_positive_or_zero=False):
|
||||
assert isinstance(value, int), name + ' is not an int'
|
||||
if check_positive_or_zero:
|
||||
assert value >= 0, name + ' is less than 0'
|
||||
elif check_positive:
|
||||
assert value > 0, name + ' is not positive'
|
||||
|
||||
|
||||
def _validate_list(value, name='value', check_empty=False):
|
||||
assert isinstance(value, list), name + ' is not a list'
|
||||
if check_empty:
|
||||
assert len(value) > 0, name + ' is empty'
|
||||
|
||||
|
||||
def _debug_print_torrent_metadata(torrent_dict):
|
||||
from pprint import pprint
|
||||
|
||||
# Temporarily remove 'pieces' from infodict for clean debug prints
|
||||
info_dict = torrent_dict.get('info', {})
|
||||
orig_pieces = info_dict.get('pieces')
|
||||
|
||||
info_dict['pieces'] = '<piece data>'
|
||||
pprint(torrent_dict)
|
||||
|
||||
info_dict['pieces'] = orig_pieces
|
332
nyaa/models.py
Normal file
332
nyaa/models.py
Normal file
|
@ -0,0 +1,332 @@
|
|||
from enum import Enum, IntEnum
|
||||
from datetime import datetime
|
||||
from nyaa import app, db
|
||||
from nyaa.torrents import create_magnet
|
||||
from sqlalchemy import func, ForeignKeyConstraint, Index
|
||||
from sqlalchemy_utils import ChoiceType, EmailType, PasswordType
|
||||
from werkzeug.security import generate_password_hash, check_password_hash
|
||||
from sqlalchemy_fulltext import FullText
|
||||
|
||||
if app.config['USE_MYSQL']:
|
||||
from sqlalchemy.dialects import mysql
|
||||
BinaryType = mysql.BINARY
|
||||
DescriptionTextType = mysql.TEXT
|
||||
MediumBlobType = mysql.MEDIUMBLOB
|
||||
COL_UTF8_GENERAL_CI = 'utf8_general_ci'
|
||||
COL_UTF8MB4_BIN = 'utf8mb4_bin'
|
||||
COL_ASCII_GENERAL_CI = 'ascii_general_ci'
|
||||
else:
|
||||
BinaryType = db.Binary
|
||||
DescriptionTextType = db.String
|
||||
MediumBlobType = db.BLOB
|
||||
COL_UTF8_GENERAL_CI = 'NOCASE'
|
||||
COL_UTF8MB4_BIN = None
|
||||
COL_ASCII_GENERAL_CI = 'NOCASE'
|
||||
|
||||
|
||||
class TorrentFlags(IntEnum):
|
||||
NONE = 0
|
||||
ANONYMOUS = 1
|
||||
HIDDEN = 2
|
||||
TRUSTED = 4
|
||||
REMAKE = 8
|
||||
COMPLETE = 16
|
||||
DELETED = 32
|
||||
|
||||
|
||||
class Torrent(db.Model):
|
||||
__tablename__ = app.config['TABLE_PREFIX'] + 'torrents'
|
||||
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
info_hash = db.Column(BinaryType(length=20), unique=True, nullable=False, index=True)
|
||||
display_name = db.Column(
|
||||
db.String(length=255, collation=COL_UTF8_GENERAL_CI), nullable=False, index=True)
|
||||
torrent_name = db.Column(db.String(length=255), nullable=False)
|
||||
information = db.Column(db.String(length=255), nullable=False)
|
||||
description = db.Column(DescriptionTextType(collation=COL_UTF8MB4_BIN), nullable=False)
|
||||
|
||||
filesize = db.Column(db.BIGINT, default=0, nullable=False, index=True)
|
||||
encoding = db.Column(db.String(length=32), nullable=False)
|
||||
flags = db.Column(db.Integer, default=0, nullable=False, index=True)
|
||||
uploader_id = db.Column(db.Integer, db.ForeignKey('users.id'), nullable=True)
|
||||
has_torrent = db.Column(db.Boolean, nullable=False, default=False)
|
||||
|
||||
created_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow, nullable=False)
|
||||
updated_time = db.Column(db.DateTime(timezone=False),
|
||||
default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
|
||||
|
||||
main_category_id = db.Column(db.Integer, db.ForeignKey(
|
||||
app.config['TABLE_PREFIX'] + 'main_categories.id'), nullable=False)
|
||||
sub_category_id = db.Column(db.Integer, nullable=False)
|
||||
redirect = db.Column(db.Integer, db.ForeignKey(
|
||||
app.config['TABLE_PREFIX'] + 'torrents.id'), nullable=True)
|
||||
|
||||
__table_args__ = (
|
||||
Index('uploader_flag_idx', 'uploader_id', 'flags'),
|
||||
ForeignKeyConstraint(
|
||||
['main_category_id', 'sub_category_id'],
|
||||
[app.config['TABLE_PREFIX'] + 'sub_categories.main_category_id',
|
||||
app.config['TABLE_PREFIX'] + 'sub_categories.id']
|
||||
), {}
|
||||
)
|
||||
|
||||
user = db.relationship('User', uselist=False, back_populates='torrents')
|
||||
main_category = db.relationship('MainCategory', uselist=False,
|
||||
back_populates='torrents', lazy="joined")
|
||||
sub_category = db.relationship('SubCategory', uselist=False, backref='torrents', lazy="joined",
|
||||
primaryjoin="and_(SubCategory.id == foreign(Torrent.sub_category_id), "
|
||||
"SubCategory.main_category_id == Torrent.main_category_id)")
|
||||
info = db.relationship('TorrentInfo', uselist=False, back_populates='torrent')
|
||||
filelist = db.relationship('TorrentFilelist', uselist=False, back_populates='torrent')
|
||||
stats = db.relationship('Statistic', uselist=False, back_populates='torrent', lazy='joined')
|
||||
trackers = db.relationship('TorrentTrackers', uselist=True, lazy='joined')
|
||||
|
||||
def __repr__(self):
|
||||
return '<{0} #{1.id} \'{1.display_name}\' {1.filesize}b>'.format(type(self).__name__, self)
|
||||
|
||||
@property
|
||||
def magnet_uri(self):
|
||||
return create_magnet(self)
|
||||
|
||||
@property
|
||||
def anonymous(self):
|
||||
return self.flags & TorrentFlags.ANONYMOUS
|
||||
|
||||
@anonymous.setter
|
||||
def anonymous(self, value):
|
||||
self.flags = (self.flags & ~TorrentFlags.ANONYMOUS) | (value and TorrentFlags.ANONYMOUS)
|
||||
|
||||
@property
|
||||
def hidden(self):
|
||||
return self.flags & TorrentFlags.HIDDEN
|
||||
|
||||
@hidden.setter
|
||||
def hidden(self, value):
|
||||
self.flags = (self.flags & ~TorrentFlags.HIDDEN) | (value and TorrentFlags.HIDDEN)
|
||||
|
||||
@property
|
||||
def deleted(self):
|
||||
return self.flags & TorrentFlags.DELETED
|
||||
|
||||
@deleted.setter
|
||||
def deleted(self, value):
|
||||
self.flags = (self.flags & ~TorrentFlags.DELETED) | (value and TorrentFlags.DELETED)
|
||||
|
||||
@property
|
||||
def trusted(self):
|
||||
return self.flags & TorrentFlags.TRUSTED
|
||||
|
||||
@trusted.setter
|
||||
def trusted(self, value):
|
||||
self.flags = (self.flags & ~TorrentFlags.TRUSTED) | (value and TorrentFlags.TRUSTED)
|
||||
|
||||
@property
|
||||
def remake(self):
|
||||
return self.flags & TorrentFlags.REMAKE
|
||||
|
||||
@remake.setter
|
||||
def remake(self, value):
|
||||
self.flags = (self.flags & ~TorrentFlags.REMAKE) | (value and TorrentFlags.REMAKE)
|
||||
|
||||
@property
|
||||
def complete(self):
|
||||
return self.flags & TorrentFlags.COMPLETE
|
||||
|
||||
@complete.setter
|
||||
def complete(self, value):
|
||||
self.flags = (self.flags & ~TorrentFlags.COMPLETE) | (value and TorrentFlags.COMPLETE)
|
||||
|
||||
@classmethod
|
||||
def by_id(cls, id):
|
||||
return cls.query.get(id)
|
||||
|
||||
@classmethod
|
||||
def by_info_hash(cls, info_hash):
|
||||
return cls.query.filter_by(info_hash=info_hash).first()
|
||||
|
||||
|
||||
class TorrentNameSearch(FullText, Torrent):
|
||||
__fulltext_columns__ = ('display_name',)
|
||||
|
||||
|
||||
class TorrentFilelist(db.Model):
|
||||
__tablename__ = app.config['TABLE_PREFIX'] + 'torrents_filelist'
|
||||
__table_args__ = {'mysql_row_format': 'COMPRESSED'}
|
||||
|
||||
torrent_id = db.Column(db.Integer, db.ForeignKey(
|
||||
app.config['TABLE_PREFIX'] + 'torrents.id', ondelete="CASCADE"), primary_key=True)
|
||||
filelist_blob = db.Column(MediumBlobType, nullable=True)
|
||||
|
||||
torrent = db.relationship('Torrent', uselist=False, back_populates='filelist')
|
||||
|
||||
|
||||
class TorrentInfo(db.Model):
|
||||
__tablename__ = app.config['TABLE_PREFIX'] + 'torrents_info'
|
||||
__table_args__ = {'mysql_row_format': 'COMPRESSED'}
|
||||
|
||||
torrent_id = db.Column(db.Integer, db.ForeignKey(
|
||||
app.config['TABLE_PREFIX'] + 'torrents.id', ondelete="CASCADE"), primary_key=True)
|
||||
info_dict = db.Column(MediumBlobType, nullable=True)
|
||||
|
||||
torrent = db.relationship('Torrent', uselist=False, back_populates='info')
|
||||
|
||||
|
||||
class Statistic(db.Model):
|
||||
__tablename__ = app.config['TABLE_PREFIX'] + 'statistics'
|
||||
|
||||
torrent_id = db.Column(db.Integer, db.ForeignKey(
|
||||
app.config['TABLE_PREFIX'] + 'torrents.id', ondelete="CASCADE"), primary_key=True)
|
||||
|
||||
seed_count = db.Column(db.Integer, default=0, nullable=False, index=True)
|
||||
leech_count = db.Column(db.Integer, default=0, nullable=False, index=True)
|
||||
download_count = db.Column(db.Integer, default=0, nullable=False, index=True)
|
||||
last_updated = db.Column(db.DateTime(timezone=False))
|
||||
|
||||
torrent = db.relationship('Torrent', uselist=False, back_populates='stats')
|
||||
|
||||
|
||||
class Trackers(db.Model):
|
||||
__tablename__ = 'trackers'
|
||||
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
uri = db.Column(db.String(length=255, collation=COL_UTF8_GENERAL_CI), nullable=False, unique=True)
|
||||
disabled = db.Column(db.Boolean, nullable=False, default=False)
|
||||
|
||||
@classmethod
|
||||
def by_uri(cls, uri):
|
||||
return cls.query.filter_by(uri=uri).first()
|
||||
|
||||
|
||||
class TorrentTrackers(db.Model):
|
||||
__tablename__ = app.config['TABLE_PREFIX'] + 'torrent_trackers'
|
||||
|
||||
torrent_id = db.Column(db.Integer, db.ForeignKey(app.config['TABLE_PREFIX'] + 'torrents.id', ondelete="CASCADE"), primary_key=True)
|
||||
tracker_id = db.Column(db.Integer, db.ForeignKey('trackers.id', ondelete="CASCADE"), primary_key=True)
|
||||
order = db.Column(db.Integer, nullable=False, index=True)
|
||||
|
||||
tracker = db.relationship('Trackers', uselist=False, lazy='joined')
|
||||
|
||||
@classmethod
|
||||
def by_torrent_id(cls, torrent_id):
|
||||
return cls.query.filter_by(torrent_id=torrent_id).order_by(cls.order.desc())
|
||||
|
||||
|
||||
class MainCategory(db.Model):
|
||||
__tablename__ = app.config['TABLE_PREFIX'] + 'main_categories'
|
||||
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
name = db.Column(db.String(length=64), nullable=False)
|
||||
|
||||
sub_categories = db.relationship('SubCategory', back_populates='main_category')
|
||||
torrents = db.relationship('Torrent', back_populates='main_category')
|
||||
|
||||
def get_category_ids(self):
|
||||
return (self.id, 0)
|
||||
|
||||
@property
|
||||
def id_as_string(self):
|
||||
return '_'.join(str(x) for x in self.get_category_ids())
|
||||
|
||||
@classmethod
|
||||
def by_id(cls, id):
|
||||
return cls.query.get(id)
|
||||
|
||||
|
||||
class SubCategory(db.Model):
|
||||
__tablename__ = app.config['TABLE_PREFIX'] + 'sub_categories'
|
||||
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
main_category_id = db.Column(db.Integer, db.ForeignKey(
|
||||
app.config['TABLE_PREFIX'] + 'main_categories.id'), primary_key=True)
|
||||
name = db.Column(db.String(length=64), nullable=False)
|
||||
|
||||
main_category = db.relationship('MainCategory', uselist=False, back_populates='sub_categories')
|
||||
# torrents = db.relationship('Torrent', back_populates='sub_category'),
|
||||
# primaryjoin="and_(Torrent.sub_category_id == foreign(SubCategory.id), "
|
||||