Updated api/upload

This commit is contained in:
kyamiko 2017-05-17 23:56:36 -04:00
parent 42ed463cc7
commit 785a8db0c8
4 changed files with 160 additions and 340 deletions

View File

@ -1,334 +1,92 @@
import flask
from werkzeug.datastructures import ImmutableMultiDict, CombinedMultiDict
from nyaa import app, db
from nyaa import models, forms
from nyaa import bencode, utils
from nyaa import bencode, backend, utils
from nyaa import torrents
import json
import os.path
from orderedset import OrderedSet
from werkzeug import secure_filename
#from orderedset import OrderedSet
#from werkzeug import secure_filename
DEBUG_API = False
# #################################### API ROUTES ####################################
CATEGORIES = [
('Anime', ['Anime Music Video', 'English-translated', 'Non-English-translated', 'Raw']),
('Audio', ['Lossless', 'Lossy']),
('Literature', ['English-translated', 'Non-English-translated', 'Raw']),
('Live Action', ['English-translated',
'Idol/Promotional Video', 'Non-English-translated', 'Raw']),
('Pictures', ['Graphics', 'Photos']),
('Software', ['Applications', 'Games']),
]
# #################################### API HELPERS ####################################
def validate_main_sub_cat(main_cat_name, sub_cat_name):
for main_cat in models.MainCategory.query.order_by(models.MainCategory.id):
if main_cat_name == main_cat.name:
for sub_cat in main_cat.sub_categories:
if sub_cat_name == sub_cat.name:
cat_id = main_cat.id_as_string
sub_cat_id = sub_cat.id_as_string
cat_sub_cat = sub_cat_id.split('_')
# print('cat: {0} sub_cat: {1}'.format(cat_sub_cat[0], cat_sub_cat[1]))
return True, cat_sub_cat[0], cat_sub_cat[1]
return False, 0, 0
def _replace_utf8_values(dict_or_list):
''' Will replace 'property' with 'property.utf-8' and remove latter if it exists.
Thanks, bitcomet! :/ '''
did_change = False
if isinstance(dict_or_list, dict):
for key in [key for key in dict_or_list.keys() if key.endswith('.utf-8')]:
dict_or_list[key.replace('.utf-8', '')] = dict_or_list.pop(key)
did_change = True
for value in dict_or_list.values():
did_change = _replace_utf8_values(value) or did_change
elif isinstance(dict_or_list, list):
for item in dict_or_list:
did_change = _replace_utf8_values(item) or did_change
return did_change
def validate_torrent_flags(torrent_flags):
_torrent_flags = ['hidden', 'remake', 'complete', 'anonymous']
if len(torrent_flags) != 4:
return False
for flag in torrent_flags:
if int(flag) not in [0, 1]:
return False
return True
# It might be good to factor this out of forms UploadForm because the same code is
# used in both files.
def validate_torrent_file(torrent_file_name, torrent_file):
# Decode and ensure data is bencoded data
def validate_user(upload_request):
auth_info = None
try:
torrent_dict = bencode.decode(torrent_file)
except (bencode.MalformedBencodeException, UnicodeError):
return False, 'Malformed torrent file'
if 'auth_info' in upload_request.files:
auth_info = json.loads(upload_request.files['auth_info'].read().decode('utf-8'))
if 'username' not in auth_info.keys() or 'password' not in auth_info.keys():
return False, None, None
# Uncomment for debug print of the torrent
# forms._debug_print_torrent_metadata(torrent_dict)
try:
forms._validate_torrent_metadata(torrent_dict)
except AssertionError as e:
return False, 'Malformed torrent metadata ({})'.format(e.args[0])
# Note! bencode will sort dict keys, as per the spec
# This may result in a different hash if the uploaded torrent does not match the
# spec, but it's their own fault for using broken software! Right?
bencoded_info_dict = bencode.encode(torrent_dict['info'])
info_hash = utils.sha1_hash(bencoded_info_dict)
# Check if the info_hash exists already in the database
existing_torrent = models.Torrent.by_info_hash(info_hash)
if existing_torrent:
return False, 'That torrent already exists (#{})'.format(existing_torrent.id)
# Torrent is legit, pass original filename and dict along
return True, forms.TorrentFileData(filename=os.path.basename(torrent_file_name),
torrent_dict=torrent_dict,
info_hash=info_hash,
bencoded_info_dict=bencoded_info_dict)
def api_upload(upload_request):
if upload_request.method == 'POST':
j = None
torrent_file = None
try:
if 'json' in upload_request.files:
f = upload_request.files['json']
j = json.loads(f.read().decode('utf-8'))
if DEBUG_API:
print(json.dumps(j, indent=4))
_json_keys = ['username',
'password',
'display_name',
'main_cat',
'sub_cat',
'flags'] # 'information' and 'description' are not required
# Check that required fields are present
for _k in _json_keys:
if _k not in j.keys():
return flask.make_response(flask.jsonify(
{"Error": "Missing JSON field: {0}.".format(_k)}), 400)
# Check that no extra fields are present
for k in j.keys():
if k not in set(_json_keys + ['information', 'description']):
return flask.make_response(flask.jsonify(
{"Error": "Incorrect JSON field(s)."}), 400)
else:
return flask.make_response(flask.jsonify({"Error": "No metadata."}), 400)
if 'torrent' in upload_request.files:
f = upload_request.files['torrent']
if DEBUG_API:
print(f.filename)
torrent_file = f
# print(f.read())
else:
return flask.make_response(flask.jsonify({"Error": "No torrent file."}), 400)
# 'username' and 'password' must have been provided as they are part of j.keys()
username = j['username']
password = j['password']
# Validate that the provided username and password belong to a valid user
username = auth_info['username']
password = auth_info['password']
user = models.User.by_username(username)
if not user:
user = models.User.by_email(username)
if (not user or password != user.password_hash
or user.status == models.UserStatusType.INACTIVE):
return flask.make_response(flask.jsonify(
{"Error": "Incorrect username or password"}), 403)
if (not user or password != user.password_hash or user.status == models.UserStatusType.INACTIVE):
return False, None, None
current_user = user
return True, user, None
display_name = j['display_name']
if (len(display_name) < 3) or (len(display_name) > 1024):
return flask.make_response(flask.jsonify(
{"Error": "Torrent name must be between 3 and 1024 characters."}), 400)
except Exception as e:
return False, None, e
main_cat_name = j['main_cat']
sub_cat_name = j['sub_cat']
cat_subcat_status, cat_id, sub_cat_id = validate_main_sub_cat(
main_cat_name, sub_cat_name)
if not cat_subcat_status:
return flask.make_response(flask.jsonify(
{"Error": "Incorrect Category / Sub-Category."}), 400)
def _create_upload_category_choices():
''' Turns categories in the database into a list of (id, name)s '''
choices = [('', '[Select a category]')]
for main_cat in models.MainCategory.query.order_by(models.MainCategory.id):
choices.append((main_cat.id_as_string, main_cat.name, True))
for sub_cat in main_cat.sub_categories:
choices.append((sub_cat.id_as_string, ' - ' + sub_cat.name))
return choices
# TODO Sanitize information
information = None
try:
information = j['information']
if len(information) > 255:
return flask.make_response(flask.jsonify(
{"Error": "Information is limited to 255 characters."}), 400)
except Exception as e:
information = ''
# TODO Sanitize description
description = None
try:
description = j['description']
limit = 10 * 1024
if len(description) > limit:
return flask.make_response(flask.jsonify(
{"Error": "Description is limited to {0} characters.".format(limit)}), 403)
except Exception as e:
description = ''
# #################################### API ROUTES ####################################
def api_upload(upload_request, user):
form_info = None
try:
form_info = json.loads(upload_request.files['torrent_info'].read().decode('utf-8'))
v_flags = validate_torrent_flags(j['flags'])
if v_flags:
torrent_flags = j['flags']
form_info_as_dict = []
for k, v in form_info.items():
if k in ['is_anonymous', 'is_hidden', 'is_remake', 'is_complete']:
if v == 'y':
form_info_as_dict.append((k, v))
else:
return flask.make_response(flask.jsonify(
{"Error": "Incorrect torrent flags."}), 400)
form_info_as_dict.append((k, v))
form_info = ImmutableMultiDict(form_info_as_dict)
torrent_status, torrent_data = validate_torrent_file(
torrent_file.filename, torrent_file.read()) # Needs validation
# print(repr(form_info))
except Exception as e:
return flask.make_response(flask.jsonify({"Failure": "Invalid form. See HELP in api_uploader.py"}), 400)
if not torrent_status:
return flask.make_response(flask.jsonify(
{"Error": "Invalid or Duplicate torrent file."}), 400)
try:
torrent_file = upload_request.files['torrent_file']
torrent_file = ImmutableMultiDict([('torrent_file', torrent_file)])
# The torrent has been validated and is safe to access with ['foo'] etc - all relevant
# keys and values have been checked for (see UploadForm in forms.py for details)
info_dict = torrent_data.torrent_dict['info']
# print(repr(torrent_file))
except Exception as e:
pass
changed_to_utf8 = _replace_utf8_values(torrent_data.torrent_dict)
form = forms.UploadForm(CombinedMultiDict((torrent_file, form_info)))
form.category.choices = _create_upload_category_choices()
torrent_filesize = info_dict.get('length') or sum(
f['length'] for f in info_dict.get('files'))
if upload_request.method == 'POST' and form.validate():
torrent = backend.handle_torrent_upload(form, user, True)
# In case no encoding, assume UTF-8.
torrent_encoding = torrent_data.torrent_dict.get('encoding', b'utf-8').decode('utf-8')
torrent = models.Torrent(info_hash=torrent_data.info_hash,
display_name=display_name,
torrent_name=torrent_data.filename,
information=information,
description=description,
encoding=torrent_encoding,
filesize=torrent_filesize,
user=current_user)
# Store bencoded info_dict
torrent.info = models.TorrentInfo(info_dict=torrent_data.bencoded_info_dict)
torrent.stats = models.Statistic()
torrent.has_torrent = True
# Fields with default value will be None before first commit, so set .flags
torrent.flags = 0
torrent.anonymous = True if torrent_flags[0] else False
torrent.hidden = True if torrent_flags[1] else False
torrent.remake = True if torrent_flags[2] else False
torrent.complete = True if torrent_flags[3] else False
# Copy trusted status from user if possible
torrent.trusted = (current_user.level >=
models.UserLevelType.TRUSTED) if current_user else False
# Set category ids
torrent.main_category_id = cat_id
torrent.sub_category_id = sub_cat_id
# To simplify parsing the filelist, turn single-file torrent into a list
torrent_filelist = info_dict.get('files')
used_path_encoding = changed_to_utf8 and 'utf-8' or torrent_encoding
parsed_file_tree = dict()
if not torrent_filelist:
# If single-file, the root will be the file-tree (no directory)
file_tree_root = parsed_file_tree
torrent_filelist = [{'length': torrent_filesize, 'path': [info_dict['name']]}]
else:
# If multi-file, use the directory name as root for files
file_tree_root = parsed_file_tree.setdefault(
info_dict['name'].decode(used_path_encoding), {})
# Parse file dicts into a tree
for file_dict in torrent_filelist:
# Decode path parts from utf8-bytes
path_parts = [path_part.decode(used_path_encoding)
for path_part in file_dict['path']]
filename = path_parts.pop()
current_directory = file_tree_root
for directory in path_parts:
current_directory = current_directory.setdefault(directory, {})
current_directory[filename] = file_dict['length']
parsed_file_tree = utils.sorted_pathdict(parsed_file_tree)
json_bytes = json.dumps(parsed_file_tree, separators=(',', ':')).encode('utf8')
torrent.filelist = models.TorrentFilelist(filelist_blob=json_bytes)
db.session.add(torrent)
db.session.flush()
# Store the users trackers
trackers = OrderedSet()
announce = torrent_data.torrent_dict.get('announce', b'').decode('ascii')
if announce:
trackers.add(announce)
# List of lists with single item
announce_list = torrent_data.torrent_dict.get('announce-list', [])
for announce in announce_list:
trackers.add(announce[0].decode('ascii'))
# Remove our trackers, maybe? TODO ?
# Search for/Add trackers in DB
db_trackers = OrderedSet()
for announce in trackers:
tracker = models.Trackers.by_uri(announce)
# Insert new tracker if not found
if not tracker:
tracker = models.Trackers(uri=announce)
db.session.add(tracker)
db_trackers.add(tracker)
db.session.flush()
# Store tracker refs in DB
for order, tracker in enumerate(db_trackers):
torrent_tracker = models.TorrentTrackers(torrent_id=torrent.id,
tracker_id=tracker.id, order=order)
db.session.add(torrent_tracker)
db.session.commit()
if app.config.get('BACKUP_TORRENT_FOLDER'):
torrent_file.seek(0, 0)
torrent_path = os.path.join(app.config['BACKUP_TORRENT_FOLDER'], '{}.{}'.format(
torrent.id, secure_filename(torrent_file.filename)))
torrent_file.save(torrent_path)
torrent_file.close()
# print('Success? {0}'.format(torrent.id))
return flask.make_response(flask.jsonify(
{"Success": "Request was processed {0}".format(torrent.id)}), 200)
except Exception as e:
print('Exception: {0}'.format(e))
return flask.make_response(flask.jsonify(
{"Error": "Incorrect JSON. Please see HELP page for examples."}), 400)
return flask.make_response(flask.jsonify({"Success": "Request was processed {0}".format(torrent.id)}), 200)
else:
return flask.make_response(flask.jsonify({"Error": "Bad request"}), 400)
# print(form.errors)
return_error_messages = []
for error_name, error_messages in form.errors.items():
# print(error_messages)
return_error_messages.extend(error_messages)
return flask.make_response(flask.jsonify({"Failure": return_error_messages}), 400)

View File

@ -26,7 +26,7 @@ def _replace_utf8_values(dict_or_list):
return did_change
def handle_torrent_upload(upload_form, uploading_user=None):
def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False):
torrent_data = upload_form.torrent_file.parsed_data
# The torrent has been validated and is safe to access with ['foo'] etc - all relevant
@ -70,12 +70,8 @@ def handle_torrent_upload(upload_form, uploading_user=None):
# Copy trusted status from user if possible
torrent.trusted = (uploading_user.level >=
models.UserLevelType.TRUSTED) if uploading_user else False
# Set category ids
torrent.main_category_id, torrent.sub_category_id = \
upload_form.category.parsed_data.get_category_ids()
# print('Main cat id: {0}, Sub cat id: {1}'.format(
# torrent.main_category_id, torrent.sub_category_id))
torrent.main_category_id, torrent.sub_category_id = upload_form.category.parsed_data.get_category_ids()
# To simplify parsing the filelist, turn single-file torrent into a list
torrent_filelist = info_dict.get('files')

View File

@ -346,7 +346,7 @@ def render_rss(label, query, use_elastic):
response = flask.make_response(rss_xml)
response.headers['Content-Type'] = 'application/xml'
# Cache for an hour
response.headers['Cache-Control'] = 'max-age={}'.format(1*5*60)
response.headers['Cache-Control'] = 'max-age={}'.format(1 * 5 * 60)
return response
@ -515,6 +515,7 @@ def _create_upload_category_choices():
@app.route('/upload', methods=['GET', 'POST'])
def upload():
form = forms.UploadForm(CombinedMultiDict((flask.request.files, flask.request.form)))
print('{0} - {1}'.format(flask.request.files, flask.request.form))
form.category.choices = _create_upload_category_choices()
if flask.request.method == 'POST' and form.validate():
torrent = backend.handle_torrent_upload(form, flask.g.user)
@ -696,8 +697,10 @@ def site_help():
# #################################### API ROUTES ####################################
# DISABLED FOR NOW
@app.route('/api/upload', methods=['POST'])
def api_upload():
api_response = api_handler.api_upload(flask.request)
is_valid_user, user, debug = api_handler.validate_user(flask.request)
if not is_valid_user:
return flask.make_response(flask.jsonify({"Failure": "Invalid username or password."}), 400)
api_response = api_handler.api_upload(flask.request, user)
return api_response

View File

@ -1,52 +1,115 @@
# api_uploader.py
# Uploads a single file
# I will create another script for batch uploading
# Uploads a single torrent file
# Works on nyaa.si
# An updated version will work on sukebei.nyaa.si
import json
# pip install requests
# http://docs.python-requests.org/en/master/user/install/
import requests
url = "http://127.0.0.1:5500/api/upload"
#url = "http://127.0.0.1:5500/api/upload"
url = "https://nyaa.si/api/upload"
# Required for Auth
# ########################## REQUIRED: YOUR USERNAME AND PASSWORD ##############################
username = ""
password = ""
# Required
torrent_name = ""
# ########################################### HELP ############################################
# ################################# CATEGORIES MUST BE EXACT ##################################
"""
Anime
Anime - AMV : "1_1"
Anime - English : "1_2"
Anime - Non-English : "1_3"
Anime - Raw : "1_4"
Audio
Lossless : "2_1"
Lossy : "2_2"
Literature
Literature - English-translated : "3_1"
Literature - Non-English : "3_2"
Literature - Non-English-Translated : "3_3"
Literature - Raw : "3_4"
Live Action
Live Action - English-translated : "4_1"
Live Action - Idol/Promotional Video : "4_2"
Live Action - Non-English-translated : "4_3"
Live Action - Raw : "4_4"
Pictures
Pictures - Graphics : "5_1"
Pictures - Photos : "5_2"
Software
Software - Applications : "6_1"
Software - Games : "6_2"
"""
# ################################# CATEGORIES MUST BE EXACT ##################################
# ###################################### EXAMPLE REQUEST ######################################
"""
# Required
main_cat = ""
file_name = "/path/to/my_file.torrent"
# Required
sub_cat = ""
category = "6_1"
# Required
display_name = "API upload example"
# May be blank
information = "API HOWTO"
# May be blank
description = "Visit #nyaa-dev@irc.rizon.net"
# Default is 'n' No
# Change to 'y' Yes to set
is_anonymous : 'n',
is_hidden : 'n',
is_remake : 'n',
is_complete : 'n'
"""
# #############################################################################################
# ######################################## CHANGE HERE ########################################
# Required
file_name = ""
# Required
category = ""
# Required
display_name = ""
# May be blank
information = ""
# May be blank
description = ""
# flags = [Hidden, Remake, Complete, Anonymous]
# 0 for NOT SET / 1 for SET
# Required
flags = [0, 0, 0, 0]
# Default is 'n' No
# Change to 'y' Yes to set
is_anonymous = 'n'
is_hidden = 'n'
is_remake = 'n'
is_complete = 'n'
# #############################################################################################
# #################################### DO NOT CHANGE BELOW ####################################
# ############################ UNLESS YOU KNOW WHAT YOU ARE DOING #############################
auth_info = {
"username" : username,
"password" : password
}
metadata={
"username": username,
"password": password,
"display_name": torrent_name,
"main_cat": main_cat,
"sub_cat": sub_cat,
"information": information,
"description": description,
"flags": flags
}
# Required
file_name = ""
"category" : category,
"display_name" : display_name,
"information" : information,
"description" : description,
"is_anonymous" : is_anonymous,
"is_hidden" : is_hidden,
"is_remake" : is_remake,
"is_complete" : is_complete
}
files = {
'json': (json.dumps(metadata)),
'torrent': ('{0}'.format(file_name), open(file_name, 'rb'), 'application/octet-stream')}
'auth_info' : (json.dumps(auth_info)),
'torrent_info' : (json.dumps(metadata)),
'torrent_file' : ('{0}'.format(file_name), open(file_name, 'rb'), 'application/octet-stream')
}
response = requests.post(url, files=files)