Updated api/upload

This commit is contained in:
kyamiko 2017-05-17 23:56:36 -04:00
parent 42ed463cc7
commit 785a8db0c8
4 changed files with 160 additions and 340 deletions

View File

@ -1,334 +1,92 @@
import flask import flask
from werkzeug.datastructures import ImmutableMultiDict, CombinedMultiDict
from nyaa import app, db from nyaa import app, db
from nyaa import models, forms from nyaa import models, forms
from nyaa import bencode, utils from nyaa import bencode, backend, utils
from nyaa import torrents from nyaa import torrents
import json import json
import os.path import os.path
from orderedset import OrderedSet #from orderedset import OrderedSet
from werkzeug import secure_filename #from werkzeug import secure_filename
DEBUG_API = False # #################################### API HELPERS ####################################
# #################################### API ROUTES ####################################
CATEGORIES = [
('Anime', ['Anime Music Video', 'English-translated', 'Non-English-translated', 'Raw']),
('Audio', ['Lossless', 'Lossy']),
('Literature', ['English-translated', 'Non-English-translated', 'Raw']),
('Live Action', ['English-translated',
'Idol/Promotional Video', 'Non-English-translated', 'Raw']),
('Pictures', ['Graphics', 'Photos']),
('Software', ['Applications', 'Games']),
]
def validate_main_sub_cat(main_cat_name, sub_cat_name): def validate_user(upload_request):
for main_cat in models.MainCategory.query.order_by(models.MainCategory.id): auth_info = None
if main_cat_name == main_cat.name:
for sub_cat in main_cat.sub_categories:
if sub_cat_name == sub_cat.name:
cat_id = main_cat.id_as_string
sub_cat_id = sub_cat.id_as_string
cat_sub_cat = sub_cat_id.split('_')
# print('cat: {0} sub_cat: {1}'.format(cat_sub_cat[0], cat_sub_cat[1]))
return True, cat_sub_cat[0], cat_sub_cat[1]
return False, 0, 0
def _replace_utf8_values(dict_or_list):
''' Will replace 'property' with 'property.utf-8' and remove latter if it exists.
Thanks, bitcomet! :/ '''
did_change = False
if isinstance(dict_or_list, dict):
for key in [key for key in dict_or_list.keys() if key.endswith('.utf-8')]:
dict_or_list[key.replace('.utf-8', '')] = dict_or_list.pop(key)
did_change = True
for value in dict_or_list.values():
did_change = _replace_utf8_values(value) or did_change
elif isinstance(dict_or_list, list):
for item in dict_or_list:
did_change = _replace_utf8_values(item) or did_change
return did_change
def validate_torrent_flags(torrent_flags):
_torrent_flags = ['hidden', 'remake', 'complete', 'anonymous']
if len(torrent_flags) != 4:
return False
for flag in torrent_flags:
if int(flag) not in [0, 1]:
return False
return True
# It might be good to factor this out of forms UploadForm because the same code is
# used in both files.
def validate_torrent_file(torrent_file_name, torrent_file):
# Decode and ensure data is bencoded data
try: try:
torrent_dict = bencode.decode(torrent_file) if 'auth_info' in upload_request.files:
except (bencode.MalformedBencodeException, UnicodeError): auth_info = json.loads(upload_request.files['auth_info'].read().decode('utf-8'))
return False, 'Malformed torrent file' if 'username' not in auth_info.keys() or 'password' not in auth_info.keys():
return False, None, None
# Uncomment for debug print of the torrent username = auth_info['username']
# forms._debug_print_torrent_metadata(torrent_dict) password = auth_info['password']
try:
forms._validate_torrent_metadata(torrent_dict)
except AssertionError as e:
return False, 'Malformed torrent metadata ({})'.format(e.args[0])
# Note! bencode will sort dict keys, as per the spec
# This may result in a different hash if the uploaded torrent does not match the
# spec, but it's their own fault for using broken software! Right?
bencoded_info_dict = bencode.encode(torrent_dict['info'])
info_hash = utils.sha1_hash(bencoded_info_dict)
# Check if the info_hash exists already in the database
existing_torrent = models.Torrent.by_info_hash(info_hash)
if existing_torrent:
return False, 'That torrent already exists (#{})'.format(existing_torrent.id)
# Torrent is legit, pass original filename and dict along
return True, forms.TorrentFileData(filename=os.path.basename(torrent_file_name),
torrent_dict=torrent_dict,
info_hash=info_hash,
bencoded_info_dict=bencoded_info_dict)
def api_upload(upload_request):
if upload_request.method == 'POST':
j = None
torrent_file = None
try:
if 'json' in upload_request.files:
f = upload_request.files['json']
j = json.loads(f.read().decode('utf-8'))
if DEBUG_API:
print(json.dumps(j, indent=4))
_json_keys = ['username',
'password',
'display_name',
'main_cat',
'sub_cat',
'flags'] # 'information' and 'description' are not required
# Check that required fields are present
for _k in _json_keys:
if _k not in j.keys():
return flask.make_response(flask.jsonify(
{"Error": "Missing JSON field: {0}.".format(_k)}), 400)
# Check that no extra fields are present
for k in j.keys():
if k not in set(_json_keys + ['information', 'description']):
return flask.make_response(flask.jsonify(
{"Error": "Incorrect JSON field(s)."}), 400)
else:
return flask.make_response(flask.jsonify({"Error": "No metadata."}), 400)
if 'torrent' in upload_request.files:
f = upload_request.files['torrent']
if DEBUG_API:
print(f.filename)
torrent_file = f
# print(f.read())
else:
return flask.make_response(flask.jsonify({"Error": "No torrent file."}), 400)
# 'username' and 'password' must have been provided as they are part of j.keys()
username = j['username']
password = j['password']
# Validate that the provided username and password belong to a valid user
user = models.User.by_username(username) user = models.User.by_username(username)
if not user: if not user:
user = models.User.by_email(username) user = models.User.by_email(username)
if (not user or password != user.password_hash if (not user or password != user.password_hash or user.status == models.UserStatusType.INACTIVE):
or user.status == models.UserStatusType.INACTIVE): return False, None, None
return flask.make_response(flask.jsonify(
{"Error": "Incorrect username or password"}), 403)
current_user = user return True, user, None
display_name = j['display_name'] except Exception as e:
if (len(display_name) < 3) or (len(display_name) > 1024): return False, None, e
return flask.make_response(flask.jsonify(
{"Error": "Torrent name must be between 3 and 1024 characters."}), 400)
main_cat_name = j['main_cat']
sub_cat_name = j['sub_cat']
cat_subcat_status, cat_id, sub_cat_id = validate_main_sub_cat( def _create_upload_category_choices():
main_cat_name, sub_cat_name) ''' Turns categories in the database into a list of (id, name)s '''
if not cat_subcat_status: choices = [('', '[Select a category]')]
return flask.make_response(flask.jsonify( for main_cat in models.MainCategory.query.order_by(models.MainCategory.id):
{"Error": "Incorrect Category / Sub-Category."}), 400) choices.append((main_cat.id_as_string, main_cat.name, True))
for sub_cat in main_cat.sub_categories:
choices.append((sub_cat.id_as_string, ' - ' + sub_cat.name))
return choices
# TODO Sanitize information
information = None
try:
information = j['information']
if len(information) > 255:
return flask.make_response(flask.jsonify(
{"Error": "Information is limited to 255 characters."}), 400)
except Exception as e:
information = ''
# TODO Sanitize description # #################################### API ROUTES ####################################
description = None def api_upload(upload_request, user):
try: form_info = None
description = j['description'] try:
limit = 10 * 1024 form_info = json.loads(upload_request.files['torrent_info'].read().decode('utf-8'))
if len(description) > limit:
return flask.make_response(flask.jsonify(
{"Error": "Description is limited to {0} characters.".format(limit)}), 403)
except Exception as e:
description = ''
v_flags = validate_torrent_flags(j['flags']) form_info_as_dict = []
if v_flags: for k, v in form_info.items():
torrent_flags = j['flags'] if k in ['is_anonymous', 'is_hidden', 'is_remake', 'is_complete']:
if v == 'y':
form_info_as_dict.append((k, v))
else: else:
return flask.make_response(flask.jsonify( form_info_as_dict.append((k, v))
{"Error": "Incorrect torrent flags."}), 400) form_info = ImmutableMultiDict(form_info_as_dict)
torrent_status, torrent_data = validate_torrent_file( # print(repr(form_info))
torrent_file.filename, torrent_file.read()) # Needs validation except Exception as e:
return flask.make_response(flask.jsonify({"Failure": "Invalid form. See HELP in api_uploader.py"}), 400)
if not torrent_status: try:
return flask.make_response(flask.jsonify( torrent_file = upload_request.files['torrent_file']
{"Error": "Invalid or Duplicate torrent file."}), 400) torrent_file = ImmutableMultiDict([('torrent_file', torrent_file)])
# The torrent has been validated and is safe to access with ['foo'] etc - all relevant # print(repr(torrent_file))
# keys and values have been checked for (see UploadForm in forms.py for details) except Exception as e:
info_dict = torrent_data.torrent_dict['info'] pass
changed_to_utf8 = _replace_utf8_values(torrent_data.torrent_dict) form = forms.UploadForm(CombinedMultiDict((torrent_file, form_info)))
form.category.choices = _create_upload_category_choices()
torrent_filesize = info_dict.get('length') or sum( if upload_request.method == 'POST' and form.validate():
f['length'] for f in info_dict.get('files')) torrent = backend.handle_torrent_upload(form, user, True)
# In case no encoding, assume UTF-8. return flask.make_response(flask.jsonify({"Success": "Request was processed {0}".format(torrent.id)}), 200)
torrent_encoding = torrent_data.torrent_dict.get('encoding', b'utf-8').decode('utf-8')
torrent = models.Torrent(info_hash=torrent_data.info_hash,
display_name=display_name,
torrent_name=torrent_data.filename,
information=information,
description=description,
encoding=torrent_encoding,
filesize=torrent_filesize,
user=current_user)
# Store bencoded info_dict
torrent.info = models.TorrentInfo(info_dict=torrent_data.bencoded_info_dict)
torrent.stats = models.Statistic()
torrent.has_torrent = True
# Fields with default value will be None before first commit, so set .flags
torrent.flags = 0
torrent.anonymous = True if torrent_flags[0] else False
torrent.hidden = True if torrent_flags[1] else False
torrent.remake = True if torrent_flags[2] else False
torrent.complete = True if torrent_flags[3] else False
# Copy trusted status from user if possible
torrent.trusted = (current_user.level >=
models.UserLevelType.TRUSTED) if current_user else False
# Set category ids
torrent.main_category_id = cat_id
torrent.sub_category_id = sub_cat_id
# To simplify parsing the filelist, turn single-file torrent into a list
torrent_filelist = info_dict.get('files')
used_path_encoding = changed_to_utf8 and 'utf-8' or torrent_encoding
parsed_file_tree = dict()
if not torrent_filelist:
# If single-file, the root will be the file-tree (no directory)
file_tree_root = parsed_file_tree
torrent_filelist = [{'length': torrent_filesize, 'path': [info_dict['name']]}]
else:
# If multi-file, use the directory name as root for files
file_tree_root = parsed_file_tree.setdefault(
info_dict['name'].decode(used_path_encoding), {})
# Parse file dicts into a tree
for file_dict in torrent_filelist:
# Decode path parts from utf8-bytes
path_parts = [path_part.decode(used_path_encoding)
for path_part in file_dict['path']]
filename = path_parts.pop()
current_directory = file_tree_root
for directory in path_parts:
current_directory = current_directory.setdefault(directory, {})
current_directory[filename] = file_dict['length']
parsed_file_tree = utils.sorted_pathdict(parsed_file_tree)
json_bytes = json.dumps(parsed_file_tree, separators=(',', ':')).encode('utf8')
torrent.filelist = models.TorrentFilelist(filelist_blob=json_bytes)
db.session.add(torrent)
db.session.flush()
# Store the users trackers
trackers = OrderedSet()
announce = torrent_data.torrent_dict.get('announce', b'').decode('ascii')
if announce:
trackers.add(announce)
# List of lists with single item
announce_list = torrent_data.torrent_dict.get('announce-list', [])
for announce in announce_list:
trackers.add(announce[0].decode('ascii'))
# Remove our trackers, maybe? TODO ?
# Search for/Add trackers in DB
db_trackers = OrderedSet()
for announce in trackers:
tracker = models.Trackers.by_uri(announce)
# Insert new tracker if not found
if not tracker:
tracker = models.Trackers(uri=announce)
db.session.add(tracker)
db_trackers.add(tracker)
db.session.flush()
# Store tracker refs in DB
for order, tracker in enumerate(db_trackers):
torrent_tracker = models.TorrentTrackers(torrent_id=torrent.id,
tracker_id=tracker.id, order=order)
db.session.add(torrent_tracker)
db.session.commit()
if app.config.get('BACKUP_TORRENT_FOLDER'):
torrent_file.seek(0, 0)
torrent_path = os.path.join(app.config['BACKUP_TORRENT_FOLDER'], '{}.{}'.format(
torrent.id, secure_filename(torrent_file.filename)))
torrent_file.save(torrent_path)
torrent_file.close()
# print('Success? {0}'.format(torrent.id))
return flask.make_response(flask.jsonify(
{"Success": "Request was processed {0}".format(torrent.id)}), 200)
except Exception as e:
print('Exception: {0}'.format(e))
return flask.make_response(flask.jsonify(
{"Error": "Incorrect JSON. Please see HELP page for examples."}), 400)
else: else:
return flask.make_response(flask.jsonify({"Error": "Bad request"}), 400) # print(form.errors)
return_error_messages = []
for error_name, error_messages in form.errors.items():
# print(error_messages)
return_error_messages.extend(error_messages)
return flask.make_response(flask.jsonify({"Failure": return_error_messages}), 400)

View File

@ -26,7 +26,7 @@ def _replace_utf8_values(dict_or_list):
return did_change return did_change
def handle_torrent_upload(upload_form, uploading_user=None): def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False):
torrent_data = upload_form.torrent_file.parsed_data torrent_data = upload_form.torrent_file.parsed_data
# The torrent has been validated and is safe to access with ['foo'] etc - all relevant # The torrent has been validated and is safe to access with ['foo'] etc - all relevant
@ -70,12 +70,8 @@ def handle_torrent_upload(upload_form, uploading_user=None):
# Copy trusted status from user if possible # Copy trusted status from user if possible
torrent.trusted = (uploading_user.level >= torrent.trusted = (uploading_user.level >=
models.UserLevelType.TRUSTED) if uploading_user else False models.UserLevelType.TRUSTED) if uploading_user else False
# Set category ids # Set category ids
torrent.main_category_id, torrent.sub_category_id = \ torrent.main_category_id, torrent.sub_category_id = upload_form.category.parsed_data.get_category_ids()
upload_form.category.parsed_data.get_category_ids()
# print('Main cat id: {0}, Sub cat id: {1}'.format(
# torrent.main_category_id, torrent.sub_category_id))
# To simplify parsing the filelist, turn single-file torrent into a list # To simplify parsing the filelist, turn single-file torrent into a list
torrent_filelist = info_dict.get('files') torrent_filelist = info_dict.get('files')

View File

@ -346,7 +346,7 @@ def render_rss(label, query, use_elastic):
response = flask.make_response(rss_xml) response = flask.make_response(rss_xml)
response.headers['Content-Type'] = 'application/xml' response.headers['Content-Type'] = 'application/xml'
# Cache for an hour # Cache for an hour
response.headers['Cache-Control'] = 'max-age={}'.format(1*5*60) response.headers['Cache-Control'] = 'max-age={}'.format(1 * 5 * 60)
return response return response
@ -515,6 +515,7 @@ def _create_upload_category_choices():
@app.route('/upload', methods=['GET', 'POST']) @app.route('/upload', methods=['GET', 'POST'])
def upload(): def upload():
form = forms.UploadForm(CombinedMultiDict((flask.request.files, flask.request.form))) form = forms.UploadForm(CombinedMultiDict((flask.request.files, flask.request.form)))
print('{0} - {1}'.format(flask.request.files, flask.request.form))
form.category.choices = _create_upload_category_choices() form.category.choices = _create_upload_category_choices()
if flask.request.method == 'POST' and form.validate(): if flask.request.method == 'POST' and form.validate():
torrent = backend.handle_torrent_upload(form, flask.g.user) torrent = backend.handle_torrent_upload(form, flask.g.user)
@ -696,8 +697,10 @@ def site_help():
# #################################### API ROUTES #################################### # #################################### API ROUTES ####################################
# DISABLED FOR NOW
@app.route('/api/upload', methods=['POST']) @app.route('/api/upload', methods=['POST'])
def api_upload(): def api_upload():
api_response = api_handler.api_upload(flask.request) is_valid_user, user, debug = api_handler.validate_user(flask.request)
if not is_valid_user:
return flask.make_response(flask.jsonify({"Failure": "Invalid username or password."}), 400)
api_response = api_handler.api_upload(flask.request, user)
return api_response return api_response

View File

@ -1,52 +1,115 @@
# api_uploader.py # Uploads a single torrent file
# Works on nyaa.si
# An updated version will work on sukebei.nyaa.si
# Uploads a single file
# I will create another script for batch uploading
import json import json
# pip install requests
# http://docs.python-requests.org/en/master/user/install/
import requests import requests
url = "http://127.0.0.1:5500/api/upload" #url = "http://127.0.0.1:5500/api/upload"
url = "https://nyaa.si/api/upload"
# Required for Auth # ########################## REQUIRED: YOUR USERNAME AND PASSWORD ##############################
username = "" username = ""
password = "" password = ""
# Required # ########################################### HELP ############################################
torrent_name = ""
# ################################# CATEGORIES MUST BE EXACT ##################################
"""
Anime
Anime - AMV : "1_1"
Anime - English : "1_2"
Anime - Non-English : "1_3"
Anime - Raw : "1_4"
Audio
Lossless : "2_1"
Lossy : "2_2"
Literature
Literature - English-translated : "3_1"
Literature - Non-English : "3_2"
Literature - Non-English-Translated : "3_3"
Literature - Raw : "3_4"
Live Action
Live Action - English-translated : "4_1"
Live Action - Idol/Promotional Video : "4_2"
Live Action - Non-English-translated : "4_3"
Live Action - Raw : "4_4"
Pictures
Pictures - Graphics : "5_1"
Pictures - Photos : "5_2"
Software
Software - Applications : "6_1"
Software - Games : "6_2"
"""
# ################################# CATEGORIES MUST BE EXACT ##################################
# ###################################### EXAMPLE REQUEST ######################################
"""
# Required # Required
main_cat = "" file_name = "/path/to/my_file.torrent"
# Required # Required
sub_cat = "" category = "6_1"
# Required
display_name = "API upload example"
# May be blank
information = "API HOWTO"
# May be blank
description = "Visit #nyaa-dev@irc.rizon.net"
# Default is 'n' No
# Change to 'y' Yes to set
is_anonymous : 'n',
is_hidden : 'n',
is_remake : 'n',
is_complete : 'n'
"""
# #############################################################################################
# ######################################## CHANGE HERE ########################################
# Required
file_name = ""
# Required
category = ""
# Required
display_name = ""
# May be blank # May be blank
information = "" information = ""
# May be blank # May be blank
description = "" description = ""
# flags = [Hidden, Remake, Complete, Anonymous] # Default is 'n' No
# 0 for NOT SET / 1 for SET # Change to 'y' Yes to set
# Required is_anonymous = 'n'
flags = [0, 0, 0, 0] is_hidden = 'n'
is_remake = 'n'
is_complete = 'n'
# #############################################################################################
# #################################### DO NOT CHANGE BELOW ####################################
# ############################ UNLESS YOU KNOW WHAT YOU ARE DOING #############################
auth_info = {
"username" : username,
"password" : password
}
metadata={ metadata={
"username": username, "category" : category,
"password": password, "display_name" : display_name,
"display_name": torrent_name, "information" : information,
"main_cat": main_cat, "description" : description,
"sub_cat": sub_cat, "is_anonymous" : is_anonymous,
"information": information, "is_hidden" : is_hidden,
"description": description, "is_remake" : is_remake,
"flags": flags "is_complete" : is_complete
} }
# Required
file_name = ""
files = { files = {
'json': (json.dumps(metadata)), 'auth_info' : (json.dumps(auth_info)),
'torrent': ('{0}'.format(file_name), open(file_name, 'rb'), 'application/octet-stream')} 'torrent_info' : (json.dumps(metadata)),
'torrent_file' : ('{0}'.format(file_name), open(file_name, 'rb'), 'application/octet-stream')
}
response = requests.post(url, files=files) response = requests.post(url, files=files)