diff --git a/nyaa/api_handler.py b/nyaa/api_handler.py index 2ce1d70..1c29897 100644 --- a/nyaa/api_handler.py +++ b/nyaa/api_handler.py @@ -1,334 +1,92 @@ import flask +from werkzeug.datastructures import ImmutableMultiDict, CombinedMultiDict + from nyaa import app, db from nyaa import models, forms -from nyaa import bencode, utils +from nyaa import bencode, backend, utils from nyaa import torrents import json import os.path -from orderedset import OrderedSet -from werkzeug import secure_filename +#from orderedset import OrderedSet +#from werkzeug import secure_filename -DEBUG_API = False -# #################################### API ROUTES #################################### -CATEGORIES = [ - ('Anime', ['Anime Music Video', 'English-translated', 'Non-English-translated', 'Raw']), - ('Audio', ['Lossless', 'Lossy']), - ('Literature', ['English-translated', 'Non-English-translated', 'Raw']), - ('Live Action', ['English-translated', - 'Idol/Promotional Video', 'Non-English-translated', 'Raw']), - ('Pictures', ['Graphics', 'Photos']), - ('Software', ['Applications', 'Games']), -] +# #################################### API HELPERS #################################### -def validate_main_sub_cat(main_cat_name, sub_cat_name): - for main_cat in models.MainCategory.query.order_by(models.MainCategory.id): - if main_cat_name == main_cat.name: - for sub_cat in main_cat.sub_categories: - if sub_cat_name == sub_cat.name: - cat_id = main_cat.id_as_string - sub_cat_id = sub_cat.id_as_string - cat_sub_cat = sub_cat_id.split('_') - # print('cat: {0} sub_cat: {1}'.format(cat_sub_cat[0], cat_sub_cat[1])) - - return True, cat_sub_cat[0], cat_sub_cat[1] - - return False, 0, 0 - - -def _replace_utf8_values(dict_or_list): - ''' Will replace 'property' with 'property.utf-8' and remove latter if it exists. - Thanks, bitcomet! :/ ''' - did_change = False - if isinstance(dict_or_list, dict): - for key in [key for key in dict_or_list.keys() if key.endswith('.utf-8')]: - dict_or_list[key.replace('.utf-8', '')] = dict_or_list.pop(key) - did_change = True - for value in dict_or_list.values(): - did_change = _replace_utf8_values(value) or did_change - elif isinstance(dict_or_list, list): - for item in dict_or_list: - did_change = _replace_utf8_values(item) or did_change - return did_change - - -def validate_torrent_flags(torrent_flags): - _torrent_flags = ['hidden', 'remake', 'complete', 'anonymous'] - - if len(torrent_flags) != 4: - return False - - for flag in torrent_flags: - if int(flag) not in [0, 1]: - return False - return True - -# It might be good to factor this out of forms UploadForm because the same code is -# used in both files. - - -def validate_torrent_file(torrent_file_name, torrent_file): - # Decode and ensure data is bencoded data +def validate_user(upload_request): + auth_info = None try: - torrent_dict = bencode.decode(torrent_file) - except (bencode.MalformedBencodeException, UnicodeError): - return False, 'Malformed torrent file' + if 'auth_info' in upload_request.files: + auth_info = json.loads(upload_request.files['auth_info'].read().decode('utf-8')) + if 'username' not in auth_info.keys() or 'password' not in auth_info.keys(): + return False, None, None - # Uncomment for debug print of the torrent - # forms._debug_print_torrent_metadata(torrent_dict) - - try: - forms._validate_torrent_metadata(torrent_dict) - except AssertionError as e: - return False, 'Malformed torrent metadata ({})'.format(e.args[0]) - - # Note! bencode will sort dict keys, as per the spec - # This may result in a different hash if the uploaded torrent does not match the - # spec, but it's their own fault for using broken software! Right? - bencoded_info_dict = bencode.encode(torrent_dict['info']) - info_hash = utils.sha1_hash(bencoded_info_dict) - - # Check if the info_hash exists already in the database - existing_torrent = models.Torrent.by_info_hash(info_hash) - if existing_torrent: - return False, 'That torrent already exists (#{})'.format(existing_torrent.id) - - # Torrent is legit, pass original filename and dict along - return True, forms.TorrentFileData(filename=os.path.basename(torrent_file_name), - torrent_dict=torrent_dict, - info_hash=info_hash, - bencoded_info_dict=bencoded_info_dict) - - -def api_upload(upload_request): - if upload_request.method == 'POST': - j = None - torrent_file = None - try: - if 'json' in upload_request.files: - f = upload_request.files['json'] - j = json.loads(f.read().decode('utf-8')) - if DEBUG_API: - print(json.dumps(j, indent=4)) - - _json_keys = ['username', - 'password', - 'display_name', - 'main_cat', - 'sub_cat', - 'flags'] # 'information' and 'description' are not required - # Check that required fields are present - for _k in _json_keys: - if _k not in j.keys(): - return flask.make_response(flask.jsonify( - {"Error": "Missing JSON field: {0}.".format(_k)}), 400) - # Check that no extra fields are present - for k in j.keys(): - if k not in set(_json_keys + ['information', 'description']): - return flask.make_response(flask.jsonify( - {"Error": "Incorrect JSON field(s)."}), 400) - else: - return flask.make_response(flask.jsonify({"Error": "No metadata."}), 400) - if 'torrent' in upload_request.files: - f = upload_request.files['torrent'] - if DEBUG_API: - print(f.filename) - torrent_file = f - # print(f.read()) - else: - return flask.make_response(flask.jsonify({"Error": "No torrent file."}), 400) - - # 'username' and 'password' must have been provided as they are part of j.keys() - username = j['username'] - password = j['password'] - # Validate that the provided username and password belong to a valid user + username = auth_info['username'] + password = auth_info['password'] user = models.User.by_username(username) if not user: user = models.User.by_email(username) - if (not user or password != user.password_hash - or user.status == models.UserStatusType.INACTIVE): - return flask.make_response(flask.jsonify( - {"Error": "Incorrect username or password"}), 403) + if (not user or password != user.password_hash or user.status == models.UserStatusType.INACTIVE): + return False, None, None - current_user = user + return True, user, None - display_name = j['display_name'] - if (len(display_name) < 3) or (len(display_name) > 1024): - return flask.make_response(flask.jsonify( - {"Error": "Torrent name must be between 3 and 1024 characters."}), 400) + except Exception as e: + return False, None, e - main_cat_name = j['main_cat'] - sub_cat_name = j['sub_cat'] - cat_subcat_status, cat_id, sub_cat_id = validate_main_sub_cat( - main_cat_name, sub_cat_name) - if not cat_subcat_status: - return flask.make_response(flask.jsonify( - {"Error": "Incorrect Category / Sub-Category."}), 400) +def _create_upload_category_choices(): + ''' Turns categories in the database into a list of (id, name)s ''' + choices = [('', '[Select a category]')] + for main_cat in models.MainCategory.query.order_by(models.MainCategory.id): + choices.append((main_cat.id_as_string, main_cat.name, True)) + for sub_cat in main_cat.sub_categories: + choices.append((sub_cat.id_as_string, ' - ' + sub_cat.name)) + return choices - # TODO Sanitize information - information = None - try: - information = j['information'] - if len(information) > 255: - return flask.make_response(flask.jsonify( - {"Error": "Information is limited to 255 characters."}), 400) - except Exception as e: - information = '' - # TODO Sanitize description - description = None - try: - description = j['description'] - limit = 10 * 1024 - if len(description) > limit: - return flask.make_response(flask.jsonify( - {"Error": "Description is limited to {0} characters.".format(limit)}), 403) - except Exception as e: - description = '' +# #################################### API ROUTES #################################### +def api_upload(upload_request, user): + form_info = None + try: + form_info = json.loads(upload_request.files['torrent_info'].read().decode('utf-8')) - v_flags = validate_torrent_flags(j['flags']) - if v_flags: - torrent_flags = j['flags'] + form_info_as_dict = [] + for k, v in form_info.items(): + if k in ['is_anonymous', 'is_hidden', 'is_remake', 'is_complete']: + if v == 'y': + form_info_as_dict.append((k, v)) else: - return flask.make_response(flask.jsonify( - {"Error": "Incorrect torrent flags."}), 400) + form_info_as_dict.append((k, v)) + form_info = ImmutableMultiDict(form_info_as_dict) - torrent_status, torrent_data = validate_torrent_file( - torrent_file.filename, torrent_file.read()) # Needs validation + # print(repr(form_info)) + except Exception as e: + return flask.make_response(flask.jsonify({"Failure": "Invalid form. See HELP in api_uploader.py"}), 400) - if not torrent_status: - return flask.make_response(flask.jsonify( - {"Error": "Invalid or Duplicate torrent file."}), 400) + try: + torrent_file = upload_request.files['torrent_file'] + torrent_file = ImmutableMultiDict([('torrent_file', torrent_file)]) - # The torrent has been validated and is safe to access with ['foo'] etc - all relevant - # keys and values have been checked for (see UploadForm in forms.py for details) - info_dict = torrent_data.torrent_dict['info'] + # print(repr(torrent_file)) + except Exception as e: + pass - changed_to_utf8 = _replace_utf8_values(torrent_data.torrent_dict) + form = forms.UploadForm(CombinedMultiDict((torrent_file, form_info))) + form.category.choices = _create_upload_category_choices() - torrent_filesize = info_dict.get('length') or sum( - f['length'] for f in info_dict.get('files')) + if upload_request.method == 'POST' and form.validate(): + torrent = backend.handle_torrent_upload(form, user, True) - # In case no encoding, assume UTF-8. - torrent_encoding = torrent_data.torrent_dict.get('encoding', b'utf-8').decode('utf-8') - - torrent = models.Torrent(info_hash=torrent_data.info_hash, - display_name=display_name, - torrent_name=torrent_data.filename, - information=information, - description=description, - encoding=torrent_encoding, - filesize=torrent_filesize, - user=current_user) - - # Store bencoded info_dict - torrent.info = models.TorrentInfo(info_dict=torrent_data.bencoded_info_dict) - torrent.stats = models.Statistic() - torrent.has_torrent = True - - # Fields with default value will be None before first commit, so set .flags - torrent.flags = 0 - - torrent.anonymous = True if torrent_flags[0] else False - torrent.hidden = True if torrent_flags[1] else False - torrent.remake = True if torrent_flags[2] else False - torrent.complete = True if torrent_flags[3] else False - # Copy trusted status from user if possible - torrent.trusted = (current_user.level >= - models.UserLevelType.TRUSTED) if current_user else False - - # Set category ids - torrent.main_category_id = cat_id - torrent.sub_category_id = sub_cat_id - # To simplify parsing the filelist, turn single-file torrent into a list - torrent_filelist = info_dict.get('files') - - used_path_encoding = changed_to_utf8 and 'utf-8' or torrent_encoding - - parsed_file_tree = dict() - if not torrent_filelist: - # If single-file, the root will be the file-tree (no directory) - file_tree_root = parsed_file_tree - torrent_filelist = [{'length': torrent_filesize, 'path': [info_dict['name']]}] - else: - # If multi-file, use the directory name as root for files - file_tree_root = parsed_file_tree.setdefault( - info_dict['name'].decode(used_path_encoding), {}) - - # Parse file dicts into a tree - for file_dict in torrent_filelist: - # Decode path parts from utf8-bytes - path_parts = [path_part.decode(used_path_encoding) - for path_part in file_dict['path']] - - filename = path_parts.pop() - current_directory = file_tree_root - - for directory in path_parts: - current_directory = current_directory.setdefault(directory, {}) - - current_directory[filename] = file_dict['length'] - - parsed_file_tree = utils.sorted_pathdict(parsed_file_tree) - - json_bytes = json.dumps(parsed_file_tree, separators=(',', ':')).encode('utf8') - torrent.filelist = models.TorrentFilelist(filelist_blob=json_bytes) - - db.session.add(torrent) - db.session.flush() - - # Store the users trackers - trackers = OrderedSet() - announce = torrent_data.torrent_dict.get('announce', b'').decode('ascii') - if announce: - trackers.add(announce) - - # List of lists with single item - announce_list = torrent_data.torrent_dict.get('announce-list', []) - for announce in announce_list: - trackers.add(announce[0].decode('ascii')) - - # Remove our trackers, maybe? TODO ? - - # Search for/Add trackers in DB - db_trackers = OrderedSet() - for announce in trackers: - tracker = models.Trackers.by_uri(announce) - - # Insert new tracker if not found - if not tracker: - tracker = models.Trackers(uri=announce) - db.session.add(tracker) - - db_trackers.add(tracker) - - db.session.flush() - - # Store tracker refs in DB - for order, tracker in enumerate(db_trackers): - torrent_tracker = models.TorrentTrackers(torrent_id=torrent.id, - tracker_id=tracker.id, order=order) - db.session.add(torrent_tracker) - - db.session.commit() - - if app.config.get('BACKUP_TORRENT_FOLDER'): - torrent_file.seek(0, 0) - torrent_path = os.path.join(app.config['BACKUP_TORRENT_FOLDER'], '{}.{}'.format( - torrent.id, secure_filename(torrent_file.filename))) - torrent_file.save(torrent_path) - torrent_file.close() - - # print('Success? {0}'.format(torrent.id)) - return flask.make_response(flask.jsonify( - {"Success": "Request was processed {0}".format(torrent.id)}), 200) - except Exception as e: - print('Exception: {0}'.format(e)) - return flask.make_response(flask.jsonify( - {"Error": "Incorrect JSON. Please see HELP page for examples."}), 400) + return flask.make_response(flask.jsonify({"Success": "Request was processed {0}".format(torrent.id)}), 200) else: - return flask.make_response(flask.jsonify({"Error": "Bad request"}), 400) + # print(form.errors) + return_error_messages = [] + for error_name, error_messages in form.errors.items(): + # print(error_messages) + return_error_messages.extend(error_messages) + + return flask.make_response(flask.jsonify({"Failure": return_error_messages}), 400) diff --git a/nyaa/backend.py b/nyaa/backend.py index 5b10397..e031f22 100644 --- a/nyaa/backend.py +++ b/nyaa/backend.py @@ -26,7 +26,7 @@ def _replace_utf8_values(dict_or_list): return did_change -def handle_torrent_upload(upload_form, uploading_user=None): +def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False): torrent_data = upload_form.torrent_file.parsed_data # The torrent has been validated and is safe to access with ['foo'] etc - all relevant @@ -70,12 +70,8 @@ def handle_torrent_upload(upload_form, uploading_user=None): # Copy trusted status from user if possible torrent.trusted = (uploading_user.level >= models.UserLevelType.TRUSTED) if uploading_user else False - # Set category ids - torrent.main_category_id, torrent.sub_category_id = \ - upload_form.category.parsed_data.get_category_ids() - # print('Main cat id: {0}, Sub cat id: {1}'.format( - # torrent.main_category_id, torrent.sub_category_id)) + torrent.main_category_id, torrent.sub_category_id = upload_form.category.parsed_data.get_category_ids() # To simplify parsing the filelist, turn single-file torrent into a list torrent_filelist = info_dict.get('files') diff --git a/nyaa/routes.py b/nyaa/routes.py index 274e4df..0c6b58f 100644 --- a/nyaa/routes.py +++ b/nyaa/routes.py @@ -346,7 +346,7 @@ def render_rss(label, query, use_elastic): response = flask.make_response(rss_xml) response.headers['Content-Type'] = 'application/xml' # Cache for an hour - response.headers['Cache-Control'] = 'max-age={}'.format(1*5*60) + response.headers['Cache-Control'] = 'max-age={}'.format(1 * 5 * 60) return response @@ -515,6 +515,7 @@ def _create_upload_category_choices(): @app.route('/upload', methods=['GET', 'POST']) def upload(): form = forms.UploadForm(CombinedMultiDict((flask.request.files, flask.request.form))) + print('{0} - {1}'.format(flask.request.files, flask.request.form)) form.category.choices = _create_upload_category_choices() if flask.request.method == 'POST' and form.validate(): torrent = backend.handle_torrent_upload(form, flask.g.user) @@ -696,8 +697,10 @@ def site_help(): # #################################### API ROUTES #################################### -# DISABLED FOR NOW @app.route('/api/upload', methods=['POST']) def api_upload(): - api_response = api_handler.api_upload(flask.request) + is_valid_user, user, debug = api_handler.validate_user(flask.request) + if not is_valid_user: + return flask.make_response(flask.jsonify({"Failure": "Invalid username or password."}), 400) + api_response = api_handler.api_upload(flask.request, user) return api_response diff --git a/utils/api_uploader.py b/utils/api_uploader.py index 64457fa..3102e83 100644 --- a/utils/api_uploader.py +++ b/utils/api_uploader.py @@ -1,52 +1,115 @@ -# api_uploader.py - - -# Uploads a single file -# I will create another script for batch uploading +# Uploads a single torrent file +# Works on nyaa.si +# An updated version will work on sukebei.nyaa.si import json +# pip install requests +# http://docs.python-requests.org/en/master/user/install/ import requests -url = "http://127.0.0.1:5500/api/upload" +#url = "http://127.0.0.1:5500/api/upload" +url = "https://nyaa.si/api/upload" -# Required for Auth +# ########################## REQUIRED: YOUR USERNAME AND PASSWORD ############################## username = "" password = "" -# Required -torrent_name = "" +# ########################################### HELP ############################################ +# ################################# CATEGORIES MUST BE EXACT ################################## +""" +Anime + Anime - AMV : "1_1" + Anime - English : "1_2" + Anime - Non-English : "1_3" + Anime - Raw : "1_4" +Audio + Lossless : "2_1" + Lossy : "2_2" +Literature + Literature - English-translated : "3_1" + Literature - Non-English : "3_2" + Literature - Non-English-Translated : "3_3" + Literature - Raw : "3_4" +Live Action + Live Action - English-translated : "4_1" + Live Action - Idol/Promotional Video : "4_2" + Live Action - Non-English-translated : "4_3" + Live Action - Raw : "4_4" +Pictures + Pictures - Graphics : "5_1" + Pictures - Photos : "5_2" +Software + Software - Applications : "6_1" + Software - Games : "6_2" +""" +# ################################# CATEGORIES MUST BE EXACT ################################## + +# ###################################### EXAMPLE REQUEST ###################################### +""" # Required -main_cat = "" +file_name = "/path/to/my_file.torrent" # Required -sub_cat = "" +category = "6_1" +# Required +display_name = "API upload example" + +# May be blank +information = "API HOWTO" +# May be blank +description = "Visit #nyaa-dev@irc.rizon.net" +# Default is 'n' No +# Change to 'y' Yes to set +is_anonymous : 'n', +is_hidden : 'n', +is_remake : 'n', +is_complete : 'n' +""" +# ############################################################################################# + +# ######################################## CHANGE HERE ######################################## +# Required +file_name = "" +# Required +category = "" +# Required +display_name = "" # May be blank information = "" # May be blank description = "" -# flags = [Hidden, Remake, Complete, Anonymous] -# 0 for NOT SET / 1 for SET -# Required -flags = [0, 0, 0, 0] +# Default is 'n' No +# Change to 'y' Yes to set +is_anonymous = 'n' +is_hidden = 'n' +is_remake = 'n' +is_complete = 'n' +# ############################################################################################# + +# #################################### DO NOT CHANGE BELOW #################################### +# ############################ UNLESS YOU KNOW WHAT YOU ARE DOING ############################# +auth_info = { + "username" : username, + "password" : password +} metadata={ - "username": username, - "password": password, - "display_name": torrent_name, - "main_cat": main_cat, - "sub_cat": sub_cat, - "information": information, - "description": description, - "flags": flags - } - -# Required -file_name = "" + "category" : category, + "display_name" : display_name, + "information" : information, + "description" : description, + "is_anonymous" : is_anonymous, + "is_hidden" : is_hidden, + "is_remake" : is_remake, + "is_complete" : is_complete +} files = { - 'json': (json.dumps(metadata)), - 'torrent': ('{0}'.format(file_name), open(file_name, 'rb'), 'application/octet-stream')} + 'auth_info' : (json.dumps(auth_info)), + 'torrent_info' : (json.dumps(metadata)), + 'torrent_file' : ('{0}'.format(file_name), open(file_name, 'rb'), 'application/octet-stream') +} response = requests.post(url, files=files)