Merge pull request #115 from kyamiko/master

Updated api/upload

nyaa/api_handler.py

@@ -1,334 +1,92 @@
 import flask
+from werkzeug.datastructures import ImmutableMultiDict, CombinedMultiDict
+
 from nyaa import app, db
 from nyaa import models, forms
-from nyaa import bencode, utils
+from nyaa import bencode, backend, utils
 from nyaa import torrents
 
 import json
 import os.path
-from orderedset import OrderedSet
+#from orderedset import OrderedSet
-from werkzeug import secure_filename
+#from werkzeug import secure_filename
 
-DEBUG_API = False
+# #################################### API HELPERS ####################################
-# #################################### API ROUTES ####################################
-CATEGORIES = [
-    ('Anime', ['Anime Music Video', 'English-translated', 'Non-English-translated', 'Raw']),
-    ('Audio', ['Lossless', 'Lossy']),
-    ('Literature', ['English-translated', 'Non-English-translated', 'Raw']),
-    ('Live Action', ['English-translated',
-                     'Idol/Promotional Video', 'Non-English-translated', 'Raw']),
-    ('Pictures', ['Graphics', 'Photos']),
-    ('Software', ['Applications', 'Games']),
-]
 
 
-def validate_main_sub_cat(main_cat_name, sub_cat_name):
+def validate_user(upload_request):
-    for main_cat in models.MainCategory.query.order_by(models.MainCategory.id):
+    auth_info = None
-        if main_cat_name == main_cat.name:
-            for sub_cat in main_cat.sub_categories:
-                if sub_cat_name == sub_cat.name:
-                    cat_id = main_cat.id_as_string
-                    sub_cat_id = sub_cat.id_as_string
-                    cat_sub_cat = sub_cat_id.split('_')
-                    # print('cat: {0} sub_cat: {1}'.format(cat_sub_cat[0], cat_sub_cat[1]))
-
-                    return True, cat_sub_cat[0], cat_sub_cat[1]
-
-        return False, 0, 0
-
-
-def _replace_utf8_values(dict_or_list):
-    ''' Will replace 'property' with 'property.utf-8' and remove latter if it exists.
-        Thanks, bitcomet! :/ '''
-    did_change = False
-    if isinstance(dict_or_list, dict):
-        for key in [key for key in dict_or_list.keys() if key.endswith('.utf-8')]:
-            dict_or_list[key.replace('.utf-8', '')] = dict_or_list.pop(key)
-            did_change = True
-        for value in dict_or_list.values():
-            did_change = _replace_utf8_values(value) or did_change
-    elif isinstance(dict_or_list, list):
-        for item in dict_or_list:
-            did_change = _replace_utf8_values(item) or did_change
-    return did_change
-
-
-def validate_torrent_flags(torrent_flags):
-    _torrent_flags = ['hidden', 'remake', 'complete', 'anonymous']
-
-    if len(torrent_flags) != 4:
-        return False
-
-    for flag in torrent_flags:
-        if int(flag) not in [0, 1]:
-            return False
-    return True
-
-# It might be good to factor this out of forms UploadForm because the same code is
-# used in both files.
-
-
-def validate_torrent_file(torrent_file_name, torrent_file):
-    # Decode and ensure data is bencoded data
     try:
-        torrent_dict = bencode.decode(torrent_file)
+        if 'auth_info' in upload_request.files:
-    except (bencode.MalformedBencodeException, UnicodeError):
+            auth_info = json.loads(upload_request.files['auth_info'].read().decode('utf-8'))
-        return False, 'Malformed torrent file'
+            if 'username' not in auth_info.keys() or 'password' not in auth_info.keys():
+                return False, None, None
 
-    # Uncomment for debug print of the torrent
+            username = auth_info['username']
-    # forms._debug_print_torrent_metadata(torrent_dict)
+            password = auth_info['password']
-
-    try:
-        forms._validate_torrent_metadata(torrent_dict)
-    except AssertionError as e:
-        return False, 'Malformed torrent metadata ({})'.format(e.args[0])
-
-    # Note! bencode will sort dict keys, as per the spec
-    # This may result in a different hash if the uploaded torrent does not match the
-    # spec, but it's their own fault for using broken software! Right?
-    bencoded_info_dict = bencode.encode(torrent_dict['info'])
-    info_hash = utils.sha1_hash(bencoded_info_dict)
-
-    # Check if the info_hash exists already in the database
-    existing_torrent = models.Torrent.by_info_hash(info_hash)
-    if existing_torrent:
-        return False, 'That torrent already exists (#{})'.format(existing_torrent.id)
-
-    # Torrent is legit, pass original filename and dict along
-    return True, forms.TorrentFileData(filename=os.path.basename(torrent_file_name),
-                                       torrent_dict=torrent_dict,
-                                       info_hash=info_hash,
-                                       bencoded_info_dict=bencoded_info_dict)
-
-
-def api_upload(upload_request):
-    if upload_request.method == 'POST':
-        j = None
-        torrent_file = None
-        try:
-            if 'json' in upload_request.files:
-                f = upload_request.files['json']
-                j = json.loads(f.read().decode('utf-8'))
-                if DEBUG_API:
-                    print(json.dumps(j, indent=4))
-
-                _json_keys = ['username',
-                              'password',
-                              'display_name',
-                              'main_cat',
-                              'sub_cat',
-                              'flags']  # 'information' and 'description' are not required
-                # Check that required fields are present
-                for _k in _json_keys:
-                    if _k not in j.keys():
-                        return flask.make_response(flask.jsonify(
-                            {"Error": "Missing JSON field: {0}.".format(_k)}), 400)
-                # Check that no extra fields are present
-                for k in j.keys():
-                    if k not in set(_json_keys + ['information', 'description']):
-                        return flask.make_response(flask.jsonify(
-                            {"Error": "Incorrect JSON field(s)."}), 400)
-            else:
-                return flask.make_response(flask.jsonify({"Error": "No metadata."}), 400)
-            if 'torrent' in upload_request.files:
-                f = upload_request.files['torrent']
-                if DEBUG_API:
-                    print(f.filename)
-                torrent_file = f
-                # print(f.read())
-            else:
-                return flask.make_response(flask.jsonify({"Error": "No torrent file."}), 400)
-
-            # 'username' and 'password' must have been provided as they are part of j.keys()
-            username = j['username']
-            password = j['password']
-            # Validate that the provided username and password belong to a valid user
             user = models.User.by_username(username)
 
             if not user:
                 user = models.User.by_email(username)
 
-            if (not user or password != user.password_hash
+            if (not user or password != user.password_hash or user.status == models.UserStatusType.INACTIVE):
-                    or user.status == models.UserStatusType.INACTIVE):
+                return False, None, None
-                return flask.make_response(flask.jsonify(
-                    {"Error": "Incorrect username or password"}), 403)
 
-            current_user = user
+            return True, user, None
 
-            display_name = j['display_name']
+    except Exception as e:
-            if (len(display_name) < 3) or (len(display_name) > 1024):
+        return False, None, e
-                return flask.make_response(flask.jsonify(
-                    {"Error": "Torrent name must be between 3 and 1024 characters."}), 400)
 
-            main_cat_name = j['main_cat']
-            sub_cat_name = j['sub_cat']
 
-            cat_subcat_status, cat_id, sub_cat_id = validate_main_sub_cat(
+def _create_upload_category_choices():
-                main_cat_name, sub_cat_name)
+    ''' Turns categories in the database into a list of (id, name)s '''
-            if not cat_subcat_status:
+    choices = [('', '[Select a category]')]
-                return flask.make_response(flask.jsonify(
+    for main_cat in models.MainCategory.query.order_by(models.MainCategory.id):
-                    {"Error": "Incorrect Category / Sub-Category."}), 400)
+        choices.append((main_cat.id_as_string, main_cat.name, True))
+        for sub_cat in main_cat.sub_categories:
+            choices.append((sub_cat.id_as_string, ' - ' + sub_cat.name))
+    return choices
 
-            # TODO Sanitize information
-            information = None
-            try:
-                information = j['information']
-                if len(information) > 255:
-                    return flask.make_response(flask.jsonify(
-                        {"Error": "Information is limited to 255 characters."}), 400)
-            except Exception as e:
-                information = ''
 
-            # TODO Sanitize description
+# #################################### API ROUTES ####################################
-            description = None
+def api_upload(upload_request, user):
-            try:
+    form_info = None
-                description = j['description']
+    try:
-                limit = 10 * 1024
+        form_info = json.loads(upload_request.files['torrent_info'].read().decode('utf-8'))
-                if len(description) > limit:
-                    return flask.make_response(flask.jsonify(
-                        {"Error": "Description is limited to {0} characters.".format(limit)}), 403)
-            except Exception as e:
-                description = ''
 
-            v_flags = validate_torrent_flags(j['flags'])
+        form_info_as_dict = []
-            if v_flags:
+        for k, v in form_info.items():
-                torrent_flags = j['flags']
+            if k in ['is_anonymous', 'is_hidden', 'is_remake', 'is_complete']:
+                if v == 'y':
+                    form_info_as_dict.append((k, v))
             else:
-                return flask.make_response(flask.jsonify(
+                form_info_as_dict.append((k, v))
-                    {"Error": "Incorrect torrent flags."}), 400)
+        form_info = ImmutableMultiDict(form_info_as_dict)
 
-            torrent_status, torrent_data = validate_torrent_file(
+        # print(repr(form_info))
-                torrent_file.filename, torrent_file.read())  # Needs validation
+    except Exception as e:
+        return flask.make_response(flask.jsonify({"Failure": "Invalid form. See HELP in api_uploader.py"}), 400)
 
-            if not torrent_status:
+    try:
-                return flask.make_response(flask.jsonify(
+        torrent_file = upload_request.files['torrent_file']
-                    {"Error": "Invalid or Duplicate torrent file."}), 400)
+        torrent_file = ImmutableMultiDict([('torrent_file', torrent_file)])
 
-            # The torrent has been  validated and is safe to access with ['foo'] etc - all relevant
+        # print(repr(torrent_file))
-            # keys and values have been checked for (see UploadForm in forms.py for details)
+    except Exception as e:
-            info_dict = torrent_data.torrent_dict['info']
+        pass
 
-            changed_to_utf8 = _replace_utf8_values(torrent_data.torrent_dict)
+    form = forms.UploadForm(CombinedMultiDict((torrent_file, form_info)))
+    form.category.choices = _create_upload_category_choices()
 
-            torrent_filesize = info_dict.get('length') or sum(
+    if upload_request.method == 'POST' and form.validate():
-                f['length'] for f in info_dict.get('files'))
+        torrent = backend.handle_torrent_upload(form, user, True)
 
-            # In case no encoding, assume UTF-8.
+        return flask.make_response(flask.jsonify({"Success": "Request was processed {0}".format(torrent.id)}), 200)
-            torrent_encoding = torrent_data.torrent_dict.get('encoding', b'utf-8').decode('utf-8')
-
-            torrent = models.Torrent(info_hash=torrent_data.info_hash,
-                                     display_name=display_name,
-                                     torrent_name=torrent_data.filename,
-                                     information=information,
-                                     description=description,
-                                     encoding=torrent_encoding,
-                                     filesize=torrent_filesize,
-                                     user=current_user)
-
-            # Store bencoded info_dict
-            torrent.info = models.TorrentInfo(info_dict=torrent_data.bencoded_info_dict)
-            torrent.stats = models.Statistic()
-            torrent.has_torrent = True
-
-            # Fields with default value will be None before first commit, so set .flags
-            torrent.flags = 0
-
-            torrent.anonymous = True if torrent_flags[0] else False
-            torrent.hidden = True if torrent_flags[1] else False
-            torrent.remake = True if torrent_flags[2] else False
-            torrent.complete = True if torrent_flags[3] else False
-            # Copy trusted status from user if possible
-            torrent.trusted = (current_user.level >=
-                               models.UserLevelType.TRUSTED) if current_user else False
-
-            # Set category ids
-            torrent.main_category_id = cat_id
-            torrent.sub_category_id = sub_cat_id
-            # To simplify parsing the filelist, turn single-file torrent into a list
-            torrent_filelist = info_dict.get('files')
-
-            used_path_encoding = changed_to_utf8 and 'utf-8' or torrent_encoding
-
-            parsed_file_tree = dict()
-            if not torrent_filelist:
-                # If single-file, the root will be the file-tree (no directory)
-                file_tree_root = parsed_file_tree
-                torrent_filelist = [{'length': torrent_filesize, 'path': [info_dict['name']]}]
-            else:
-                # If multi-file, use the directory name as root for files
-                file_tree_root = parsed_file_tree.setdefault(
-                    info_dict['name'].decode(used_path_encoding), {})
-
-            # Parse file dicts into a tree
-            for file_dict in torrent_filelist:
-                # Decode path parts from utf8-bytes
-                path_parts = [path_part.decode(used_path_encoding)
-                              for path_part in file_dict['path']]
-
-                filename = path_parts.pop()
-                current_directory = file_tree_root
-
-                for directory in path_parts:
-                    current_directory = current_directory.setdefault(directory, {})
-
-                current_directory[filename] = file_dict['length']
-
-            parsed_file_tree = utils.sorted_pathdict(parsed_file_tree)
-
-            json_bytes = json.dumps(parsed_file_tree, separators=(',', ':')).encode('utf8')
-            torrent.filelist = models.TorrentFilelist(filelist_blob=json_bytes)
-
-            db.session.add(torrent)
-            db.session.flush()
-
-            # Store the users trackers
-            trackers = OrderedSet()
-            announce = torrent_data.torrent_dict.get('announce', b'').decode('ascii')
-            if announce:
-                trackers.add(announce)
-
-            # List of lists with single item
-            announce_list = torrent_data.torrent_dict.get('announce-list', [])
-            for announce in announce_list:
-                trackers.add(announce[0].decode('ascii'))
-
-            # Remove our trackers, maybe? TODO ?
-
-            # Search for/Add trackers in DB
-            db_trackers = OrderedSet()
-            for announce in trackers:
-                tracker = models.Trackers.by_uri(announce)
-
-                # Insert new tracker if not found
-                if not tracker:
-                    tracker = models.Trackers(uri=announce)
-                    db.session.add(tracker)
-
-                db_trackers.add(tracker)
-
-            db.session.flush()
-
-            # Store tracker refs in DB
-            for order, tracker in enumerate(db_trackers):
-                torrent_tracker = models.TorrentTrackers(torrent_id=torrent.id,
-                                                         tracker_id=tracker.id, order=order)
-                db.session.add(torrent_tracker)
-
-            db.session.commit()
-
-            if app.config.get('BACKUP_TORRENT_FOLDER'):
-                torrent_file.seek(0, 0)
-                torrent_path = os.path.join(app.config['BACKUP_TORRENT_FOLDER'], '{}.{}'.format(
-                    torrent.id, secure_filename(torrent_file.filename)))
-                torrent_file.save(torrent_path)
-            torrent_file.close()
-
-            # print('Success? {0}'.format(torrent.id))
-            return flask.make_response(flask.jsonify(
-                {"Success": "Request was processed {0}".format(torrent.id)}), 200)
-        except Exception as e:
-            print('Exception: {0}'.format(e))
-            return flask.make_response(flask.jsonify(
-                {"Error": "Incorrect JSON. Please see HELP page for examples."}), 400)
     else:
-        return flask.make_response(flask.jsonify({"Error": "Bad request"}), 400)
+        # print(form.errors)
+        return_error_messages = []
+        for error_name, error_messages in form.errors.items():
+            # print(error_messages)
+            return_error_messages.extend(error_messages)
+
+        return flask.make_response(flask.jsonify({"Failure": return_error_messages}), 400)

nyaa/backend.py

@@ -26,7 +26,7 @@ def _replace_utf8_values(dict_or_list):
     return did_change
 
 
-def handle_torrent_upload(upload_form, uploading_user=None):
+def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False):
     torrent_data = upload_form.torrent_file.parsed_data
 
     # The torrent has been  validated and is safe to access with ['foo'] etc - all relevant
@@ -70,12 +70,8 @@ def handle_torrent_upload(upload_form, uploading_user=None):
     # Copy trusted status from user if possible
     torrent.trusted = (uploading_user.level >=
                        models.UserLevelType.TRUSTED) if uploading_user else False
-
     # Set category ids
-    torrent.main_category_id, torrent.sub_category_id = \
+    torrent.main_category_id, torrent.sub_category_id = upload_form.category.parsed_data.get_category_ids()
-        upload_form.category.parsed_data.get_category_ids()
-    # print('Main cat id: {0}, Sub cat id: {1}'.format(
-    #    torrent.main_category_id, torrent.sub_category_id))
 
     # To simplify parsing the filelist, turn single-file torrent into a list
     torrent_filelist = info_dict.get('files')

nyaa/routes.py

@@ -346,7 +346,7 @@ def render_rss(label, query, use_elastic):
     response = flask.make_response(rss_xml)
     response.headers['Content-Type'] = 'application/xml'
     # Cache for an hour
-    response.headers['Cache-Control'] = 'max-age={}'.format(1*5*60)
+    response.headers['Cache-Control'] = 'max-age={}'.format(1 * 5 * 60)
     return response
 
 
@@ -515,6 +515,7 @@ def _create_upload_category_choices():
 @app.route('/upload', methods=['GET', 'POST'])
 def upload():
     form = forms.UploadForm(CombinedMultiDict((flask.request.files, flask.request.form)))
+    #print('{0} - {1}'.format(flask.request.files, flask.request.form))
     form.category.choices = _create_upload_category_choices()
     if flask.request.method == 'POST' and form.validate():
         torrent = backend.handle_torrent_upload(form, flask.g.user)
@@ -696,8 +697,10 @@ def site_help():
 
 
 # #################################### API ROUTES ####################################
-# DISABLED FOR NOW
 @app.route('/api/upload', methods=['POST'])
 def api_upload():
-    api_response = api_handler.api_upload(flask.request)
+    is_valid_user, user, debug = api_handler.validate_user(flask.request)
+    if not is_valid_user:
+        return flask.make_response(flask.jsonify({"Failure": "Invalid username or password."}), 400)
+    api_response = api_handler.api_upload(flask.request, user)
     return api_response

utils/api_uploader.py

@@ -1,52 +1,115 @@
-# api_uploader.py
+# Uploads a single torrent file
-
+# Works on nyaa.si
-
+# An updated version will work on sukebei.nyaa.si
-# Uploads a single file
-# I will create another script for batch uploading
 
 import json
+# pip install requests
+# http://docs.python-requests.org/en/master/user/install/
 import requests
 
-url = "http://127.0.0.1:5500/api/upload"
+#url = "http://127.0.0.1:5500/api/upload"
+url = "https://nyaa.si/api/upload"
 
-# Required for Auth
+# ########################## REQUIRED: YOUR USERNAME AND PASSWORD ##############################
 username = ""
 password = ""
 
-# Required
+# ########################################### HELP ############################################
-torrent_name = ""
 
+# ################################# CATEGORIES MUST BE EXACT ##################################
+"""
+Anime
+    Anime - AMV                          : "1_1"
+    Anime - English                      : "1_2"
+    Anime - Non-English                  : "1_3"
+    Anime - Raw                          : "1_4"
+Audio
+    Lossless                             : "2_1"
+    Lossy                                : "2_2"
+Literature
+    Literature - English-translated      : "3_1"
+    Literature - Non-English             : "3_2"
+    Literature - Non-English-Translated  : "3_3"
+    Literature - Raw                     : "3_4"
+Live Action
+    Live Action - English-translated     : "4_1"
+    Live Action - Idol/Promotional Video : "4_2"
+    Live Action - Non-English-translated : "4_3"
+    Live Action - Raw                    : "4_4"
+Pictures
+    Pictures - Graphics                  : "5_1"
+    Pictures - Photos                    : "5_2"
+Software
+    Software - Applications              : "6_1"
+    Software - Games                     : "6_2"
+"""
+# ################################# CATEGORIES MUST BE EXACT ##################################
+
+# ###################################### EXAMPLE REQUEST ######################################
+"""
 # Required
-main_cat = ""
+file_name = "/path/to/my_file.torrent"
 # Required
-sub_cat = ""
+category = "6_1"
+# Required
+display_name = "API upload example"
+
+# May be blank
+information = "API HOWTO"
+# May be blank
+description = "Visit #nyaa-dev@irc.rizon.net"
+# Default is 'n' No
+# Change to 'y' Yes to set
+is_anonymous : 'n',
+is_hidden    : 'n',
+is_remake    : 'n',
+is_complete  : 'n'
+"""
+# #############################################################################################
+
+# ######################################## CHANGE HERE ########################################
+# Required
+file_name = ""
+# Required
+category = ""
+# Required
+display_name = ""
 
 # May be blank
 information = ""
 # May be blank
 description = ""
-# flags = [Hidden, Remake, Complete, Anonymous]
+# Default is 'n' No
-# 0 for NOT SET / 1 for SET
+# Change to 'y' Yes to set
-# Required
+is_anonymous = 'n'
-flags = [0, 0, 0, 0]
+is_hidden    = 'n'
+is_remake    = 'n'
+is_complete  = 'n'
+# #############################################################################################
+
+# #################################### DO NOT CHANGE BELOW ####################################
+# ############################ UNLESS YOU KNOW WHAT YOU ARE DOING #############################
+auth_info = {
+    "username"     : username,
+    "password"     : password
+}
 
 metadata={
-    "username": username,
+    "category"     : category,
-    "password": password,
+    "display_name" : display_name,
-    "display_name": torrent_name,
+    "information"  : information,
-    "main_cat": main_cat,
+    "description"  : description,
-    "sub_cat": sub_cat,
+    "is_anonymous" : is_anonymous,
-    "information": information,
+    "is_hidden"    : is_hidden,
-    "description": description,
+    "is_remake"    : is_remake,
-    "flags": flags
+    "is_complete"  : is_complete
-    }
+}
-
-# Required
-file_name = ""
 
 files = {
-    'json': (json.dumps(metadata)),
+    'auth_info'    : (json.dumps(auth_info)),
-    'torrent': ('{0}'.format(file_name), open(file_name, 'rb'), 'application/octet-stream')}
+    'torrent_info' : (json.dumps(metadata)),
+    'torrent_file' : ('{0}'.format(file_name), open(file_name, 'rb'), 'application/octet-stream')
+}
 
 response = requests.post(url, files=files)