From 3b87337c71b6b5bfb02365190ad1276606ddae51 Mon Sep 17 00:00:00 2001 From: sharkykh Date: Sun, 14 May 2017 09:21:47 +0300 Subject: [PATCH 01/21] RSS: Add elements & refactor description, size, seeders, leechers, downloads --- nyaa/templates/rss.xml | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/nyaa/templates/rss.xml b/nyaa/templates/rss.xml index e1787d2..c6fdcdd 100644 --- a/nyaa/templates/rss.xml +++ b/nyaa/templates/rss.xml @@ -5,33 +5,31 @@ {{ url_for('home', _external=True) }} {% for torrent in torrent_query %} - {% if torrent.has_torrent %} {{ torrent.display_name }} + {{ torrent.description }} {% if use_elastic %} - {{ url_for('download_torrent', torrent_id=torrent.meta.id, _external=True) }} - {{ url_for('view_torrent', torrent_id=torrent.meta.id, _external=True) }} - {{ torrent.created_time|rfc822_es }} + {% if torrent.has_torrent %} + {{ url_for('download_torrent', torrent_id=torrent.meta.id, _external=True) }} + {% else %} + {{ create_magnet_from_info(torrent.display_name, torrent.info_hash) }} + {% endif %} + {{ url_for('view_torrent', torrent_id=torrent.meta.id, _external=True) }} + {{ torrent.created_time|rfc822_es }} {% else %} - {{ url_for('download_torrent', torrent_id=torrent.id, _external=True) }} - {{ url_for('view_torrent', torrent_id=torrent.id, _external=True) }} - {{ torrent.created_time|rfc822 }} - {% endif %} - - {% else %} - - {{ torrent.display_name }} - {% if use_elastic %} - {{ create_magnet_from_info(torrent.display_name, torrent.info_hash) }} - {{ url_for('view_torrent', torrent_id=torrent.meta.id, _external=True) }} - {{ torrent.created_time|rfc822_es }} - {% else %} - {{ torrent.magnet_uri }} - {{ url_for('view_torrent', torrent_id=torrent.id, _external=True) }} - {{ torrent.created_time|rfc822 }} + {% if torrent.has_torrent %} + {{ url_for('download_torrent', torrent_id=torrent.id, _external=True) }} + {% else %} + {{ torrent.magnet_uri }} + {% endif %} + {{ url_for('view_torrent', torrent_id=torrent.id, _external=True) }} + {{ torrent.created_time|rfc822 }} {% endif %} + {{ torrent.filesize }} + {{ torrent.stats.seed_count }} + {{ torrent.stats.leech_count }} + {{ torrent.stats.download_count }} - {% endif %} {% endfor %} From d4621a23a7a10378ac7eac94947e3ba78e6b41fe Mon Sep 17 00:00:00 2001 From: sharkykh Date: Sun, 14 May 2017 09:23:30 +0300 Subject: [PATCH 02/21] Add bare torrent hash lambda is probably not the best way to go, suggestions for improvement are welcome! --- nyaa/routes.py | 1 + nyaa/templates/rss.xml | 1 + 2 files changed, 2 insertions(+) diff --git a/nyaa/routes.py b/nyaa/routes.py index 48c8428..27d2dfb 100644 --- a/nyaa/routes.py +++ b/nyaa/routes.py @@ -303,6 +303,7 @@ def render_rss(label, query, use_elastic): use_elastic=use_elastic, term=label, site_url=flask.request.url_root, + compute_hash=lambda x: base64.b32encode(x).decode('utf-8'), torrent_query=query) response = flask.make_response(rss_xml) response.headers['Content-Type'] = 'application/xml' diff --git a/nyaa/templates/rss.xml b/nyaa/templates/rss.xml index c6fdcdd..91da605 100644 --- a/nyaa/templates/rss.xml +++ b/nyaa/templates/rss.xml @@ -29,6 +29,7 @@ {{ torrent.stats.seed_count }} {{ torrent.stats.leech_count }} {{ torrent.stats.download_count }} + {{ compute_hash(torrent.info_hash) }} {% endfor %} From e2eb3fb33c4ea28807e1860bc40f9c17130547b0 Mon Sep 17 00:00:00 2001 From: sharkykh Date: Sun, 14 May 2017 10:18:45 +0300 Subject: [PATCH 03/21] Category (main+sub) --- nyaa/templates/rss.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/nyaa/templates/rss.xml b/nyaa/templates/rss.xml index 91da605..f962a0b 100644 --- a/nyaa/templates/rss.xml +++ b/nyaa/templates/rss.xml @@ -25,6 +25,7 @@ {{ url_for('view_torrent', torrent_id=torrent.id, _external=True) }} {{ torrent.created_time|rfc822 }} {% endif %} + {{ torrent.main_category.name }} - {{ torrent.sub_category.name }} {{ torrent.filesize }} {{ torrent.stats.seed_count }} {{ torrent.stats.leech_count }} From 837ecde795186480975df20959a09a5345b4381f Mon Sep 17 00:00:00 2001 From: sharkykh Date: Sun, 14 May 2017 11:02:07 +0300 Subject: [PATCH 04/21] Wrap description in a CDATA tag --- nyaa/templates/rss.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nyaa/templates/rss.xml b/nyaa/templates/rss.xml index f962a0b..97eaba7 100644 --- a/nyaa/templates/rss.xml +++ b/nyaa/templates/rss.xml @@ -7,7 +7,7 @@ {% for torrent in torrent_query %} {{ torrent.display_name }} - {{ torrent.description }} + {% if use_elastic %} {% if torrent.has_torrent %} {{ url_for('download_torrent', torrent_id=torrent.meta.id, _external=True) }} From c691087730f52345cfdbf4c72b056625b70cb5a4 Mon Sep 17 00:00:00 2001 From: Kfir Hadas Date: Sun, 14 May 2017 18:57:57 +0300 Subject: [PATCH 05/21] Use formatted file size since that what was used by Nyaa. --- nyaa/templates/rss.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nyaa/templates/rss.xml b/nyaa/templates/rss.xml index 97eaba7..839e55a 100644 --- a/nyaa/templates/rss.xml +++ b/nyaa/templates/rss.xml @@ -26,7 +26,7 @@ {{ torrent.created_time|rfc822 }} {% endif %} {{ torrent.main_category.name }} - {{ torrent.sub_category.name }} - {{ torrent.filesize }} + {{ torrent.filesize | filesizeformat(True) }} {{ torrent.stats.seed_count }} {{ torrent.stats.leech_count }} {{ torrent.stats.download_count }} From daef4a9c6a31b7a6be6c5764402b3f2bf88b9932 Mon Sep 17 00:00:00 2001 From: TheAMM Date: Fri, 19 May 2017 19:25:48 +0300 Subject: [PATCH 06/21] Finalize RSS ES fix --- nyaa/models.py | 9 +++++++++ nyaa/routes.py | 1 - nyaa/templates/rss.xml | 24 ++++++++++++++++-------- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/nyaa/models.py b/nyaa/models.py index 189c635..b7d9b6c 100644 --- a/nyaa/models.py +++ b/nyaa/models.py @@ -8,6 +8,7 @@ from werkzeug.security import generate_password_hash, check_password_hash from sqlalchemy_fulltext import FullText import re +import base64 from markupsafe import escape as escape_markup from urllib.parse import unquote as unquote_url @@ -121,6 +122,14 @@ class Torrent(db.Model): # Escaped return escape_markup(self.information) + @property + def info_hash_as_b32(self): + return base64.b32encode(self.info_hash).decode('utf-8') + + @property + def info_hash_as_hex(self): + return self.info_hash.hex() + @property def magnet_uri(self): return create_magnet(self) diff --git a/nyaa/routes.py b/nyaa/routes.py index cfae637..fe4ccff 100644 --- a/nyaa/routes.py +++ b/nyaa/routes.py @@ -367,7 +367,6 @@ def render_rss(label, query, use_elastic): use_elastic=use_elastic, term=label, site_url=flask.request.url_root, - compute_hash=lambda x: base64.b32encode(x).decode('utf-8'), torrent_query=query) response = flask.make_response(rss_xml) response.headers['Content-Type'] = 'application/xml' diff --git a/nyaa/templates/rss.xml b/nyaa/templates/rss.xml index 839e55a..53fc8e8 100644 --- a/nyaa/templates/rss.xml +++ b/nyaa/templates/rss.xml @@ -1,13 +1,13 @@ - {{ config.SITE_NAME }} Torrent File RSS (No magnets) + {{ config.SITE_NAME }} Torrent File RSS RSS Feed for {{ term }} {{ url_for('home', _external=True) }} {% for torrent in torrent_query %} {{ torrent.display_name }} - + {# #} {% if use_elastic %} {% if torrent.has_torrent %} {{ url_for('download_torrent', torrent_id=torrent.meta.id, _external=True) }} @@ -16,6 +16,11 @@ {% endif %} {{ url_for('view_torrent', torrent_id=torrent.meta.id, _external=True) }} {{ torrent.created_time|rfc822_es }} + + {{- torrent.seed_count }} + {{- torrent.leech_count }} + {{- torrent.download_count }} + {{- torrent.info_hash }} {% else %} {% if torrent.has_torrent %} {{ url_for('download_torrent', torrent_id=torrent.id, _external=True) }} @@ -24,13 +29,16 @@ {% endif %} {{ url_for('view_torrent', torrent_id=torrent.id, _external=True) }} {{ torrent.created_time|rfc822 }} + + {{- torrent.stats.seed_count }} + {{- torrent.stats.leech_count }} + {{- torrent.stats.download_count }} + {{- torrent.info_hash_as_hex }} {% endif %} - {{ torrent.main_category.name }} - {{ torrent.sub_category.name }} - {{ torrent.filesize | filesizeformat(True) }} - {{ torrent.stats.seed_count }} - {{ torrent.stats.leech_count }} - {{ torrent.stats.download_count }} - {{ compute_hash(torrent.info_hash) }} + {% set cat_id = use_elastic and ((torrent.main_category_id|string) + '_' + (torrent.sub_category_id|string)) or torrent.sub_category.id_as_string %} + {{- cat_id }} + {{- category_name(cat_id) }} + {{- torrent.filesize | filesizeformat(True) }} {% endfor %} From 68b5bc045b0c911198f836038735990c688a9baa Mon Sep 17 00:00:00 2001 From: TheAMM Date: Fri, 19 May 2017 20:11:20 +0300 Subject: [PATCH 07/21] Add "magnets" parameter for RSS, slightly clean up home and view_user --- nyaa/routes.py | 150 +++++++++++++++++++++++------------------ nyaa/templates/rss.xml | 4 +- 2 files changed, 86 insertions(+), 68 deletions(-) diff --git a/nyaa/routes.py b/nyaa/routes.py index fe4ccff..38984c6 100644 --- a/nyaa/routes.py +++ b/nyaa/routes.py @@ -135,24 +135,44 @@ def get_category_id_map(): app.register_blueprint(api_handler.api_blueprint, url_prefix='/api') +def chain_get(source, *args): + ''' Tries to return values from source by the given keys. + Returns None if none match. + Note: can return a None from the source. ''' + sentinel = object() + for key in args: + value = source.get(key, sentinel) + if value is not sentinel: + return value + return None @app.route('/rss', defaults={'rss': True}) @app.route('/', defaults={'rss': False}) def home(rss): - if flask.request.args.get('page') == 'rss': - rss = True + render_as_rss = rss + req_args = flask.request.args + if req_args.get('page') == 'rss': + render_as_rss = True - term = flask.request.args.get('q', flask.request.args.get('term')) - sort = flask.request.args.get('s') - order = flask.request.args.get('o') - category = flask.request.args.get('c', flask.request.args.get('cats')) - quality_filter = flask.request.args.get('f', flask.request.args.get('filter')) - user_name = flask.request.args.get('u', flask.request.args.get('user')) - page = flask.request.args.get('p', flask.request.args.get('offset', 1, int), int) + search_term = chain_get(req_args, 'q', 'term') - per_page = app.config.get('RESULTS_PER_PAGE') - if not per_page: - per_page = DEFAULT_PER_PAGE + sort_key = req_args.get('s') + sort_order = req_args.get('o') + + category = chain_get(req_args, 'c', 'cats') + quality_filter = chain_get(req_args, 'f', 'filter') + + user_name = chain_get(req_args, 'u', 'user') + page_number = chain_get(req_args, 'p', 'page', 'offset') + try: + page_number = max(1, int(page_number)) + except (ValueError, TypeError): + page_number = 1 + + # Check simply if the key exists + use_magnet_links = 'magnets' in req_args or 'm' in req_args + + results_per_page = app.config.get('RESULTS_PER_PAGE', DEFAULT_PER_PAGE) user_id = None if user_name: @@ -163,13 +183,13 @@ def home(rss): query_args = { 'user': user_id, - 'sort': sort or 'id', - 'order': order or 'desc', + 'sort': sort_key or 'id', + 'order': sort_order or 'desc', 'category': category or '0_0', 'quality_filter': quality_filter or '0', - 'page': page, - 'rss': rss, - 'per_page': per_page + 'page': page_number, + 'rss': render_as_rss, + 'per_page': results_per_page } if flask.g.user: @@ -179,28 +199,26 @@ def home(rss): # If searching, we get results from elastic search use_elastic = app.config.get('USE_ELASTIC_SEARCH') - if use_elastic and term: - query_args['term'] = term + if use_elastic and search_term: + query_args['term'] = search_term - max_search_results = app.config.get('ES_MAX_SEARCH_RESULT') - if not max_search_results: - max_search_results = DEFAULT_MAX_SEARCH_RESULT + max_search_results = app.config.get('ES_MAX_SEARCH_RESULT', DEFAULT_MAX_SEARCH_RESULT) # Only allow up to (max_search_results / page) pages - max_page = min(query_args['page'], int(math.ceil(max_search_results / float(per_page)))) + max_page = min(query_args['page'], int(math.ceil(max_search_results / results_per_page))) query_args['page'] = max_page query_args['max_search_results'] = max_search_results query_results = search_elastic(**query_args) - if rss: - return render_rss('/', query_results, use_elastic=True) + if render_as_rss: + return render_rss('"{}"'.format(search_term), query_results, use_elastic=True, magnet_links=use_magnet_links) else: - rss_query_string = _generate_query_string(term, category, quality_filter, user_name) + rss_query_string = _generate_query_string(search_term, category, quality_filter, user_name) max_results = min(max_search_results, query_results['hits']['total']) # change p= argument to whatever you change page_parameter to or pagination breaks - pagination = Pagination(p=query_args['page'], per_page=per_page, + pagination = Pagination(p=query_args['page'], per_page=results_per_page, total=max_results, bs_version=3, page_parameter='p', display_msg=SERACH_PAGINATE_DISPLAY_MSG) return flask.render_template('home.html', @@ -214,13 +232,13 @@ def home(rss): if use_elastic: query_args['term'] = '' else: # Otherwise, use db search for everything - query_args['term'] = term or '' + query_args['term'] = search_term or '' query = search_db(**query_args) - if rss: - return render_rss('/', query, use_elastic=False) + if render_as_rss: + return render_rss('Home', query, use_elastic=False, magnet_links=use_magnet_links) else: - rss_query_string = _generate_query_string(term, category, quality_filter, user_name) + rss_query_string = _generate_query_string(search_term, category, quality_filter, user_name) # Use elastic is always false here because we only hit this section # if we're browsing without a search term (which means we default to DB) # or if ES is disabled @@ -259,38 +277,39 @@ def view_user(user_name): return flask.redirect('/user/' + user.username) - level = 'Regular' - if user.is_admin: - level = 'Moderator' - if user.is_superadmin: # check this second because user can be admin AND superadmin - level = 'Administrator' - elif user.is_trusted: - level = 'Trusted' + user_level = ['Regular', 'Trusted', 'Moderator', 'Administrator'][user.level] - term = flask.request.args.get('q') - sort = flask.request.args.get('s') - order = flask.request.args.get('o') - category = flask.request.args.get('c') - quality_filter = flask.request.args.get('f') - page = flask.request.args.get('p') - if page: - page = int(page) + req_args = flask.request.args - per_page = app.config.get('RESULTS_PER_PAGE') - if not per_page: - per_page = DEFAULT_PER_PAGE + search_term = chain_get(req_args, 'q', 'term') + + sort_key = req_args.get('s') + sort_order = req_args.get('o') + + category = chain_get(req_args, 'c', 'cats') + quality_filter = chain_get(req_args, 'f', 'filter') + + user_name = chain_get(req_args, 'u', 'user') + page_number = chain_get(req_args, 'p', 'page', 'offset') + try: + page_number = max(1, int(page_number)) + except (ValueError, TypeError): + page_number = 1 + + results_per_page = app.config.get('RESULTS_PER_PAGE', DEFAULT_PER_PAGE) query_args = { - 'term': term or '', + 'term': search_term or '', 'user': user.id, - 'sort': sort or 'id', - 'order': order or 'desc', + 'sort': sort_key or 'id', + 'order': sort_order or 'desc', 'category': category or '0_0', 'quality_filter': quality_filter or '0', - 'page': page or 1, + 'page': page_number, 'rss': False, - 'per_page': per_page + 'per_page': results_per_page } + print(query_args) if flask.g.user: query_args['logged_in_user'] = flask.g.user @@ -298,17 +317,15 @@ def view_user(user_name): query_args['admin'] = True # Use elastic search for term searching - rss_query_string = _generate_query_string(term, category, quality_filter, user_name) + rss_query_string = _generate_query_string(search_term, category, quality_filter, user_name) use_elastic = app.config.get('USE_ELASTIC_SEARCH') - if use_elastic and term: - query_args['term'] = term + if use_elastic and search_term: + query_args['term'] = search_term - max_search_results = app.config.get('ES_MAX_SEARCH_RESULT') - if not max_search_results: - max_search_results = DEFAULT_MAX_SEARCH_RESULT + max_search_results = app.config.get('ES_MAX_SEARCH_RESULT', DEFAULT_MAX_SEARCH_RESULT) # Only allow up to (max_search_results / page) pages - max_page = min(query_args['page'], int(math.ceil(max_search_results / float(per_page)))) + max_page = min(query_args['page'], int(math.ceil(max_search_results / results_per_page))) query_args['page'] = max_page query_args['max_search_results'] = max_search_results @@ -317,7 +334,7 @@ def view_user(user_name): max_results = min(max_search_results, query_results['hits']['total']) # change p= argument to whatever you change page_parameter to or pagination breaks - pagination = Pagination(p=query_args['page'], per_page=per_page, + pagination = Pagination(p=query_args['page'], per_page=results_per_page, total=max_results, bs_version=3, page_parameter='p', display_msg=SERACH_PAGINATE_DISPLAY_MSG) return flask.render_template('user.html', @@ -328,7 +345,7 @@ def view_user(user_name): user=user, user_page=True, rss_filter=rss_query_string, - level=level, + level=user_level, admin=admin, superadmin=superadmin, form=form) @@ -337,7 +354,7 @@ def view_user(user_name): if use_elastic: query_args['term'] = '' else: - query_args['term'] = term or '' + query_args['term'] = search_term or '' query = search_db(**query_args) return flask.render_template('user.html', use_elastic=False, @@ -346,7 +363,7 @@ def view_user(user_name): user=user, user_page=True, rss_filter=rss_query_string, - level=level, + level=user_level, admin=admin, superadmin=superadmin, form=form) @@ -362,9 +379,10 @@ def _jinja2_filter_rfc822(datestr, fmt=None): return formatdate(float(datetime.strptime(datestr, '%Y-%m-%dT%H:%M:%S').strftime('%s'))) -def render_rss(label, query, use_elastic): +def render_rss(label, query, use_elastic, magnet_links=False): rss_xml = flask.render_template('rss.xml', use_elastic=use_elastic, + magnet_links=magnet_links, term=label, site_url=flask.request.url_root, torrent_query=query) diff --git a/nyaa/templates/rss.xml b/nyaa/templates/rss.xml index 53fc8e8..7664da9 100644 --- a/nyaa/templates/rss.xml +++ b/nyaa/templates/rss.xml @@ -9,7 +9,7 @@ {{ torrent.display_name }} {# #} {% if use_elastic %} - {% if torrent.has_torrent %} + {% if torrent.has_torrent and not magnet_links %} {{ url_for('download_torrent', torrent_id=torrent.meta.id, _external=True) }} {% else %} {{ create_magnet_from_info(torrent.display_name, torrent.info_hash) }} @@ -22,7 +22,7 @@ {{- torrent.download_count }} {{- torrent.info_hash }} {% else %} - {% if torrent.has_torrent %} + {% if torrent.has_torrent and not magnet_links %} {{ url_for('download_torrent', torrent_id=torrent.id, _external=True) }} {% else %} {{ torrent.magnet_uri }} From 358c75036bcd80a475bd39460a143a2c23da7d30 Mon Sep 17 00:00:00 2001 From: TheAMM Date: Fri, 19 May 2017 23:20:15 +0300 Subject: [PATCH 08/21] Fix RSS link generation on user pages Also removes a debug print (oops #2) --- nyaa/routes.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nyaa/routes.py b/nyaa/routes.py index 38984c6..4bb4e8f 100644 --- a/nyaa/routes.py +++ b/nyaa/routes.py @@ -289,7 +289,6 @@ def view_user(user_name): category = chain_get(req_args, 'c', 'cats') quality_filter = chain_get(req_args, 'f', 'filter') - user_name = chain_get(req_args, 'u', 'user') page_number = chain_get(req_args, 'p', 'page', 'offset') try: page_number = max(1, int(page_number)) @@ -309,7 +308,6 @@ def view_user(user_name): 'rss': False, 'per_page': results_per_page } - print(query_args) if flask.g.user: query_args['logged_in_user'] = flask.g.user From 4b4a7b983073c9b80668bb93f1ed19d735d6353a Mon Sep 17 00:00:00 2001 From: TheAMM Date: Sat, 20 May 2017 10:12:32 +0300 Subject: [PATCH 09/21] Support BitComet empty directories (ew) This will allow most (if not all) torrents that have been rejected by 'Malformed torrent metadata (path part is empty)' Adjusts _validate_bytes to disable empty check Adds the empty check to file tree parsing --- nyaa/backend.py | 4 +++- nyaa/forms.py | 16 ++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/nyaa/backend.py b/nyaa/backend.py index 5257f28..87caf00 100644 --- a/nyaa/backend.py +++ b/nyaa/backend.py @@ -100,7 +100,9 @@ def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False): for directory in path_parts: current_directory = current_directory.setdefault(directory, {}) - current_directory[filename] = file_dict['length'] + # Don't add empty filenames (BitComet directory) + if filename: + current_directory[filename] = file_dict['length'] parsed_file_tree = utils.sorted_pathdict(parsed_file_tree) diff --git a/nyaa/forms.py b/nyaa/forms.py index 09bef8d..0f0e073 100644 --- a/nyaa/forms.py +++ b/nyaa/forms.py @@ -280,7 +280,7 @@ class TorrentFileData(object): def _validate_trackers(torrent_dict, tracker_to_check_for=None): announce = torrent_dict.get('announce') - announce_string = _validate_bytes(announce, 'announce', 'utf-8') + announce_string = _validate_bytes(announce, 'announce', test_decode='utf-8') tracker_found = tracker_to_check_for and ( announce_string.lower() == tracker_to_check_for.lower()) or False @@ -292,7 +292,7 @@ def _validate_trackers(torrent_dict, tracker_to_check_for=None): for announce in announce_list: _validate_list(announce, 'announce-list item') - announce_string = _validate_bytes(announce[0], 'announce-list item url', 'utf-8') + announce_string = _validate_bytes(announce[0], 'announce-list item url', test_decode='utf-8') if tracker_to_check_for and announce_string.lower() == tracker_to_check_for.lower(): tracker_found = True @@ -308,7 +308,7 @@ def _validate_torrent_metadata(torrent_dict): assert isinstance(info_dict, dict), 'info is not a dict' encoding_bytes = torrent_dict.get('encoding', b'utf-8') - encoding = _validate_bytes(encoding_bytes, 'encoding', 'utf-8').lower() + encoding = _validate_bytes(encoding_bytes, 'encoding', test_decode='utf-8').lower() name = info_dict.get('name') _validate_bytes(name, 'name', test_decode=encoding) @@ -330,17 +330,21 @@ def _validate_torrent_metadata(torrent_dict): path_list = file_dict.get('path') _validate_list(path_list, 'path') - for path_part in path_list: + # Validate possible directory names + for path_part in path_list[:-1]: _validate_bytes(path_part, 'path part', test_decode=encoding) + # Validate actual filename, allow b'' to specify an empty directory + _validate_bytes(path_list[-1], 'filename', check_empty=False, test_decode=encoding) else: length = info_dict.get('length') _validate_number(length, 'length', check_positive=True) -def _validate_bytes(value, name='value', test_decode=None): +def _validate_bytes(value, name='value', check_empty=True, test_decode=None): assert isinstance(value, bytes), name + ' is not bytes' - assert len(value) > 0, name + ' is empty' + if check_empty: + assert len(value) > 0, name + ' is empty' if test_decode: try: return value.decode(test_decode) From 95d2bff614d02219164a2209d118ac34a3a54785 Mon Sep 17 00:00:00 2001 From: TheAMM Date: Sat, 20 May 2017 10:16:01 +0300 Subject: [PATCH 10/21] Add cascade to Torrent relationships for entry nuking (debugging purposes) Doesn't change the schema, only how SQA will handle session.delete(Torrent.by_id(1337)) --- nyaa/models.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/nyaa/models.py b/nyaa/models.py index b7d9b6c..70fa0a7 100644 --- a/nyaa/models.py +++ b/nyaa/models.py @@ -89,10 +89,14 @@ class Torrent(db.Model): primaryjoin=( "and_(SubCategory.id == foreign(Torrent.sub_category_id), " "SubCategory.main_category_id == Torrent.main_category_id)")) - info = db.relationship('TorrentInfo', uselist=False, back_populates='torrent') - filelist = db.relationship('TorrentFilelist', uselist=False, back_populates='torrent') - stats = db.relationship('Statistic', uselist=False, back_populates='torrent', lazy='joined') - trackers = db.relationship('TorrentTrackers', uselist=True, lazy='joined') + info = db.relationship('TorrentInfo', uselist=False, + cascade="all, delete-orphan", back_populates='torrent') + filelist = db.relationship('TorrentFilelist', uselist=False, + cascade="all, delete-orphan", back_populates='torrent') + stats = db.relationship('Statistic', uselist=False, + cascade="all, delete-orphan", back_populates='torrent', lazy='joined') + trackers = db.relationship('TorrentTrackers', uselist=True, + cascade="all, delete-orphan", lazy='joined') def __repr__(self): return '<{0} #{1.id} \'{1.display_name}\' {1.filesize}b>'.format(type(self).__name__, self) From 2ccf23a1f3b5547c54bdf3d555216dbfa9f13fe0 Mon Sep 17 00:00:00 2001 From: TheAMM Date: Sat, 20 May 2017 21:56:22 +0300 Subject: [PATCH 11/21] Clean up models.User.level helpers --- nyaa/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nyaa/models.py b/nyaa/models.py index 70fa0a7..ca00e9a 100644 --- a/nyaa/models.py +++ b/nyaa/models.py @@ -382,15 +382,15 @@ class User(db.Model): @property def is_admin(self): - return self.level is UserLevelType.ADMIN or self.level is UserLevelType.SUPERADMIN + return self.level >= UserLevelType.ADMIN @property def is_superadmin(self): - return self.level is UserLevelType.SUPERADMIN + return self.level == UserLevelType.SUPERADMIN @property def is_trusted(self): - return self.level is UserLevelType.TRUSTED + return self.level >= UserLevelType.TRUSTED # class Session(db.Model): From 752a5a1f9546e74225818d19efb2040fd5e63ce7 Mon Sep 17 00:00:00 2001 From: TheAMM Date: Sat, 20 May 2017 21:56:22 +0300 Subject: [PATCH 12/21] Clean up models.User.level helpers --- nyaa/backend.py | 4 ++-- nyaa/models.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nyaa/backend.py b/nyaa/backend.py index 87caf00..6be9b5d 100644 --- a/nyaa/backend.py +++ b/nyaa/backend.py @@ -68,8 +68,8 @@ def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False): torrent.remake = upload_form.is_remake.data torrent.complete = upload_form.is_complete.data # Copy trusted status from user if possible - torrent.trusted = (uploading_user.level >= - models.UserLevelType.TRUSTED) if uploading_user else False + can_mark_trusted = uploading_user and uploading_user.is_trusted + torrent.trusted = upload_form.is_trusted.data if can_mark_trusted else False # Set category ids torrent.main_category_id, torrent.sub_category_id = \ upload_form.category.parsed_data.get_category_ids() diff --git a/nyaa/models.py b/nyaa/models.py index 70fa0a7..ca00e9a 100644 --- a/nyaa/models.py +++ b/nyaa/models.py @@ -382,15 +382,15 @@ class User(db.Model): @property def is_admin(self): - return self.level is UserLevelType.ADMIN or self.level is UserLevelType.SUPERADMIN + return self.level >= UserLevelType.ADMIN @property def is_superadmin(self): - return self.level is UserLevelType.SUPERADMIN + return self.level == UserLevelType.SUPERADMIN @property def is_trusted(self): - return self.level is UserLevelType.TRUSTED + return self.level >= UserLevelType.TRUSTED # class Session(db.Model): From b9d88e89604b2e4401ced78955d0720c34e34f07 Mon Sep 17 00:00:00 2001 From: TheAMM Date: Sat, 20 May 2017 22:00:45 +0300 Subject: [PATCH 13/21] Restructure upload.html and edit.html as well as route logic Rename variables and reformats user/admin logic Add an is_trusted field to upload and edit forms Restructure fields on upload and edit pages Add simple styling for checkboxes Add titles (mouseover) for checkboxes with crude explanations Show Anonymous checkbox during upload and check & disable it for guests Show Trusted checkbox for users at or above Trusted level Adjust description field rendering to show field label above it Add title (mouseover) for edit icon on torrent page Show uploader for admins on anonymous torrents Show uploader for admins when editing others' torrents --- nyaa/forms.py | 2 + nyaa/routes.py | 58 ++++++++++++-------- nyaa/templates/_formhelpers.html | 54 +++++++++--------- nyaa/templates/edit.html | 94 +++++++++++++++----------------- nyaa/templates/upload.html | 73 +++++++++++-------------- nyaa/templates/view.html | 11 +++- 6 files changed, 149 insertions(+), 143 deletions(-) diff --git a/nyaa/forms.py b/nyaa/forms.py index 0f0e073..d7cea26 100644 --- a/nyaa/forms.py +++ b/nyaa/forms.py @@ -153,6 +153,7 @@ class EditForm(FlaskForm): is_remake = BooleanField('Remake') is_anonymous = BooleanField('Anonymous') is_complete = BooleanField('Complete') + is_trusted = BooleanField('Trusted') information = StringField('Information', [ Length(max=255, message='Information must be at most %(max)d characters long.') @@ -200,6 +201,7 @@ class UploadForm(FlaskForm): is_remake = BooleanField('Remake') is_anonymous = BooleanField('Anonymous') is_complete = BooleanField('Complete') + is_trusted = BooleanField('Trusted') information = StringField('Information', [ Length(max=255, message='Information must be at most %(max)d characters long.') diff --git a/nyaa/routes.py b/nyaa/routes.py index 4bb4e8f..f41f591 100644 --- a/nyaa/routes.py +++ b/nyaa/routes.py @@ -275,7 +275,7 @@ def view_user(user_name): db.session.add(user) db.session.commit() - return flask.redirect('/user/' + user.username) + return flask.redirect(flask.url_for('view_user', user_name=user.username)) user_level = ['Regular', 'Trusted', 'Moderator', 'Administrator'][user.level] @@ -579,16 +579,17 @@ def upload(): def view_torrent(torrent_id): torrent = models.Torrent.by_id(torrent_id) + viewer = flask.g.user + if not torrent: flask.abort(404) - if torrent.deleted and (not flask.g.user or not flask.g.user.is_admin): + # Only allow admins see deleted torrents + if torrent.deleted and not (viewer and viewer.is_admin): flask.abort(404) - if flask.g.user: - can_edit = flask.g.user is torrent.user or flask.g.user.is_admin - else: - can_edit = False + # Only allow owners and admins to edit torrents + can_edit = viewer and (viewer is torrent.user or viewer.is_admin) files = None if torrent.filelist: @@ -596,6 +597,7 @@ def view_torrent(torrent_id): return flask.render_template('view.html', torrent=torrent, files=files, + viewer=viewer, can_edit=can_edit) @@ -604,15 +606,18 @@ def edit_torrent(torrent_id): torrent = models.Torrent.by_id(torrent_id) form = forms.EditForm(flask.request.form) form.category.choices = _create_upload_category_choices() - category = str(torrent.main_category_id) + "_" + str(torrent.sub_category_id) + + editor = flask.g.user if not torrent: flask.abort(404) - if torrent.deleted and (not flask.g.user or not flask.g.user.is_admin): + # Only allow admins edit deleted torrents + if torrent.deleted and not (editor and editor.is_admin): flask.abort(404) - if not flask.g.user or (flask.g.user is not torrent.user and not flask.g.user.is_admin): + # Only allow torrent owners or admins edit torrents + if not editor or not (editor is torrent.user or editor.is_admin): flask.abort(403) if flask.request.method == 'POST' and form.validate(): @@ -622,36 +627,43 @@ def edit_torrent(torrent_id): torrent.display_name = (form.display_name.data or '').strip() torrent.information = (form.information.data or '').strip() torrent.description = (form.description.data or '').strip() - if flask.g.user.is_admin: - torrent.deleted = form.is_deleted.data + torrent.hidden = form.is_hidden.data torrent.remake = form.is_remake.data torrent.complete = form.is_complete.data torrent.anonymous = form.is_anonymous.data + if editor.is_trusted: + torrent.trusted = form.is_trusted.data + if editor.is_admin: + torrent.deleted = form.is_deleted.data + db.session.commit() flask.flash(flask.Markup( 'Torrent has been successfully edited! Changes might take a few minutes to show up.'), 'info') - return flask.redirect('/view/' + str(torrent_id)) + return flask.redirect(flask.url_for('view_torrent', torrent_id=torrent.id)) else: - # Setup form with pre-formatted form. - form.category.data = category - form.display_name.data = torrent.display_name - form.information.data = torrent.information - form.description.data = torrent.description - form.is_hidden.data = torrent.hidden - if flask.g.user.is_admin: + if flask.request.method != 'POST': + # Fill form data only if the POST didn't fail + form.category.data = torrent.sub_category.id_as_string + form.display_name.data = torrent.display_name + form.information.data = torrent.information + form.description.data = torrent.description + + form.is_hidden.data = torrent.hidden + form.is_remake.data = torrent.remake + form.is_complete.data = torrent.complete + form.is_anonymous.data = torrent.anonymous + + form.is_trusted.data = torrent.trusted form.is_deleted.data = torrent.deleted - form.is_remake.data = torrent.remake - form.is_complete.data = torrent.complete - form.is_anonymous.data = torrent.anonymous return flask.render_template('edit.html', form=form, torrent=torrent, - admin=flask.g.user.is_admin) + editor=editor) @app.route('/view//magnet') diff --git a/nyaa/templates/_formhelpers.html b/nyaa/templates/_formhelpers.html index a84581e..2588d04 100644 --- a/nyaa/templates/_formhelpers.html +++ b/nyaa/templates/_formhelpers.html @@ -1,10 +1,12 @@ -{% macro render_field(field) %} +{% macro render_field(field, render_label=True) %} {% if field.errors %}
{% else %}
{% endif %} + {% if render_label %} {{ field.label(class='control-label') }} + {% endif %} {{ field(title=field.description,**kwargs) | safe }} {% if field.errors %}
@@ -27,33 +29,33 @@ {% macro render_markdown_editor(field, field_name='') %} {% if field.errors %} -
+
{% else %} -
+
{% endif %} -
- -
-
- {{ render_field(field, class_='form-control markdown-source') }} -
-
- {{ field.label(class='control-label') }} -
-
-
-
+
+ {{ field.label(class='control-label') }} + +
+
+ {{ render_field(field, False, class_='form-control markdown-source') }} +
+
+
+
+
+
{% endmacro %} diff --git a/nyaa/templates/edit.html b/nyaa/templates/edit.html index 6ea01d4..d35b752 100644 --- a/nyaa/templates/edit.html +++ b/nyaa/templates/edit.html @@ -4,79 +4,73 @@ {% from "_formhelpers.html" import render_field %} {% from "_formhelpers.html" import render_markdown_editor %} -

Edit Torrent

+{% set torrent_url = url_for('view_torrent', torrent_id=torrent.id) %} +

+ Edit Torrent #{{torrent.id}} + {% if (torrent.user != None) and (torrent.user != editor) %} + (by {{ torrent.user.username }}) + {% endif %} +

{{ form.csrf_token }} +
-
- {{ render_field(form.category, class_='form-control')}} +
+ {{ render_field(form.display_name, class_='form-control', placeholder='Display name') }} +
+
+ {{ render_field(form.category, class_='form-control')}}
-
-
- {{ render_field(form.display_name, class_='form-control', placeholder='Display name') }} +
+ {{ render_field(form.information, class_='form-control', placeholder='Your website or IRC channel') }}
-
+
+ +
+ {% if editor.is_admin %} + + {% endif %} -
-
- {{ render_field(form.information, class_='form-control', placeholder='Your website or IRC channel') }} -
-
- -
-
- {{ render_markdown_editor(form.description, field_name='description') }} -
-
- - {% if admin %} -
-
- -
-
- {% endif %} - -
-
-
-
- -
-
-
-
- -
-
-
-
- +
+ {{ render_markdown_editor(form.description, field_name='description') }}
diff --git a/nyaa/templates/upload.html b/nyaa/templates/upload.html index 0a49282..803fec4 100644 --- a/nyaa/templates/upload.html +++ b/nyaa/templates/upload.html @@ -16,68 +16,57 @@ {% if config.ENFORCE_MAIN_ANNOUNCE_URL %}

Important: Please include {{config.MAIN_ANNOUNCE_URL}} in your trackers

{% endif %}
-
- {{ render_upload(form.torrent_file, accept=".torrent") }} +
+ {{ render_upload(form.torrent_file, accept=".torrent") }}
-
-
- {{ render_field(form.category, class_='form-control')}} +
+ {{ render_field(form.display_name, class_='form-control', placeholder='Display name') }} +
+
+ {{ render_field(form.category, class_='form-control')}}
-
-
- {{ render_field(form.display_name, class_='form-control', placeholder='Display name') }} -
+
- -
-
+
+
{{ render_field(form.information, class_='form-control', placeholder='Your website or IRC channel') }}
-
- -
-
- {{ render_markdown_editor(form.description, field_name='description') }} -
-
- -
-
-
+
+
+ {{ render_markdown_editor(form.description, field_name='description') }}
diff --git a/nyaa/templates/view.html b/nyaa/templates/view.html index 6c835b8..cfc0530 100644 --- a/nyaa/templates/view.html +++ b/nyaa/templates/view.html @@ -5,7 +5,7 @@

{% if can_edit %} - + {% endif %} {{ torrent.display_name }}

@@ -23,7 +23,14 @@
Submitter:
-
{% if not torrent.anonymous and torrent.user %}{{ torrent.user.username }}{% else %}Anonymous{% endif %}
+
+ {% set user_url = torrent.user and url_for('view_user', user_name=torrent.user.username) %} + {%- if not torrent.anonymous and torrent.user -%} + {{ torrent.user.username }} + {%- else -%} + Anonymous {% if torrent.user and (viewer == torrent.user or viewer.is_admin) %}({{ torrent.user.username }}){% endif %} + {%- endif -%} +
Seeders:
{% if config.ENABLE_SHOW_STATS %}{{ torrent.stats.seed_count }}{% else %}Coming soon{% endif %}
From af0cca2f8c936cdde98f0db0d03ea80ae416e8ce Mon Sep 17 00:00:00 2001 From: TheAMM Date: Sat, 20 May 2017 22:50:40 +0300 Subject: [PATCH 14/21] Display full category names on upload/edit category lists --- nyaa/routes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nyaa/routes.py b/nyaa/routes.py index f41f591..23a68d2 100644 --- a/nyaa/routes.py +++ b/nyaa/routes.py @@ -555,7 +555,8 @@ def _create_upload_category_choices(): cat_names = id_map[key] is_main_cat = key.endswith('_0') - cat_name = is_main_cat and cat_names[0] or (' - ' + cat_names[1]) + # cat_name = is_main_cat and cat_names[0] or (' - ' + cat_names[1]) + cat_name = ' - '.join(cat_names) choices.append((key, cat_name, is_main_cat)) return choices From f31efcdaa1c6ad7e8898f2e93690f2c2f6573e66 Mon Sep 17 00:00:00 2001 From: TheAMM Date: Sat, 20 May 2017 22:50:49 +0300 Subject: [PATCH 15/21] Align fields better, move Anonymous as first flag --- nyaa/templates/edit.html | 4 ++-- nyaa/templates/upload.html | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/nyaa/templates/edit.html b/nyaa/templates/edit.html index d35b752..44dfd51 100644 --- a/nyaa/templates/edit.html +++ b/nyaa/templates/edit.html @@ -25,10 +25,10 @@
-
+
{{ render_field(form.information, class_='form-control', placeholder='Your website or IRC channel') }}
-
+
{% if editor.is_admin %} diff --git a/nyaa/templates/upload.html b/nyaa/templates/upload.html index 803fec4..778a37d 100644 --- a/nyaa/templates/upload.html +++ b/nyaa/templates/upload.html @@ -32,12 +32,16 @@
-
+
{{ render_field(form.information, class_='form-control', placeholder='Your website or IRC channel') }}
-
+
+ - {% if user.is_trusted %}