[Config change] Upload ratelimit for non-trusted uploaders (#384)

* Implement upload ratelimit for non-trusted uploaders

Users may upload X torrents in Y minutes after which they
will have to wait Z minutes between uploads.

* Show torrent period count when ratelimited

* Only ratelimit new accounts

config.example.py

@@ -90,6 +90,15 @@ TRACKER_API_AUTH = 'topsecret'
 ## Account ##
 #############
 
+# Limit torrent upload rate
+RATELIMIT_UPLOADS = True
+RATELIMIT_ACCOUNT_AGE = 7 * 24 * 3600
+# After uploading MAX_UPLOAD_BURST torrents within UPLOAD_BURST_DURATION,
+# the following uploads must be at least UPLOAD_TIMEOUT seconds after the previous upload.
+MAX_UPLOAD_BURST = 5
+UPLOAD_BURST_DURATION = 45 * 60
+UPLOAD_TIMEOUT = 15 * 60
+
 # Torrents uploaded without an account must be at least this big in total (bytes)
 # Set to 0 to disable
 MINIMUM_ANONYMOUS_TORRENT_SIZE = 1 * 1024 * 1024

nyaa/backend.py

@@ -1,5 +1,6 @@
 import json
 import os
+from datetime import datetime, timedelta
 from ipaddress import ip_address
 from urllib.parse import urlencode
 from urllib.request import urlopen
@@ -7,6 +8,7 @@ from urllib.request import urlopen
 import flask
 from werkzeug import secure_filename
 
+import sqlalchemy
 from orderedset import OrderedSet
 
 from nyaa import models, utils
@@ -16,7 +18,7 @@ app = flask.current_app
 
 
 class TorrentExtraValidationException(Exception):
-    def __init__(self, errors):
+    def __init__(self, errors={}):
         self.errors = errors
 
 
@@ -105,6 +107,40 @@ def validate_torrent_post_upload(torrent, upload_form=None):
         raise TorrentExtraValidationException(errors)
 
 
+def check_uploader_ratelimit(user):
+    ''' Figures out if user (or IP address from flask.request) may
+        upload within upload ratelimit.
+        Returns a tuple of current datetime, count of torrents uploaded
+        within burst duration and timestamp for next allowed upload. '''
+    now = datetime.utcnow()
+    next_allowed_time = now
+
+    Torrent = models.Torrent
+
+    def filter_uploader(query):
+        if user:
+            return query.filter(Torrent.user == user)
+        else:
+            return query.filter(Torrent.uploader_ip == ip_address(flask.request.remote_addr).packed)
+
+    time_range_start = datetime.utcnow() - timedelta(seconds=app.config['UPLOAD_BURST_DURATION'])
+    # Count torrents uploaded by user/ip within given time period
+    torrent_count_query = db.session.query(sqlalchemy.func.count(Torrent.id))
+    torrent_count = filter_uploader(torrent_count_query).filter(
+        Torrent.created_time >= time_range_start).scalar()
+
+    # If user has reached burst limit...
+    if torrent_count >= app.config['MAX_UPLOAD_BURST']:
+        # Check how long ago their latest torrent was (we know at least one will exist)
+        last_torrent = filter_uploader(Torrent.query).order_by(Torrent.created_time.desc()).first()
+        after_timeout = last_torrent.created_time + timedelta(seconds=app.config['UPLOAD_TIMEOUT'])
+
+        if now < after_timeout:
+            next_allowed_time = after_timeout
+
+    return now, torrent_count, next_allowed_time
+
+
 def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False):
     ''' Stores a torrent to the database.
         May throw TorrentExtraValidationException if the form/torrent fails
@@ -112,6 +148,18 @@ def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False):
         relevant fields on the given form. '''
     torrent_data = upload_form.torrent_file.parsed_data
 
+    # Anonymous uploaders and non-trusted uploaders
+    no_or_new_account = (not uploading_user
+                         or (uploading_user.age < app.config['RATELIMIT_ACCOUNT_AGE']
+                             and not uploading_user.is_trusted))
+
+    if app.config['RATELIMIT_UPLOADS'] and no_or_new_account:
+        now, torrent_count, next_time = check_uploader_ratelimit(uploading_user)
+        if next_time > now:
+            # This will flag the dialog in upload.html red and tell API users what's wrong
+            upload_form.ratelimit.errors = ["You've gone over the upload ratelimit."]
+            raise TorrentExtraValidationException()
+
     # Delete exisiting torrent which is marked as deleted
     if torrent_data.db_id is not None:
         models.Torrent.query.filter_by(id=torrent_data.db_id).delete()

nyaa/forms.py

@@ -323,6 +323,8 @@ class UploadForm(FlaskForm):
         Length(max=10 * 1024, message='Description must be at most %(max)d characters long.')
     ])
 
+    ratelimit = HiddenField()
+
     def validate_torrent_file(form, field):
         # Decode and ensure data is bencoded data
         try:

nyaa/static/js/main.js

@@ -91,6 +91,7 @@ function _format_time_difference(seconds) {
 	if (seconds < 0) {
 		suffix = "";
 		prefix = "After ";
+		seconds = -seconds;
 	} else if (seconds == 0) {
 		return "Just now"
 	}

nyaa/template_utils.py

@@ -94,6 +94,13 @@ def get_utc_timestamp(datetime_str):
     return int((datetime.strptime(datetime_str, '%Y-%m-%dT%H:%M:%S') - UTC_EPOCH).total_seconds())
 
 
+@bp.app_template_filter('utc_timestamp')
+def get_utc_timestamp_seconds(datetime_instance):
+    """ Returns a UTC POSIX timestamp, as seconds """
+    UTC_EPOCH = datetime.utcfromtimestamp(0)
+    return int((datetime_instance - UTC_EPOCH).total_seconds())
+
+
 @bp.app_template_filter('display_time')
 def get_display_time(datetime_str):
     return datetime.strptime(datetime_str, '%Y-%m-%dT%H:%M:%S').strftime('%Y-%m-%d %H:%M')

nyaa/templates/upload.html

@@ -20,6 +20,23 @@
 	{% if config.ENFORCE_MAIN_ANNOUNCE_URL %}<p><strong>Important:</strong> Please include <kbd>{{ config.MAIN_ANNOUNCE_URL }}</kbd> in your trackers.</p>{% endif %}
 	<p><strong>Important:</strong> Make sure you have read <strong><a href="{{ url_for('site.rules') }}">the rules</a></strong> before uploading!</p>
 	<br>
+
+	{% if show_ratelimit %}
+	{% set ratelimit_class = 'danger' if upload_form.ratelimit.errors else 'warning' %}
+	<div class="row">
+		<div class="col-md-12">
+			<div class="alert alert-{{ ratelimit_class }}" role="alert">
+				<div>You've reached your maximum upload rate ({{ config.MAX_UPLOAD_BURST }} per {{ config.UPLOAD_BURST_DURATION // 60}} minutes, you have <b>{{ ratelimit_count }}</b>) and will now have to wait {{ config.UPLOAD_TIMEOUT // 60 }} minutes between uploads.</div>
+				{% if next_upload_time %}
+				<div>You may upload again at <b data-timestamp="{{ next_upload_time|utc_timestamp }}">{{ next_upload_time.strftime('%Y-%m-%d %H:%M UTC') }}</b>.</div>
+				{% else %}
+				<div>You may upload again now.</div>
+				{% endif %}
+			</div>
+		</div>
+	</div>
+	{% endif %}
+
 	<div class="row">
 		<div class="col-md-12">
 		{{ render_upload(upload_form.torrent_file, accept=".torrent") }}

nyaa/utils.py

@@ -37,7 +37,6 @@ def cached_function(f):
     @functools.wraps(f)
     def decorator(*args, **kwargs):
         if f._cached_value is sentinel:
-            print('Evaluating', f, args, kwargs)
             f._cached_value = f(*args, **kwargs)
         return f._cached_value
     return decorator

nyaa/views/torrents.py

@@ -387,6 +387,21 @@ def upload():
     upload_form = forms.UploadForm(CombinedMultiDict((flask.request.files, flask.request.form)))
     upload_form.category.choices = _create_upload_category_choices()
 
+    show_ratelimit = False
+    next_upload_time = None
+    ratelimit_count = 0
+
+    # Anonymous uploaders and non-trusted uploaders
+
+    no_or_new_account = (not flask.g.user
+                         or (flask.g.user.age < app.config['RATELIMIT_ACCOUNT_AGE']
+                             and not flask.g.user.is_trusted))
+
+    if app.config['RATELIMIT_UPLOADS'] and no_or_new_account:
+        now, ratelimit_count, next_upload_time = backend.check_uploader_ratelimit(flask.g.user)
+        show_ratelimit = ratelimit_count >= app.config['MAX_UPLOAD_BURST']
+        next_upload_time = next_upload_time if next_upload_time > now else None
+
     if flask.request.method == 'POST' and upload_form.validate():
         try:
             torrent = backend.handle_torrent_upload(upload_form, flask.g.user)
@@ -397,7 +412,11 @@ def upload():
 
     # If we get here with a POST, it means the form data was invalid: return a non-okay status
     status_code = 400 if flask.request.method == 'POST' else 200
-    return flask.render_template('upload.html', upload_form=upload_form), status_code
+    return flask.render_template('upload.html',
+                                 upload_form=upload_form,
+                                 show_ratelimit=show_ratelimit,
+                                 ratelimit_count=ratelimit_count,
+                                 next_upload_time=next_upload_time), status_code
 
 
 @cached_function