[Schema change] Add webseed support (BEP-19) (#317)

Store webseeds in Trackers table with is_webseed flag
Adjusts torrent creation accordingly

migrations/versions/ffd23e570f92_add_is_webseed_to_trackers.py

@@ -0,0 +1,24 @@
+"""Add is_webseed to Trackers
+
+Revision ID: ffd23e570f92
+Revises: 1add911660a6
+Create Date: 2017-07-29 19:03:58.244769
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'ffd23e570f92'
+down_revision = '1add911660a6'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('trackers', sa.Column('is_webseed', sa.Boolean(), nullable=False))
+
+
+def downgrade():
+    op.drop_column('trackers', 'is_webseed')

nyaa/backend.py

@@ -139,6 +139,10 @@ def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False):
     for announce in announce_list:
         trackers.add(announce[0].decode('ascii'))
 
+    # Store webseeds
+    webseed_list = torrent_data.torrent_dict.get('url-list', [])
+    webseeds = OrderedSet(webseed.decode('utf-8') for webseed in webseed_list)
+
     # Remove our trackers, maybe? TODO ?
 
     # Search for/Add trackers in DB
@@ -151,9 +155,27 @@ def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False):
             tracker = models.Trackers(uri=announce)
             db.session.add(tracker)
             db.session.flush()
+        elif tracker.is_webseed:
+            # If we have an announce marked webseed (user error, malicy?), reset it.
+            # Better to have "bad" announces than "hiding" proper announces in webseeds/url-list.
+            tracker.is_webseed = False
+            db.session.flush()
 
         db_trackers.add(tracker)
 
+    # Same for webseeds
+    for webseed_url in webseeds:
+        webseed = models.Trackers.by_uri(webseed_url)
+
+        if not webseed:
+            webseed = models.Trackers(uri=webseed_url, is_webseed=True)
+            db.session.add(webseed)
+            db.session.flush()
+
+        # Don't add trackers into webseeds
+        if webseed.is_webseed:
+            db_trackers.add(webseed)
+
     # Store tracker refs in DB
     for order, tracker in enumerate(db_trackers):
         torrent_tracker = models.TorrentTrackers(torrent_id=torrent.id,

nyaa/forms.py

@@ -343,6 +343,16 @@ def _validate_trackers(torrent_dict, tracker_to_check_for=None):
     return tracker_found
 
 
+# http://www.bittorrent.org/beps/bep_0019.html
+def _validate_webseeds(torrent_dict):
+    webseed_list = torrent_dict.get('url-list')
+    if webseed_list is not None:
+        _validate_list(webseed_list, 'url-list')
+
+        for webseed_url in webseed_list:
+            _validate_bytes(webseed_url, 'url-list item', test_decode='utf-8')
+
+
 def _validate_torrent_metadata(torrent_dict):
     ''' Validates a torrent metadata dict, raising AssertionError on errors '''
     assert isinstance(torrent_dict, dict), 'torrent metadata is not a dict'
@@ -384,6 +394,8 @@ def _validate_torrent_metadata(torrent_dict):
         length = info_dict.get('length')
         _validate_number(length, 'length', check_positive=True)
 
+    _validate_webseeds(torrent_dict)
+
 
 def _validate_bytes(value, name='value', check_empty=True, test_decode=None):
     assert isinstance(value, bytes), name + ' is not bytes'

nyaa/models.py

@@ -327,6 +327,7 @@ class Trackers(db.Model):
     id = db.Column(db.Integer, primary_key=True)
     uri = db.Column(db.String(length=255, collation=COL_UTF8_GENERAL_CI),
                     nullable=False, unique=True)
+    is_webseed = db.Column(db.Boolean, nullable=False, default=False)
     disabled = db.Column(db.Boolean, nullable=False, default=False)
 
     @classmethod

nyaa/torrents.py

@@ -37,8 +37,9 @@ def default_trackers():
     return USED_TRACKERS[:]
 
 
-def get_trackers(torrent):
+def get_trackers_and_webseeds(torrent):
     trackers = OrderedSet()
+    webseeds = OrderedSet()
 
     # Our main one first
     main_announce_url = app.config.get('MAIN_ANNOUNCE_URL')
@@ -46,14 +47,20 @@ def get_trackers(torrent):
         trackers.add(main_announce_url)
 
     # then the user ones
-    torrent_trackers = torrent.trackers
+    torrent_trackers = torrent.trackers  # here be webseeds too
     for torrent_tracker in torrent_trackers:
-        trackers.add(torrent_tracker.tracker.uri)
+        tracker = torrent_tracker.tracker
+
+        # separate potential webseeds
+        if tracker.is_webseed:
+            webseeds.add(tracker.uri)
+        else:
+            trackers.add(tracker.uri)
 
     # and finally our tracker list
     trackers.update(default_trackers())
 
-    return list(trackers)
+    return list(trackers), list(webseeds)
 
 
 def get_default_trackers():
@@ -103,9 +110,12 @@ def create_magnet_from_es_info():
     return dict(create_magnet_from_es_info=_create_magnet_from_es_info)
 
 
-def create_default_metadata_base(torrent, trackers=None):
+def create_default_metadata_base(torrent, trackers=None, webseeds=None):
-    if trackers is None:
+    if trackers is None or webseeds is None:
-        trackers = get_trackers(torrent)
+        db_trackers, db_webseeds = get_trackers_and_webseeds(torrent)
+
+        trackers = db_trackers if trackers is None else trackers
+        webseeds = db_webseeds if webseeds is None else webseeds
 
     metadata_base = {
         'created by': 'NyaaV2',
@@ -120,6 +130,10 @@ def create_default_metadata_base(torrent, trackers=None):
         # Yes, it's a list of lists with a single element inside.
         metadata_base['announce-list'] = [[tracker] for tracker in trackers[:MAX_TRACKERS]]
 
+    # Add webseeds
+    if webseeds:
+        metadata_base['url-list'] = webseeds
+
     return metadata_base