From 1bc36c5a17241bdcb26d2906bc4072340c64cc11 Mon Sep 17 00:00:00 2001 From: Anna-Maria Meriniemi Date: Sun, 30 Jul 2017 00:00:39 +0300 Subject: [PATCH] [Schema change] Add webseed support (BEP-19) (#317) Store webseeds in Trackers table with is_webseed flag Adjusts torrent creation accordingly --- ...ffd23e570f92_add_is_webseed_to_trackers.py | 24 ++++++++++++++++ nyaa/backend.py | 22 +++++++++++++++ nyaa/forms.py | 12 ++++++++ nyaa/models.py | 1 + nyaa/torrents.py | 28 ++++++++++++++----- 5 files changed, 80 insertions(+), 7 deletions(-) create mode 100644 migrations/versions/ffd23e570f92_add_is_webseed_to_trackers.py diff --git a/migrations/versions/ffd23e570f92_add_is_webseed_to_trackers.py b/migrations/versions/ffd23e570f92_add_is_webseed_to_trackers.py new file mode 100644 index 0000000..61db4b7 --- /dev/null +++ b/migrations/versions/ffd23e570f92_add_is_webseed_to_trackers.py @@ -0,0 +1,24 @@ +"""Add is_webseed to Trackers + +Revision ID: ffd23e570f92 +Revises: 1add911660a6 +Create Date: 2017-07-29 19:03:58.244769 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'ffd23e570f92' +down_revision = '1add911660a6' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('trackers', sa.Column('is_webseed', sa.Boolean(), nullable=False)) + + +def downgrade(): + op.drop_column('trackers', 'is_webseed') diff --git a/nyaa/backend.py b/nyaa/backend.py index e075681..2b76c8a 100644 --- a/nyaa/backend.py +++ b/nyaa/backend.py @@ -139,6 +139,10 @@ def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False): for announce in announce_list: trackers.add(announce[0].decode('ascii')) + # Store webseeds + webseed_list = torrent_data.torrent_dict.get('url-list', []) + webseeds = OrderedSet(webseed.decode('utf-8') for webseed in webseed_list) + # Remove our trackers, maybe? TODO ? # Search for/Add trackers in DB @@ -151,9 +155,27 @@ def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False): tracker = models.Trackers(uri=announce) db.session.add(tracker) db.session.flush() + elif tracker.is_webseed: + # If we have an announce marked webseed (user error, malicy?), reset it. + # Better to have "bad" announces than "hiding" proper announces in webseeds/url-list. + tracker.is_webseed = False + db.session.flush() db_trackers.add(tracker) + # Same for webseeds + for webseed_url in webseeds: + webseed = models.Trackers.by_uri(webseed_url) + + if not webseed: + webseed = models.Trackers(uri=webseed_url, is_webseed=True) + db.session.add(webseed) + db.session.flush() + + # Don't add trackers into webseeds + if webseed.is_webseed: + db_trackers.add(webseed) + # Store tracker refs in DB for order, tracker in enumerate(db_trackers): torrent_tracker = models.TorrentTrackers(torrent_id=torrent.id, diff --git a/nyaa/forms.py b/nyaa/forms.py index bb8b5a7..1f62fcb 100644 --- a/nyaa/forms.py +++ b/nyaa/forms.py @@ -343,6 +343,16 @@ def _validate_trackers(torrent_dict, tracker_to_check_for=None): return tracker_found +# http://www.bittorrent.org/beps/bep_0019.html +def _validate_webseeds(torrent_dict): + webseed_list = torrent_dict.get('url-list') + if webseed_list is not None: + _validate_list(webseed_list, 'url-list') + + for webseed_url in webseed_list: + _validate_bytes(webseed_url, 'url-list item', test_decode='utf-8') + + def _validate_torrent_metadata(torrent_dict): ''' Validates a torrent metadata dict, raising AssertionError on errors ''' assert isinstance(torrent_dict, dict), 'torrent metadata is not a dict' @@ -384,6 +394,8 @@ def _validate_torrent_metadata(torrent_dict): length = info_dict.get('length') _validate_number(length, 'length', check_positive=True) + _validate_webseeds(torrent_dict) + def _validate_bytes(value, name='value', check_empty=True, test_decode=None): assert isinstance(value, bytes), name + ' is not bytes' diff --git a/nyaa/models.py b/nyaa/models.py index 3ebba06..dc6b6af 100644 --- a/nyaa/models.py +++ b/nyaa/models.py @@ -327,6 +327,7 @@ class Trackers(db.Model): id = db.Column(db.Integer, primary_key=True) uri = db.Column(db.String(length=255, collation=COL_UTF8_GENERAL_CI), nullable=False, unique=True) + is_webseed = db.Column(db.Boolean, nullable=False, default=False) disabled = db.Column(db.Boolean, nullable=False, default=False) @classmethod diff --git a/nyaa/torrents.py b/nyaa/torrents.py index d4b39c0..963e072 100644 --- a/nyaa/torrents.py +++ b/nyaa/torrents.py @@ -37,8 +37,9 @@ def default_trackers(): return USED_TRACKERS[:] -def get_trackers(torrent): +def get_trackers_and_webseeds(torrent): trackers = OrderedSet() + webseeds = OrderedSet() # Our main one first main_announce_url = app.config.get('MAIN_ANNOUNCE_URL') @@ -46,14 +47,20 @@ def get_trackers(torrent): trackers.add(main_announce_url) # then the user ones - torrent_trackers = torrent.trackers + torrent_trackers = torrent.trackers # here be webseeds too for torrent_tracker in torrent_trackers: - trackers.add(torrent_tracker.tracker.uri) + tracker = torrent_tracker.tracker + + # separate potential webseeds + if tracker.is_webseed: + webseeds.add(tracker.uri) + else: + trackers.add(tracker.uri) # and finally our tracker list trackers.update(default_trackers()) - return list(trackers) + return list(trackers), list(webseeds) def get_default_trackers(): @@ -103,9 +110,12 @@ def create_magnet_from_es_info(): return dict(create_magnet_from_es_info=_create_magnet_from_es_info) -def create_default_metadata_base(torrent, trackers=None): - if trackers is None: - trackers = get_trackers(torrent) +def create_default_metadata_base(torrent, trackers=None, webseeds=None): + if trackers is None or webseeds is None: + db_trackers, db_webseeds = get_trackers_and_webseeds(torrent) + + trackers = db_trackers if trackers is None else trackers + webseeds = db_webseeds if webseeds is None else webseeds metadata_base = { 'created by': 'NyaaV2', @@ -120,6 +130,10 @@ def create_default_metadata_base(torrent, trackers=None): # Yes, it's a list of lists with a single element inside. metadata_base['announce-list'] = [[tracker] for tracker in trackers[:MAX_TRACKERS]] + # Add webseeds + if webseeds: + metadata_base['url-list'] = webseeds + return metadata_base