diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..495b891 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,18 @@ +language: python + +python: + - 3.6 + +dist: xenial +sudo: false + +cache: pip + +install: + - pip install --upgrade pycodestyle + +script: + - pycodestyle nyaa/ --show-source --max-line-length=100 + +notifications: + email: false diff --git a/README.md b/README.md index 843d0ac..ab46758 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,6 @@ # NyaaV2 -## Setup: - -- Create your virtualenv, for example with `pyvenv venv` -- Enter your virtualenv with `source venv/bin/activate` -- Install dependencies with `pip install -r requirements.txt` -- Run `python db_create.py` to create the database -- Start the dev server with `python run.py` - -## Updated Setup (python 3.6.1): +## Setup - Install dependencies https://github.com/pyenv/pyenv/wiki/Common-build-problems - Install `pyenv` https://github.com/pyenv/pyenv/blob/master/README.md#installation @@ -20,7 +12,7 @@ - Copy `config.example.py` into `config.py` - Change TABLE_PREFIX to `nyaa_` or `sukebei_` depending on the site -## Setting up MySQL/MariaDB database for advanced functionality +### Setting up MySQL/MariaDB database for advanced functionality - Enable `USE_MYSQL` flag in config.py - Install latest mariadb by following instructions here https://downloads.mariadb.org/mariadb/repositories/ - Tested versions: `mysql Ver 15.1 Distrib 10.0.30-MariaDB, for debian-linux-gnu (x86_64) using readline 5.2` @@ -35,18 +27,18 @@ - `CREATE DATABASE nyaav2 DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;` - `SOURCE ~/path/to/database/nyaa_maria_vx.sql` -## Finishing up +### Finishing up - Run `python db_create.py` to create the database - Load the .sql file - `mysql -u user -p nyaav2` - `SOURCE cocks.sql` - Remember to change the default user password to an empty string to disable logging in - Start the dev server with `python run.py` -- Deactivate `source deactivate` +- When you are finished developing, deactivate your virtualenv with `source deactivate` -# Enabling ElasticSearch +## Enabling ElasticSearch -## Basics +### Basics - Install jdk `sudo apt-get install openjdk-8-jdk` - Install elasticsearch https://www.elastic.co/guide/en/elasticsearch/reference/current/deb.html - `sudo systemctl enable elasticsearch.service` @@ -54,7 +46,7 @@ - Run `curl -XGET 'localhost:9200'` and make sure ES is running - Optional: install Kabana as a search frontend for ES -## Enable MySQL Binlogging +### Enable MySQL Binlogging - Add the `[mariadb]` bin-log section to my.cnf and reload mysql server - Connect to mysql - `SHOW VARIABLES LIKE 'binlog_format';` @@ -62,7 +54,7 @@ - Connect to root user - `GRANT REPLICATION SLAVE ON *.* TO 'test'@'localhost';` where test is the user you will be running `sync_es.py` with -## Setting up ES +### Setting up ES - Run `./create_es.sh` and this creates two indicies: `nyaa` and `sukebei` - The output should show `acknowledged: true` twice - The safest bet is to disable the webapp here to ensure there's no database writes @@ -70,7 +62,7 @@ - Run `python import_to_es.py` with `SITE_FLAVOR` set to `sukebei` - These will take some time to run as it's indexing -## Setting up sync_es.py +### Setting up sync_es.py - Sync_es.py keeps the ElasticSearch index updated by reading the BinLog - Configure the MySQL options with the user where you granted the REPLICATION permissions - Connect to MySQL, run `SHOW MASTER STATUS;`. @@ -78,9 +70,13 @@ - Set up `sync_es.py` as a service and run it, preferably as the system/root - Make sure `sync_es.py` runs within venv with the right dependencies -## Good to go! -- After that, enable the `USE_ELASTIC_SEARCH` flag and restart the webapp and you're good to go +Enable the `USE_ELASTIC_SEARCH` flag in `config.py`, restart the application, and you're good to go. +## Database migrations +- Uses [flask-Migrate](https://flask-migrate.readthedocs.io/) +- Run `./db_migrate.py db migrate` to generate the migration script after database model changes. +- Take a look at the result in `migrations/versions/...` to make sure nothing went wrong. +- Run `./db_migrate.py db upgrade` to upgrade your database. ## Code Quality: - Remember to follow PEP8 style guidelines and run `./lint.sh` before committing. diff --git a/config.example.py b/config.example.py index 73702b9..3b07da0 100644 --- a/config.example.py +++ b/config.example.py @@ -11,7 +11,7 @@ ENABLE_SHOW_STATS = False BASE_DIR = os.path.abspath(os.path.dirname(__file__)) if USE_MYSQL: - SQLALCHEMY_DATABASE_URI = ('mysql://test:test123@localhost/nyaav2') + SQLALCHEMY_DATABASE_URI = ('mysql://test:test123@localhost/nyaav2?charset=utf8mb4') else: SQLALCHEMY_DATABASE_URI = ( 'sqlite:///' + os.path.join(BASE_DIR, 'test.db') + '?check_same_thread=False') diff --git a/db_create.py b/db_create.py index 99c4a85..d881377 100644 --- a/db_create.py +++ b/db_create.py @@ -33,9 +33,3 @@ if not existing_cats: db.session.add(main_cat) db.session.commit() - -# Create fulltext index - -if app.config['USE_MYSQL']: - db.engine.execute('ALTER TABLE ' + app.config['TABLE_PREFIX'] + 'torrents ADD FULLTEXT KEY (display_name)') - diff --git a/db_migrate.py b/db_migrate.py new file mode 100644 index 0000000..8d4f8f0 --- /dev/null +++ b/db_migrate.py @@ -0,0 +1,13 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +from nyaa import app, db +from flask_script import Manager +from flask_migrate import Migrate, MigrateCommand + +migrate = Migrate(app, db) + +manager = Manager(app) +manager.add_command("db", MigrateCommand) + +if __name__ == "__main__": + manager.run() diff --git a/es_mapping.yml b/es_mapping.yml index 8659291..1fc72ad 100644 --- a/es_mapping.yml +++ b/es_mapping.yml @@ -21,6 +21,7 @@ settings: - resolution - lowercase - my_ngram + - word_delimit filter: my_ngram: type: edgeNGram @@ -28,7 +29,11 @@ settings: max_gram: 15 resolution: type: pattern_capture - patterns: ["(\\d+)x(\\d+)"] + patterns: ["(\\d+)[xX](\\d+)"] + word_delimit: + type: word_delimiter + preserve_original: true + split_on_numerics: false char_filter: my_char_filter: type: mapping diff --git a/migrations/README b/migrations/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/migrations/alembic.ini b/migrations/alembic.ini new file mode 100644 index 0000000..f8ed480 --- /dev/null +++ b/migrations/alembic.ini @@ -0,0 +1,45 @@ +# A generic, single database configuration. + +[alembic] +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 0000000..4593816 --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,87 @@ +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig +import logging + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) +logger = logging.getLogger('alembic.env') + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +from flask import current_app +config.set_main_option('sqlalchemy.url', + current_app.config.get('SQLALCHEMY_DATABASE_URI')) +target_metadata = current_app.extensions['migrate'].db.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure(url=url) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + # this callback is used to prevent an auto-migration from being generated + # when there are no changes to the schema + # reference: http://alembic.readthedocs.org/en/latest/cookbook.html + def process_revision_directives(context, revision, directives): + if getattr(config.cmd_opts, 'autogenerate', False): + script = directives[0] + if script.upgrade_ops.is_empty(): + directives[:] = [] + logger.info('No changes in schema detected.') + + engine = engine_from_config(config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + connection = engine.connect() + context.configure(connection=connection, + target_metadata=target_metadata, + process_revision_directives=process_revision_directives, + **current_app.extensions['migrate'].configure_args) + + try: + with context.begin_transaction(): + context.run_migrations() + finally: + connection.close() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 0000000..2c01563 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/3001f79b7722_add_torrents.uploader_ip.py b/migrations/versions/3001f79b7722_add_torrents.uploader_ip.py new file mode 100644 index 0000000..152c440 --- /dev/null +++ b/migrations/versions/3001f79b7722_add_torrents.uploader_ip.py @@ -0,0 +1,30 @@ +"""Add uploader_ip column to torrents table. + +Revision ID: 3001f79b7722 +Revises: +Create Date: 2017-05-21 18:01:35.472717 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '3001f79b7722' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('nyaa_torrents', sa.Column('uploader_ip', sa.Binary(), nullable=True)) + op.add_column('sukebei_torrents', sa.Column('uploader_ip', sa.Binary(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('nyaa_torrents', 'uploader_ip') + op.drop_column('sukebei_torrents', 'uploader_ip') + # ### end Alembic commands ### diff --git a/migrations/versions/d0eeb8049623_add_comments.py b/migrations/versions/d0eeb8049623_add_comments.py new file mode 100644 index 0000000..f6fab5b --- /dev/null +++ b/migrations/versions/d0eeb8049623_add_comments.py @@ -0,0 +1,48 @@ +"""Add comments table. + +Revision ID: d0eeb8049623 +Revises: 3001f79b7722 +Create Date: 2017-05-22 22:58:12.039149 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'd0eeb8049623' +down_revision = '3001f79b7722' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('nyaa_comments', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('torrent_id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.Column('created_time', sa.DateTime(), nullable=True), + sa.Column('text', sa.String(length=255, collation='utf8mb4_bin'), nullable=False), + sa.ForeignKeyConstraint(['torrent_id'], ['nyaa_torrents.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('sukebei_comments', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('torrent_id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.Column('created_time', sa.DateTime(), nullable=True), + sa.Column('text', sa.String(length=255, collation='utf8mb4_bin'), nullable=False), + sa.ForeignKeyConstraint(['torrent_id'], ['sukebei_torrents.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('nyaa_comments') + op.drop_table('sukebei_comments') + # ### end Alembic commands ### diff --git a/nyaa/__init__.py b/nyaa/__init__.py index 7e934cb..d8c04bf 100644 --- a/nyaa/__init__.py +++ b/nyaa/__init__.py @@ -12,6 +12,7 @@ app.config.from_object('config') # Database app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False +app.config['MYSQL_DATABASE_CHARSET'] = 'utf8mb4' # Don't refresh cookie each request app.config['SESSION_REFRESH_EACH_REQUEST'] = False @@ -37,7 +38,7 @@ if not app.config['DEBUG']: def internal_error(exception): app.logger.error(exception) flask.flash(flask.Markup( - 'An error occured! Debugging information has been logged.'), 'danger') + 'An error occurred! Debug information has been logged.'), 'danger') return flask.redirect('/') # Get git commit hash diff --git a/nyaa/api_handler.py b/nyaa/api_handler.py index 6638af7..65049b7 100644 --- a/nyaa/api_handler.py +++ b/nyaa/api_handler.py @@ -6,6 +6,9 @@ from nyaa import models, forms from nyaa import bencode, backend, utils from nyaa import torrents +# For _create_upload_category_choices +from nyaa import routes + import functools import json import os.path @@ -42,84 +45,7 @@ def api_require_user(f): return decorator -def validate_user(upload_request): - auth_info = None - try: - if 'auth_info' in upload_request.files: - auth_info = json.loads(upload_request.files['auth_info'].read().decode('utf-8')) - if 'username' not in auth_info.keys() or 'password' not in auth_info.keys(): - return False, None, None - - username = auth_info['username'] - password = auth_info['password'] - user = models.User.by_username(username) - - if not user: - user = models.User.by_email(username) - - if not user or password != user.password_hash or \ - user.status == models.UserStatusType.INACTIVE: - return False, None, None - - return True, user, None - else: - return False, None, None - - except Exception as e: - return False, None, e - - -def _create_upload_category_choices(): - ''' Turns categories in the database into a list of (id, name)s ''' - choices = [('', '[Select a category]')] - for main_cat in models.MainCategory.query.order_by(models.MainCategory.id): - choices.append((main_cat.id_as_string, main_cat.name, True)) - for sub_cat in main_cat.sub_categories: - choices.append((sub_cat.id_as_string, ' - ' + sub_cat.name)) - return choices - - # #################################### API ROUTES #################################### -def api_upload(upload_request, user): - form_info = None - try: - form_info = json.loads(upload_request.files['torrent_info'].read().decode('utf-8')) - - form_info_as_dict = [] - for k, v in form_info.items(): - if k in ['is_anonymous', 'is_hidden', 'is_remake', 'is_complete']: - if v: - form_info_as_dict.append((k, v)) - else: - form_info_as_dict.append((k, v)) - form_info = ImmutableMultiDict(form_info_as_dict) - except Exception as e: - return flask.make_response(flask.jsonify( - {'Failure': ['Invalid data. See HELP in api_uploader.py']}), 400) - - try: - torrent_file = upload_request.files['torrent_file'] - torrent_file = ImmutableMultiDict([('torrent_file', torrent_file)]) - except Exception as e: - return flask.make_response(flask.jsonify( - {'Failure': ['No torrent file was attached.']}), 400) - - form = forms.UploadForm(CombinedMultiDict((torrent_file, form_info))) - form.category.choices = _create_upload_category_choices() - - if upload_request.method == 'POST' and form.validate(): - torrent = backend.handle_torrent_upload(form, user, True) - - return flask.make_response(flask.jsonify({'Success': int('{0}'.format(torrent.id))}), 200) - else: - return_error_messages = [] - for error_name, error_messages in form.errors.items(): - return_error_messages.extend(error_messages) - - return flask.make_response(flask.jsonify({'Failure': return_error_messages}), 400) - -# V2 below - # Map UploadForm fields to API keys UPLOAD_API_FORM_KEYMAP = { @@ -134,19 +60,20 @@ UPLOAD_API_FORM_KEYMAP = { 'is_trusted': 'trusted' } UPLOAD_API_FORM_KEYMAP_REVERSE = {v: k for k, v in UPLOAD_API_FORM_KEYMAP.items()} -UPLOAD_API_KEYS = [ - 'name', - 'category', - 'anonymous', - 'hidden', - 'complete', - 'remake', - 'trusted', - 'information', - 'description' -] +UPLOAD_API_DEFAULTS = { + 'name': '', + 'category': '', + 'anonymous': False, + 'hidden': False, + 'complete': False, + 'remake': False, + 'trusted': True, + 'information': '', + 'description': '' +} +@api_blueprint.route('/upload', methods=['POST']) @api_blueprint.route('/v2/upload', methods=['POST']) @basic_auth_user @api_require_user @@ -158,16 +85,21 @@ def v2_api_upload(): request_data_field = flask.request.form.get('torrent_data') if request_data_field is None: return flask.jsonify({'errors': ['missing torrent_data field']}), 400 - request_data = json.loads(request_data_field) + + try: + request_data = json.loads(request_data_field) + except json.decoder.JSONDecodeError: + return flask.jsonify({'errors': ['unable to parse valid JSON in torrent_data']}), 400 # Map api keys to upload form fields - for key in UPLOAD_API_KEYS: + for key, default in UPLOAD_API_DEFAULTS.items(): mapped_key = UPLOAD_API_FORM_KEYMAP_REVERSE.get(key, key) - mapped_dict[mapped_key] = request_data.get(key) or '' + value = request_data.get(key, default) + mapped_dict[mapped_key] = value if value is not None else default # Flask-WTF (very helpfully!!) automatically grabs the request form, so force a None formdata - upload_form = forms.UploadForm(None, data=mapped_dict) - upload_form.category.choices = _create_upload_category_choices() + upload_form = forms.UploadForm(None, data=mapped_dict, meta={'csrf': False}) + upload_form.category.choices = routes._create_upload_category_choices() if upload_form.validate(): torrent = backend.handle_torrent_upload(upload_form, flask.g.user) @@ -186,3 +118,139 @@ def v2_api_upload(): # Map errors back from form fields into the api keys mapped_errors = {UPLOAD_API_FORM_KEYMAP.get(k, k): v for k, v in upload_form.errors.items()} return flask.jsonify({'errors': mapped_errors}), 400 + + +# #################################### TEMPORARY #################################### + +from orderedset import OrderedSet # noqa: E402 + + +@api_blueprint.route('/ghetto_import', methods=['POST']) +def ghetto_import(): + if flask.request.remote_addr != '127.0.0.1': + return flask.error(403) + + torrent_file = flask.request.files.get('torrent') + + try: + torrent_dict = bencode.decode(torrent_file) + # field.data.close() + except (bencode.MalformedBencodeException, UnicodeError): + return 'Malformed torrent file', 500 + + try: + forms._validate_torrent_metadata(torrent_dict) + except AssertionError as e: + return 'Malformed torrent metadata ({})'.format(e.args[0]), 500 + + try: + tracker_found = forms._validate_trackers(torrent_dict) + except AssertionError as e: + return 'Malformed torrent trackers ({})'.format(e.args[0]), 500 + + bencoded_info_dict = bencode.encode(torrent_dict['info']) + info_hash = utils.sha1_hash(bencoded_info_dict) + + # Check if the info_hash exists already in the database + torrent = models.Torrent.by_info_hash(info_hash) + if not torrent: + return 'This torrent does not exists', 500 + + if torrent.has_torrent: + return 'This torrent already has_torrent', 500 + + # Torrent is legit, pass original filename and dict along + torrent_data = forms.TorrentFileData(filename=os.path.basename(torrent_file.filename), + torrent_dict=torrent_dict, + info_hash=info_hash, + bencoded_info_dict=bencoded_info_dict) + + # The torrent has been validated and is safe to access with ['foo'] etc - all relevant + # keys and values have been checked for (see UploadForm in forms.py for details) + info_dict = torrent_data.torrent_dict['info'] + + changed_to_utf8 = backend._replace_utf8_values(torrent_data.torrent_dict) + + torrent_filesize = info_dict.get('length') or sum( + f['length'] for f in info_dict.get('files')) + + # In case no encoding, assume UTF-8. + torrent_encoding = torrent_data.torrent_dict.get('encoding', b'utf-8').decode('utf-8') + + # Store bencoded info_dict + torrent.info = models.TorrentInfo(info_dict=torrent_data.bencoded_info_dict) + torrent.has_torrent = True + + # To simplify parsing the filelist, turn single-file torrent into a list + torrent_filelist = info_dict.get('files') + + used_path_encoding = changed_to_utf8 and 'utf-8' or torrent_encoding + + parsed_file_tree = dict() + if not torrent_filelist: + # If single-file, the root will be the file-tree (no directory) + file_tree_root = parsed_file_tree + torrent_filelist = [{'length': torrent_filesize, 'path': [info_dict['name']]}] + else: + # If multi-file, use the directory name as root for files + file_tree_root = parsed_file_tree.setdefault( + info_dict['name'].decode(used_path_encoding), {}) + + # Parse file dicts into a tree + for file_dict in torrent_filelist: + # Decode path parts from utf8-bytes + path_parts = [path_part.decode(used_path_encoding) for path_part in file_dict['path']] + + filename = path_parts.pop() + current_directory = file_tree_root + + for directory in path_parts: + current_directory = current_directory.setdefault(directory, {}) + + # Don't add empty filenames (BitComet directory) + if filename: + current_directory[filename] = file_dict['length'] + + parsed_file_tree = utils.sorted_pathdict(parsed_file_tree) + + json_bytes = json.dumps(parsed_file_tree, separators=(',', ':')).encode('utf8') + torrent.filelist = models.TorrentFilelist(filelist_blob=json_bytes) + + db.session.add(torrent) + db.session.flush() + + # Store the users trackers + trackers = OrderedSet() + announce = torrent_data.torrent_dict.get('announce', b'').decode('ascii') + if announce: + trackers.add(announce) + + # List of lists with single item + announce_list = torrent_data.torrent_dict.get('announce-list', []) + for announce in announce_list: + trackers.add(announce[0].decode('ascii')) + + # Remove our trackers, maybe? TODO ? + + # Search for/Add trackers in DB + db_trackers = OrderedSet() + for announce in trackers: + tracker = models.Trackers.by_uri(announce) + + # Insert new tracker if not found + if not tracker: + tracker = models.Trackers(uri=announce) + db.session.add(tracker) + db.session.flush() + + db_trackers.add(tracker) + + # Store tracker refs in DB + for order, tracker in enumerate(db_trackers): + torrent_tracker = models.TorrentTrackers(torrent_id=torrent.id, + tracker_id=tracker.id, order=order) + db.session.add(torrent_tracker) + + db.session.commit() + + return 'success' diff --git a/nyaa/backend.py b/nyaa/backend.py index 6be9b5d..c2b1fe1 100644 --- a/nyaa/backend.py +++ b/nyaa/backend.py @@ -1,3 +1,4 @@ +import flask from nyaa import app, db from nyaa import models, forms from nyaa import bencode, utils @@ -8,6 +9,7 @@ import json from werkzeug import secure_filename from collections import OrderedDict from orderedset import OrderedSet +from ipaddress import ip_address def _replace_utf8_values(dict_or_list): @@ -53,7 +55,8 @@ def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False): description=description, encoding=torrent_encoding, filesize=torrent_filesize, - user=uploading_user) + user=uploading_user, + uploader_ip=ip_address(flask.request.remote_addr).packed) # Store bencoded info_dict torrent.info = models.TorrentInfo(info_dict=torrent_data.bencoded_info_dict) @@ -69,7 +72,9 @@ def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False): torrent.complete = upload_form.is_complete.data # Copy trusted status from user if possible can_mark_trusted = uploading_user and uploading_user.is_trusted + # To do, automatically mark trusted if user is trusted unless user specifies otherwise torrent.trusted = upload_form.is_trusted.data if can_mark_trusted else False + # Set category ids torrent.main_category_id, torrent.sub_category_id = \ upload_form.category.parsed_data.get_category_ids() @@ -134,11 +139,10 @@ def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False): if not tracker: tracker = models.Trackers(uri=announce) db.session.add(tracker) + db.session.flush() db_trackers.add(tracker) - db.session.flush() - # Store tracker refs in DB for order, tracker in enumerate(db_trackers): torrent_tracker = models.TorrentTrackers(torrent_id=torrent.id, diff --git a/nyaa/forms.py b/nyaa/forms.py index 7197a61..b523111 100644 --- a/nyaa/forms.py +++ b/nyaa/forms.py @@ -1,3 +1,4 @@ +import flask from nyaa import db, app from nyaa.models import User from nyaa import bencode, utils, models @@ -15,6 +16,7 @@ from wtforms.widgets import Select as SelectWidget from wtforms.widgets import html_params, HTMLString from flask_wtf.recaptcha import RecaptchaField +from flask_wtf.recaptcha.validators import Recaptcha as RecaptchaValidator class Unique(object): @@ -35,7 +37,7 @@ class Unique(object): _username_validator = Regexp( - r'[a-zA-Z0-9_\-]+', + r'^[a-zA-Z0-9_\-]+$', message='Your username must only consist of alphanumerics and _- (a-zA-Z0-9_-)') @@ -124,11 +126,18 @@ class DisabledSelectField(SelectField): raise ValueError(self.gettext('Not a valid choice')) +class CommentForm(FlaskForm): + comment = TextAreaField('Make a comment', [ + Length(min=3, max=255, message='Comment must be at least %(min)d characters ' + 'long and %(max)d at most.'), + DataRequired() + ]) + + class EditForm(FlaskForm): display_name = StringField('Torrent display name', [ - Length(min=3, max=255, - message='Torrent display name must be at least %(min)d characters long ' - 'and %(max)d at most.') + Length(min=3, max=255, message='Torrent display name must be at least %(min)d characters ' + 'long and %(max)d at most.') ]) category = DisabledSelectField('Category') @@ -164,10 +173,6 @@ class EditForm(FlaskForm): class UploadForm(FlaskForm): - - class Meta: - csrf = False - torrent_file = FileField('Torrent file', [ FileRequired() ]) @@ -179,6 +184,16 @@ class UploadForm(FlaskForm): '%(max)d at most.') ]) + if app.config['USE_RECAPTCHA']: + # Captcha only for not logged in users + _recaptcha_validator = RecaptchaValidator() + + def _validate_recaptcha(form, field): + if not flask.g.user: + return UploadForm._recaptcha_validator(form, field) + + recaptcha = RecaptchaField(validators=[_validate_recaptcha]) + # category = SelectField('Category') category = DisabledSelectField('Category') @@ -263,7 +278,7 @@ class UploadForm(FlaskForm): class UserForm(FlaskForm): - user_class = DisabledSelectField('Change User Class') + user_class = SelectField('Change User Class') def validate_user_class(form, field): if not field.data: @@ -309,7 +324,8 @@ def _validate_trackers(torrent_dict, tracker_to_check_for=None): for announce in announce_list: _validate_list(announce, 'announce-list item') - announce_string = _validate_bytes(announce[0], 'announce-list item url', test_decode='utf-8') + announce_string = _validate_bytes( + announce[0], 'announce-list item url', test_decode='utf-8') if tracker_to_check_for and announce_string.lower() == tracker_to_check_for.lower(): tracker_found = True diff --git a/nyaa/models.py b/nyaa/models.py index 609786c..66d25e2 100644 --- a/nyaa/models.py +++ b/nyaa/models.py @@ -1,3 +1,4 @@ +import flask from enum import Enum, IntEnum from datetime import datetime, timezone from nyaa import app, db @@ -6,11 +7,13 @@ from sqlalchemy import func, ForeignKeyConstraint, Index from sqlalchemy_utils import ChoiceType, EmailType, PasswordType from werkzeug.security import generate_password_hash, check_password_hash from sqlalchemy_fulltext import FullText +from ipaddress import ip_address import re import base64 from markupsafe import escape as escape_markup -from urllib.parse import unquote as unquote_url +from urllib.parse import urlencode, unquote as unquote_url +from hashlib import md5 if app.config['USE_MYSQL']: from sqlalchemy.dialects import mysql @@ -61,6 +64,7 @@ class Torrent(db.Model): encoding = db.Column(db.String(length=32), nullable=False) flags = db.Column(db.Integer, default=0, nullable=False, index=True) uploader_id = db.Column(db.Integer, db.ForeignKey('users.id'), nullable=True) + uploader_ip = db.Column(db.Binary(length=16), default=None, nullable=True) has_torrent = db.Column(db.Boolean, nullable=False, default=False) created_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow, nullable=False) @@ -92,11 +96,13 @@ class Torrent(db.Model): info = db.relationship('TorrentInfo', uselist=False, cascade="all, delete-orphan", back_populates='torrent') filelist = db.relationship('TorrentFilelist', uselist=False, - cascade="all, delete-orphan", back_populates='torrent') + cascade="all, delete-orphan", back_populates='torrent') stats = db.relationship('Statistic', uselist=False, - cascade="all, delete-orphan", back_populates='torrent', lazy='joined') - trackers = db.relationship('TorrentTrackers', uselist=True, - cascade="all, delete-orphan", lazy='joined') + cascade="all, delete-orphan", back_populates='torrent', lazy='joined') + trackers = db.relationship('TorrentTrackers', uselist=True, cascade="all, delete-orphan", + lazy='joined', order_by='TorrentTrackers.order') + comments = db.relationship('Comment', uselist=True, + cascade="all, delete-orphan") def __repr__(self): return '<{0} #{1.id} \'{1.display_name}\' {1.filesize}b>'.format(type(self).__name__, self) @@ -138,6 +144,11 @@ class Torrent(db.Model): def magnet_uri(self): return create_magnet(self) + @property + def uploader_ip_string(self): + if self.uploader_ip: + return str(ip_address(self.uploader_ip)) + @property def anonymous(self): return self.flags & TorrentFlags.ANONYMOUS @@ -194,6 +205,11 @@ class Torrent(db.Model): def by_info_hash(cls, info_hash): return cls.query.filter_by(info_hash=info_hash).first() + @classmethod + def by_info_hash_hex(cls, info_hash_hex): + info_hash_bytes = bytearray.fromhex(info_hash_hex) + return cls.by_info_hash(info_hash_bytes) + class TorrentNameSearch(FullText, Torrent): __fulltext_columns__ = ('display_name',) @@ -310,10 +326,31 @@ class SubCategory(db.Model): return cls.query.get((sub_cat_id, main_cat_id)) +class Comment(db.Model): + __tablename__ = DB_TABLE_PREFIX + 'comments' + + id = db.Column(db.Integer, primary_key=True) + torrent_id = db.Column(db.Integer, db.ForeignKey( + DB_TABLE_PREFIX + 'torrents.id', ondelete='CASCADE'), nullable=False) + user_id = db.Column(db.Integer, db.ForeignKey('users.id', ondelete='CASCADE')) + created_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow) + text = db.Column(db.String(length=255, collation=COL_UTF8MB4_BIN), nullable=False) + + user = db.relationship('User', uselist=False, back_populates='comments', lazy="joined") + + def __repr__(self): + return '' % self.id + + @property + def created_utc_timestamp(self): + ''' Returns a UTC POSIX timestamp, as seconds ''' + return (self.created_time - UTC_EPOCH).total_seconds() + + class UserLevelType(IntEnum): REGULAR = 0 TRUSTED = 1 - ADMIN = 2 + MODERATOR = 2 SUPERADMIN = 3 @@ -339,8 +376,8 @@ class User(db.Model): last_login_date = db.Column(db.DateTime(timezone=False), default=None, nullable=True) last_login_ip = db.Column(db.Binary(length=16), default=None, nullable=True) - torrents = db.relationship('Torrent', back_populates='user', lazy="dynamic") - + torrents = db.relationship('Torrent', back_populates='user', lazy='dynamic') + comments = db.relationship('Comment', back_populates='user', lazy='dynamic') # session = db.relationship('Session', uselist=False, back_populates='user') def __init__(self, username, email, password): @@ -363,6 +400,39 @@ class User(db.Model): ] return all(checks) + def gravatar_url(self): + # from http://en.gravatar.com/site/implement/images/python/ + size = 120 + # construct the url + default_avatar = flask.url_for('static', filename='img/avatar/default.png', _external=True) + gravatar_url = 'https://www.gravatar.com/avatar/{}?{}'.format( + md5(self.email.encode('utf-8').lower()).hexdigest(), + urlencode({'d': default_avatar, 's': str(size)})) + return gravatar_url + + @property + def userlevel_str(self): + if self.level == UserLevelType.REGULAR: + return 'User' + elif self.level == UserLevelType.TRUSTED: + return 'Trusted' + elif self.level >= UserLevelType.MODERATOR: + return 'Moderator' + + @property + def userlevel_color(self): + if self.level == UserLevelType.REGULAR: + return 'default' + elif self.level == UserLevelType.TRUSTED: + return 'success' + elif self.level >= UserLevelType.MODERATOR: + return 'purple' + + @property + def ip_string(self): + if self.last_login_ip: + return str(ip_address(self.last_login_ip)) + @classmethod def by_id(cls, id): return cls.query.get(id) @@ -382,8 +452,8 @@ class User(db.Model): return cls.by_username(username_or_email) or cls.by_email(username_or_email) @property - def is_admin(self): - return self.level >= UserLevelType.ADMIN + def is_moderator(self): + return self.level >= UserLevelType.MODERATOR @property def is_superadmin(self): diff --git a/nyaa/routes.py b/nyaa/routes.py index b7a34e6..17daf99 100644 --- a/nyaa/routes.py +++ b/nyaa/routes.py @@ -7,11 +7,13 @@ from nyaa import torrents from nyaa import backend from nyaa import api_handler from nyaa.search import search_elastic, search_db +from sqlalchemy.orm import joinedload import config +import re import json from datetime import datetime, timedelta -import ipaddress +from ipaddress import ip_address import os.path import base64 from urllib.parse import quote @@ -35,6 +37,10 @@ SERACH_PAGINATE_DISPLAY_MSG = ('Displaying results {start}-{end} out of {total} 'what you were looking for.') +# For static_cachebuster +_static_cache = {} + + def redirect_url(): url = flask.request.args.get('next') or \ flask.request.referrer or \ @@ -44,6 +50,31 @@ def redirect_url(): return url +@app.template_global() +def static_cachebuster(static_filename): + ''' Adds a ?t= cachebuster to the given path, if the file exists. + Results are cached in memory and persist until app restart! ''' + # Instead of timestamps, we could use commit hashes (we already load it in __init__) + # But that'd mean every static resource would get cache busted. This lets unchanged items + # stay in the cache. + + if app.debug: + # Do not bust cache on debug (helps debugging) + return static_filename + + # Get file mtime if not already cached. + if static_filename not in _static_cache: + file_path = os.path.join(app.config['BASE_DIR'], 'nyaa', static_filename[1:]) + if os.path.exists(file_path): + file_mtime = int(os.path.getmtime(file_path)) + _static_cache[static_filename] = static_filename + '?t=' + str(file_mtime) + else: + # Throw a warning? + _static_cache[static_filename] = static_filename + + return _static_cache[static_filename] + + @app.template_global() def modify_query(**new_values): args = flask.request.args.copy() @@ -132,8 +163,6 @@ def get_category_id_map(): # Routes start here # -app.register_blueprint(api_handler.api_blueprint, url_prefix='/api') - def chain_get(source, *args): ''' Tries to return values from source by the given keys. Returns None if none match. @@ -145,6 +174,7 @@ def chain_get(source, *args): return value return None + @app.route('/rss', defaults={'rss': True}) @app.route('/', defaults={'rss': False}) def home(rss): @@ -180,6 +210,26 @@ def home(rss): flask.abort(404) user_id = user.id + special_results = { + 'first_word_user': None, + 'query_sans_user': None, + 'infohash_torrent': None + } + # Add advanced features to searches (but not RSS or user searches) + if search_term and not render_as_rss and not user_id: + # Check if the first word of the search is an existing user + user_word_match = re.match(r'^([a-zA-Z0-9_-]+) *(.*|$)', search_term) + if user_word_match: + special_results['first_word_user'] = models.User.by_username(user_word_match.group(1)) + special_results['query_sans_user'] = user_word_match.group(2) + + # Check if search is a 40-char torrent hash + infohash_match = re.match(r'(?i)^([a-f0-9]{40})$', search_term) + if infohash_match: + # Check for info hash in database + matched_torrent = models.Torrent.by_info_hash_hex(infohash_match.group(1)) + special_results['infohash_torrent'] = matched_torrent + query_args = { 'user': user_id, 'sort': sort_key or 'id', @@ -193,9 +243,17 @@ def home(rss): if flask.g.user: query_args['logged_in_user'] = flask.g.user - if flask.g.user.is_admin: # God mode + if flask.g.user.is_moderator: # God mode query_args['admin'] = True + infohash_torrent = special_results.get('infohash_torrent') + if infohash_torrent: + # infohash_torrent is only set if this is not RSS or userpage search + flask.flash(flask.Markup('You were redirected here because ' + 'the given hash matched this torrent.'), 'info') + # Redirect user from search to the torrent if we found one with the specific info_hash + return flask.redirect(flask.url_for('view_torrent', torrent_id=infohash_torrent.id)) + # If searching, we get results from elastic search use_elastic = app.config.get('USE_ELASTIC_SEARCH') if use_elastic and search_term: @@ -212,9 +270,12 @@ def home(rss): query_results = search_elastic(**query_args) if render_as_rss: - return render_rss('"{}"'.format(search_term), query_results, use_elastic=True, magnet_links=use_magnet_links) + return render_rss( + '"{}"'.format(search_term), query_results, + use_elastic=True, magnet_links=use_magnet_links) else: - rss_query_string = _generate_query_string(search_term, category, quality_filter, user_name) + rss_query_string = _generate_query_string( + search_term, category, quality_filter, user_name) max_results = min(max_search_results, query_results['hits']['total']) # change p= argument to whatever you change page_parameter to or pagination breaks pagination = Pagination(p=query_args['page'], per_page=results_per_page, @@ -225,7 +286,8 @@ def home(rss): pagination=pagination, torrent_query=query_results, search=query_args, - rss_filter=rss_query_string) + rss_filter=rss_query_string, + special_results=special_results) else: # If ES is enabled, default to db search for browsing if use_elastic: @@ -237,7 +299,8 @@ def home(rss): if render_as_rss: return render_rss('Home', query, use_elastic=False, magnet_links=use_magnet_links) else: - rss_query_string = _generate_query_string(search_term, category, quality_filter, user_name) + rss_query_string = _generate_query_string( + search_term, category, quality_filter, user_name) # Use elastic is always false here because we only hit this section # if we're browsing without a search term (which means we default to DB) # or if ES is disabled @@ -245,7 +308,8 @@ def home(rss): use_elastic=False, torrent_query=query, search=query_args, - rss_filter=rss_query_string) + rss_filter=rss_query_string, + special_results=special_results) @app.route('/user/', methods=['GET', 'POST']) @@ -255,22 +319,23 @@ def view_user(user_name): if not user: flask.abort(404) - if flask.g.user and flask.g.user.id != user.id: - admin = flask.g.user.is_admin - superadmin = flask.g.user.is_superadmin - else: - admin = False - superadmin = False + admin_form = None + if flask.g.user and flask.g.user.is_moderator and flask.g.user.level > user.level: + admin_form = forms.UserForm() + default, admin_form.user_class.choices = _create_user_class_choices(user) + if flask.request.method == 'GET': + admin_form.user_class.data = default - form = forms.UserForm() - form.user_class.choices = _create_user_class_choices() - if flask.request.method == 'POST' and form.validate(): - selection = form.user_class.data + if flask.request.method == 'POST' and admin_form and admin_form.validate(): + selection = admin_form.user_class.data if selection == 'regular': user.level = models.UserLevelType.REGULAR elif selection == 'trusted': user.level = models.UserLevelType.TRUSTED + elif selection == 'moderator': + user.level = models.UserLevelType.MODERATOR + db.session.add(user) db.session.commit() @@ -310,7 +375,7 @@ def view_user(user_name): if flask.g.user: query_args['logged_in_user'] = flask.g.user - if flask.g.user.is_admin: # God mode + if flask.g.user.is_moderator: # God mode query_args['admin'] = True # Use elastic search for term searching @@ -343,9 +408,7 @@ def view_user(user_name): user_page=True, rss_filter=rss_query_string, level=user_level, - admin=admin, - superadmin=superadmin, - form=form) + admin_form=admin_form) # Similar logic as home page else: if use_elastic: @@ -361,9 +424,7 @@ def view_user(user_name): user_page=True, rss_filter=rss_query_string, level=user_level, - admin=admin, - superadmin=superadmin, - form=form) + admin_form=admin_form) @app.template_filter('rfc822') @@ -416,7 +477,7 @@ def login(): return flask.redirect(flask.url_for('login')) user.last_login_date = datetime.utcnow() - user.last_login_ip = ipaddress.ip_address(flask.request.remote_addr).packed + user.last_login_ip = ip_address(flask.request.remote_addr).packed db.session.add(user) db.session.commit() @@ -450,7 +511,7 @@ def register(): if flask.request.method == 'POST' and form.validate(): user = models.User(username=form.username.data.strip(), email=form.email.data.strip(), password=form.password.data) - user.last_login_ip = ipaddress.ip_address(flask.request.remote_addr).packed + user.last_login_ip = ip_address(flask.request.remote_addr).packed db.session.add(user) db.session.commit() @@ -478,14 +539,6 @@ def profile(): form = forms.ProfileForm(flask.request.form) - level = 'Regular' - if flask.g.user.is_admin: - level = 'Moderator' - if flask.g.user.is_superadmin: # check this second because we can be admin AND superadmin - level = 'Administrator' - elif flask.g.user.is_trusted: - level = 'Trusted' - if flask.request.method == 'POST' and form.validate(): user = flask.g.user new_email = form.email.data.strip() @@ -515,12 +568,7 @@ def profile(): flask.g.user = user return flask.redirect('/profile') - _user = models.User.by_id(flask.g.user.id) - username = _user.username - current_email = _user.email - - return flask.render_template('profile.html', form=form, name=username, email=current_email, - level=level) + return flask.render_template('profile.html', form=form) @app.route('/user/activate/') @@ -562,34 +610,65 @@ def _create_upload_category_choices(): @app.route('/upload', methods=['GET', 'POST']) def upload(): - form = forms.UploadForm(CombinedMultiDict((flask.request.files, flask.request.form))) - form.category.choices = _create_upload_category_choices() + upload_form = forms.UploadForm(CombinedMultiDict((flask.request.files, flask.request.form))) + upload_form.category.choices = _create_upload_category_choices() - if flask.request.method == 'POST' and form.validate(): - torrent = backend.handle_torrent_upload(form, flask.g.user) + if flask.request.method == 'POST' and upload_form.validate(): + torrent = backend.handle_torrent_upload(upload_form, flask.g.user) return flask.redirect('/view/' + str(torrent.id)) else: # If we get here with a POST, it means the form data was invalid: return a non-okay status status_code = 400 if flask.request.method == 'POST' else 200 - return flask.render_template('upload.html', form=form, user=flask.g.user), status_code + return flask.render_template('upload.html', upload_form=upload_form), status_code -@app.route('/view/') +@app.route('/view/', methods=['GET', 'POST']) def view_torrent(torrent_id): - torrent = models.Torrent.by_id(torrent_id) - - viewer = flask.g.user - + if flask.request.method == 'POST': + torrent = models.Torrent.by_id(torrent_id) + else: + torrent = models.Torrent.query \ + .options(joinedload('filelist'), + joinedload('comments')) \ + .filter_by(id=torrent_id) \ + .first() if not torrent: flask.abort(404) # Only allow admins see deleted torrents - if torrent.deleted and not (viewer and viewer.is_admin): + if torrent.deleted and not (flask.g.user and flask.g.user.is_moderator): flask.abort(404) + comment_form = None + if flask.g.user: + comment_form = forms.CommentForm() + + if flask.request.method == 'POST': + if not flask.g.user: + flask.abort(403) + + if comment_form.validate(): + comment_text = (comment_form.comment.data or '').strip() + + comment = models.Comment( + torrent_id=torrent_id, + user_id=flask.g.user.id, + text=comment_text) + + db.session.add(comment) + db.session.commit() + + torrent_count = models.Comment.query.filter_by(torrent_id=torrent.id).count() + + flask.flash('Comment successfully posted.', 'success') + + return flask.redirect(flask.url_for('view_torrent', + torrent_id=torrent_id, + _anchor='com-' + str(torrent_count))) + # Only allow owners and admins to edit torrents - can_edit = viewer and (viewer is torrent.user or viewer.is_admin) + can_edit = flask.g.user and (flask.g.user is torrent.user or flask.g.user.is_moderator) files = None if torrent.filelist: @@ -598,11 +677,32 @@ def view_torrent(torrent_id): report_form = forms.ReportForm() return flask.render_template('view.html', torrent=torrent, files=files, - viewer=viewer, + comment_form=comment_form, + comments=torrent.comments, can_edit=can_edit, report_form=report_form) +@app.route('/view//comment//delete', methods=['POST']) +def delete_comment(torrent_id, comment_id): + if not flask.g.user: + flask.abort(403) + + comment = models.Comment.query.filter_by(id=comment_id).first() + if not comment: + flask.abort(404) + + if not (comment.user.id == flask.g.user.id or flask.g.user.is_moderator): + flask.abort(403) + + db.session.delete(comment) + db.session.commit() + + flask.flash('Comment successfully deleted.', 'success') + + return flask.redirect(flask.url_for('view_torrent', torrent_id=torrent_id)) + + @app.route('/view//edit', methods=['GET', 'POST']) def edit_torrent(torrent_id): torrent = models.Torrent.by_id(torrent_id) @@ -615,11 +715,11 @@ def edit_torrent(torrent_id): flask.abort(404) # Only allow admins edit deleted torrents - if torrent.deleted and not (editor and editor.is_admin): + if torrent.deleted and not (flask.g.user and flask.g.user.is_moderator): flask.abort(404) # Only allow torrent owners or admins edit torrents - if not editor or not (editor is torrent.user or editor.is_admin): + if not flask.g.user or not (flask.g.user is torrent.user or flask.g.user.is_moderator): flask.abort(403) if flask.request.method == 'POST' and form.validate(): @@ -635,15 +735,16 @@ def edit_torrent(torrent_id): torrent.complete = form.is_complete.data torrent.anonymous = form.is_anonymous.data - if editor.is_trusted: + if flask.g.user.is_trusted: torrent.trusted = form.is_trusted.data - if editor.is_admin: + if flask.g.user.is_moderator: torrent.deleted = form.is_deleted.data db.session.commit() flask.flash(flask.Markup( - 'Torrent has been successfully edited! Changes might take a few minutes to show up.'), 'info') + 'Torrent has been successfully edited! Changes might take a few minutes to show up.'), + 'info') return flask.redirect(flask.url_for('view_torrent', torrent_id=torrent.id)) else: @@ -664,8 +765,7 @@ def edit_torrent(torrent_id): return flask.render_template('edit.html', form=form, - torrent=torrent, - editor=editor) + torrent=torrent) @app.route('/view//magnet') @@ -679,15 +779,16 @@ def redirect_magnet(torrent_id): @app.route('/view//torrent') +@app.route('/download/.torrent') def download_torrent(torrent_id): torrent = models.Torrent.by_id(torrent_id) - if not torrent: + if not torrent or not torrent.has_torrent: flask.abort(404) resp = flask.Response(_get_cached_torrent_file(torrent)) resp.headers['Content-Type'] = 'application/x-bittorrent' - resp.headers['Content-Disposition'] = 'inline; filename*=UTF-8\'\'{}'.format( + resp.headers['Content-Disposition'] = 'inline; filename="{0}"; filename*=UTF-8\'\'{0}'.format( quote(torrent.torrent_name.encode('utf-8'))) return resp @@ -717,7 +818,7 @@ def submit_report(torrent_id): @app.route('/reports', methods=['GET', 'POST']) def view_reports(): - if not flask.g.user or not flask.g.user.is_admin: + if not flask.g.user or not flask.g.user.is_moderator: flask.abort(403) page = flask.request.args.get('p', flask.request.args.get('offset', 1, int), int) @@ -800,14 +901,55 @@ def send_verification_email(to_address, activ_link): server.quit() -def _create_user_class_choices(): +def _create_user_class_choices(user): choices = [('regular', 'Regular')] - if flask.g.user and flask.g.user.is_superadmin: - choices.append(('trusted', 'Trusted')) - return choices + default = 'regular' + if flask.g.user: + if flask.g.user.is_moderator: + choices.append(('trusted', 'Trusted')) + if flask.g.user.is_superadmin: + choices.append(('moderator', 'Moderator')) + if user: + if user.is_moderator: + default = 'moderator' + elif user.is_trusted: + default = 'trusted' + + return default, choices + + +@app.template_filter() +def timesince(dt, default='just now'): + """ + Returns string representing "time since" e.g. + 3 minutes ago, 5 hours ago etc. + Date and time (UTC) are returned if older than 1 day. + """ + + now = datetime.utcnow() + diff = now - dt + + periods = ( + (diff.days, 'day', 'days'), + (diff.seconds / 3600, 'hour', 'hours'), + (diff.seconds / 60, 'minute', 'minutes'), + (diff.seconds, 'second', 'seconds'), + ) + + if diff.days >= 1: + return dt.strftime('%Y-%m-%d %H:%M UTC') + else: + for period, singular, plural in periods: + + if period >= 1: + return '%d %s ago' % (period, singular if int(period) == 1 else plural) + + return default # #################################### STATIC PAGES #################################### + + @app.route('/rules', methods=['GET']) def site_rules(): return flask.render_template('rules.html') @@ -818,11 +960,11 @@ def site_help(): return flask.render_template('help.html') +@app.route('/xmlns/nyaa', methods=['GET']) +def xmlns_nyaa(): + return flask.render_template('xmlns.html') + + # #################################### API ROUTES #################################### -@app.route('/api/upload', methods=['POST']) -def api_upload(): - is_valid_user, user, debug = api_handler.validate_user(flask.request) - if not is_valid_user: - return flask.make_response(flask.jsonify({"Failure": "Invalid username or password."}), 400) - api_response = api_handler.api_upload(flask.request, user) - return api_response + +app.register_blueprint(api_handler.api_blueprint, url_prefix='/api') diff --git a/nyaa/static/css/bootstrap-xl-mod.css b/nyaa/static/css/bootstrap-xl-mod.css new file mode 100644 index 0000000..8187043 --- /dev/null +++ b/nyaa/static/css/bootstrap-xl-mod.css @@ -0,0 +1,277 @@ +/* +* CSS file with Bootstrap grid classes for screens bigger than 1600px. Just add this file after the Bootstrap CSS file and you will be able to juse col-xl, col-xl-push, hidden-xl, etc. +* +* Author: Marc van Nieuwenhuijzen +* Company: WebVakman +* Site: WebVakman.nl +* +* Edited to be for >=1480px with container width of 1400px for Nyaa.si +* Also edited to not fuck up column gutters. +*/ + +@media (min-width: 1200px) and (max-width: 1479px) { + .hidden-lg { + display: none !important; + } +} + + +.visible-xl-block, +.visible-xl-inline, +.visible-xl-inline-block, +.visible-xl{ + display: none !important; +} + + + +@media (min-width: 1480px) { + .container { + width: 1400px; + } + + .col-xl-1, .col-xl-2, .col-xl-3, .col-xl-4, .col-xl-5, .col-xl-6, .col-xl-7, .col-xl-8, .col-xl-9, .col-xl-10, .col-xl-11, .col-xl-12 { + position: relative; + min-height: 1px; + padding-right: 15px; + padding-left: 15px; + + float: left; + } + + .col-xl-12 { + width: 100%; + } + + .col-xl-11 { + width: 91.66666667%; + } + + .col-xl-10 { + width: 83.33333333%; + } + + .col-xl-9 { + width: 75%; + } + + .col-xl-8 { + width: 66.66666667%; + } + + .col-xl-7 { + width: 58.33333333%; + } + + .col-xl-6 { + width: 50%; + } + + .col-xl-5 { + width: 41.66666667%; + } + + .col-xl-4 { + width: 33.33333333%; + } + + .col-xl-3 { + width: 25%; + } + + .col-xl-2 { + width: 16.66666667%; + } + + .col-xl-1 { + width: 8.33333333%; + } + + .col-xl-pull-12 { + right: 100%; + } + + .col-xl-pull-11 { + right: 91.66666667%; + } + + .col-xl-pull-10 { + right: 83.33333333%; + } + + .col-xl-pull-9 { + right: 75%; + } + + .col-xl-pull-8 { + right: 66.66666667%; + } + + .col-xl-pull-7 { + right: 58.33333333%; + } + + .col-xl-pull-6 { + right: 50%; + } + + .col-xl-pull-5 { + right: 41.66666667%; + } + + .col-xl-pull-4 { + right: 33.33333333%; + } + + .col-xl-pull-3 { + right: 25%; + } + + .col-xl-pull-2 { + right: 16.66666667%; + } + + .col-xl-pull-1 { + right: 8.33333333%; + } + + .col-xl-pull-0 { + right: auto; + } + + .col-xl-push-12 { + left: 100%; + } + + .col-xl-push-11 { + left: 91.66666667%; + } + + .col-xl-push-10 { + left: 83.33333333%; + } + + .col-xl-push-9 { + left: 75%; + } + + .col-xl-push-8 { + left: 66.66666667%; + } + + .col-xl-push-7 { + left: 58.33333333%; + } + + .col-xl-push-6 { + left: 50%; + } + + .col-xl-push-5 { + left: 41.66666667%; + } + + .col-xl-push-4 { + left: 33.33333333%; + } + + .col-xl-push-3 { + left: 25%; + } + + .col-xl-push-2 { + left: 16.66666667%; + } + + .col-xl-push-1 { + left: 8.33333333%; + } + + .col-xl-push-0 { + left: auto; + } + + .col-xl-offset-12 { + margin-left: 100%; + } + + .col-xl-offset-11 { + margin-left: 91.66666667%; + } + + .col-xl-offset-10 { + margin-left: 83.33333333%; + } + + .col-xl-offset-9 { + margin-left: 75%; + } + + .col-xl-offset-8 { + margin-left: 66.66666667%; + } + + .col-xl-offset-7 { + margin-left: 58.33333333%; + } + + .col-xl-offset-6 { + margin-left: 50%; + } + + .col-xl-offset-5 { + margin-left: 41.66666667%; + } + + .col-xl-offset-4 { + margin-left: 33.33333333%; + } + + .col-xl-offset-3 { + margin-left: 25%; + } + + .col-xl-offset-2 { + margin-left: 16.66666667%; + } + + .col-xl-offset-1 { + margin-left: 8.33333333%; + } + + .col-xl-offset-0 { + margin-left: 0; + } + + .visible-xl { + display: block !important; + } + + table.visible-xl { + display: table; + } + + tr.visible-xl { + display: table-row !important; + } + + th.visible-xl, td.visible-xl { + display: table-cell !important; + } + + .visible-xl-block { + display: block !important; + } + + .visible-xl-inline { + display: inline !important; + } + + .visible-xl-inline-block { + display: inline-block !important; + } + + .hidden-xl { + display: none !important; + } +} diff --git a/nyaa/static/css/main.css b/nyaa/static/css/main.css index 07e5a6a..3368282 100644 --- a/nyaa/static/css/main.css +++ b/nyaa/static/css/main.css @@ -96,6 +96,10 @@ table.torrent-list tbody tr td a:visited { padding: 1em 0; } +.markdown-source { + min-height: 360px; +} + @media (max-width: 991px){ .panel-body .col-md-5 { margin-left: 20px; @@ -218,3 +222,106 @@ table.torrent-list tbody tr td a:visited { ul.nav-tabs#profileTabs { margin-bottom: 15px; } + +.comment-panel { + width: 99%; + margin: 0 auto; + margin-top: 10px; + margin-bottom: 10px; +} +.comment-panel:target { + border-color: black; + border-width: 2px; +} + +.text-purple, a.text-purple:visited { color: #a760bc; } +a.text-purple:hover, a.text-purple:active, a.text-purple:focus { color: #a760e0; } + +.comment-content { + word-break: break-all; +} +.comment-content img { + max-width: 100%; + max-height: 600px; +} + +.comment-box { + width: 95%; + margin: 0 auto; + margin-top: 30px; + margin-bottom: 10px; +} + +.delete-comment-form { + position: relative; + float: right; +} + +.avatar { + max-width: 120px; +} + +.btn-grey { + color: #000000; + background-color: #cccfd2; + border-color: #ccc; +} +.btn-grey:hover, .btn-grey:focus, .btn-grey:active, .btn-grey.active, .open > .dropdown-toggle.btn-grey { + background-color: #aaaaaa; +} +.btn span.glyphicon-check { + display: none; +} +.btn.active span.glyphicon-check { + display: inline; +} +.btn span.glyphicon-unchecked { + display: inline; +} +.btn.active span.glyphicon-unchecked { + display: none; +} +.center { + float: none; + margin: 0 auto; + text-align: center; +} + +/* Torrent file list */ +.torrent-file-list ul { + padding: 5px 20px 0px 20px; + list-style: none; + display: none; +} + +.torrent-file-list > ul { + display: block; /* First level always visible */ + padding: 0; + margin: 0; +} + +.torrent-file-list ul[data-show] { + /* Used to show first level's items based on amount */ + display: block; +} + +.torrent-file-list li:not(:last-of-type) { + margin-bottom: 5px; +} + +.torrent-file-list i.fa { + padding-right: 5px; +} + +.torrent-file-list i.fa-folder-open { + padding-right: 3px; +} + +.torrent-file-list a.folder { + font-weight: bold; + text-decoration: none; +} + +.torrent-file-list .file-size { + font-weight: bold; +} diff --git a/nyaa/static/img/avatar/default.png b/nyaa/static/img/avatar/default.png new file mode 100644 index 0000000..7fbef13 Binary files /dev/null and b/nyaa/static/img/avatar/default.png differ diff --git a/nyaa/static/js/main.js b/nyaa/static/js/main.js index 01e7067..9fd1a5d 100644 --- a/nyaa/static/js/main.js +++ b/nyaa/static/js/main.js @@ -43,6 +43,7 @@ $(document).ready(function() { } }); + // Drag & Drop zone for upload page $('body').on('dragenter', function(event) { event.preventDefault(); dropZone.css({ 'visibility': 'visible', 'opacity': 1 }); @@ -63,6 +64,13 @@ $(document).ready(function() { $('#torrent_file')[0].files = files; $(this).css({ 'visibility': 'hidden', 'opacity': 0 }); }); + + // Collapsible file lists + $('.torrent-file-list a.folder').click(function(e) { + e.preventDefault(); + $(this).blur().children('i').toggleClass('fa-folder-open fa-folder'); + $(this).next().stop().slideToggle(250); + }); }); function _format_time_difference(seconds) { @@ -80,6 +88,8 @@ function _format_time_difference(seconds) { if (seconds < 0) { suffix = ""; prefix = "After "; + } else if (seconds == 0) { + return "Just now" } var parts = []; @@ -96,11 +106,12 @@ function _format_time_difference(seconds) { } return prefix + parts.join(" ") + suffix; } -function _format_date(date) { +function _format_date(date, show_seconds) { var pad = function (n) { return ("00" + n).slice(-2); } var ymd = date.getFullYear() + "-" + pad(date.getMonth()+1) + "-" + pad(date.getDate()); var hm = pad(date.getHours()) + ":" + pad(date.getMinutes()); - return ymd + " " + hm; + var s = show_seconds ? ":" + pad(date.getSeconds()) : "" + return ymd + " " + hm + s; } // Add title text to elements with data-timestamp attribute @@ -111,11 +122,20 @@ document.addEventListener("DOMContentLoaded", function(event) { for (var i = 0; i < timestamp_targets.length; i++) { var target = timestamp_targets[i]; var torrent_timestamp = parseInt(target.getAttribute('data-timestamp')); + var swap_flag = target.getAttribute('data-timestamp-swap') != null; + if (torrent_timestamp) { var timedelta = now_timestamp - torrent_timestamp; - target.setAttribute('title', _format_time_difference(timedelta)); - target.innerText = _format_date(new Date(torrent_timestamp*1000)); + var formatted_date = _format_date(new Date(torrent_timestamp*1000), swap_flag); + var formatted_timedelta = _format_time_difference(timedelta); + if (swap_flag) { + target.setAttribute('title', formatted_date); + target.innerText = formatted_timedelta; + } else { + target.setAttribute('title', formatted_timedelta); + target.innerText = formatted_date; + } } }; diff --git a/nyaa/templates/404.html b/nyaa/templates/404.html index f964daf..d3fe555 100644 --- a/nyaa/templates/404.html +++ b/nyaa/templates/404.html @@ -1,5 +1,8 @@ {% extends "layout.html" %} {% block title %}404 Not Found :: {{ config.SITE_NAME }}{% endblock %} +{% block metatags %} + +{% endblock %} {% block body %}

404 Not Found

The path you requested does not exist on this server.

diff --git a/nyaa/templates/_formhelpers.html b/nyaa/templates/_formhelpers.html index 2588d04..02fb743 100644 --- a/nyaa/templates/_formhelpers.html +++ b/nyaa/templates/_formhelpers.html @@ -37,12 +37,12 @@ {{ field.label(class='control-label') }}
  • - + Preview
    • @@ -119,4 +119,4 @@ {% endif %} -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/nyaa/templates/edit.html b/nyaa/templates/edit.html index 44dfd51..07fc734 100644 --- a/nyaa/templates/edit.html +++ b/nyaa/templates/edit.html @@ -7,7 +7,7 @@ {% set torrent_url = url_for('view_torrent', torrent_id=torrent.id) %}

    Edit Torrent #{{torrent.id}} - {% if (torrent.user != None) and (torrent.user != editor) %} + {% if (torrent.user != None) and (torrent.user != g.user) %} (by {{ torrent.user.username }}) {% endif %}

    @@ -29,41 +29,54 @@ {{ render_field(form.information, class_='form-control', placeholder='Your website or IRC channel') }}
    - -
    - {% if editor.is_admin %} - - {% endif %} - - - - - - {# Only allow changing anonymous status when an uploader exists #} - {% if torrent.uploader_id %} - - {% endif %} - {% if editor.is_trusted %} - - {% endif %} +
    +
    + {# Only allow changing anonymous status when an uploader exists #} + {% if torrent.uploader_id %} + + {% endif %} + + {% if g.user.is_moderator %} + + {% endif %} +
    + +
    + + + {% if g.user.is_trusted %} + + {% endif %}
    diff --git a/nyaa/templates/home.html b/nyaa/templates/home.html index 8840da1..289716b 100644 --- a/nyaa/templates/home.html +++ b/nyaa/templates/home.html @@ -1,15 +1,22 @@ {% extends "layout.html" %} {% block title %}{% if search.term %}{{ search.term | e}}{% else %}Browse{% endif %} :: {{ config.SITE_NAME }}{% endblock %} +{% block metatags %} +{% if search.term %} + +{% else %} + +{% endif %} +{% endblock %} {% block body %} +{% if search["term"] == '' %}
    -

    5/18 Update: We've added an upload api for ease of uploading. See documentation here.

    -

    5/17 Update: We've added faster and more accurate search! In addition to your typical keyword search in both English and other languages, you can also now use powerful operators - like clockwork planet -horrible or commie|horrible|cartel yowamushi to search. For all supported operators, please click here. More features are coming soon!


    +

    2017-05-22 Update: We've added comments. You can change your avatar using Gravatar or if you don't like gravatar you can just stick with our spify default avatar.

    +

    2017-05-22 Update: We've updated our upload API to v2 (v1 is now disabled!). See documentation here.

    We welcome you to provide feedback at #nyaa-dev@irc.rizon.net

    Our GitHub: https://github.com/nyaadevs - creating issues for features and faults is recommendable!

    - +{% endif %} {% include "search_results.html" %} diff --git a/nyaa/templates/layout.html b/nyaa/templates/layout.html index 0b50d53..f4faac7 100644 --- a/nyaa/templates/layout.html +++ b/nyaa/templates/layout.html @@ -1,235 +1,309 @@ - - - - - {% block title %}{{ config.SITE_NAME }}{% endblock %} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - {% include "flashes.html" %} - - {% block body %}{% endblock %} -
    - - - - - - + + + + + {% block title %}{{ config.SITE_NAME }}{% endblock %} + + + + + + + + + + + + {% block metatags %} + {# Filled by children #} + {% endblock %} + + + + {# These are extracted here for the dark mode toggle #} + {% set bootstrap_light = static_cachebuster('/static/css/bootstrap.min.css') %} + {% set bootstrap_dark = static_cachebuster('/static/css/bootstrap-dark.min.css') %} + + + + + + + + + + + + + + + + + + + + + + + + + +
    + {% include "flashes.html" %} + + {% block body %}{% endblock %} +
    + + + + diff --git a/nyaa/templates/login.html b/nyaa/templates/login.html index fbdd202..5899a0d 100644 --- a/nyaa/templates/login.html +++ b/nyaa/templates/login.html @@ -1,5 +1,8 @@ {% extends "layout.html" %} {% block title %}Login :: {{ config.SITE_NAME }}{% endblock %} +{% block metatags %} + +{% endblock %} {% block body %} {% from "_formhelpers.html" import render_field %} diff --git a/nyaa/templates/profile.html b/nyaa/templates/profile.html index 827d11f..dcba274 100644 --- a/nyaa/templates/profile.html +++ b/nyaa/templates/profile.html @@ -3,17 +3,17 @@ {% block body %} {% from "_formhelpers.html" import render_field %} -

    Profile of {{ name }}

    +

    Profile of {{ g.user.username }}

    -
    -
{% elif files %}

-
Too many files to display.
+ Too many files to display.

@@ -125,13 +126,66 @@

-
File list is not available for this torrent.
+ File list is not available for this torrent.

{% endif %} -