Nyaa development helper (tool) (#324)

Add new tool for developing (lint/autopep8/isort/test)

New tool uses flake8 and isort for lint checks.
Deprecate existing tool (still works)
Update readme
Update Travis config

.travis.yml

@@ -20,12 +20,12 @@ install:
   - pip install -r requirements.txt
   - pip install pytest-cov
   - sed "s/mysql:\/\/test:test123@/mysql:\/\/root:@/" config.example.py > config.py
-  - python db_create.py
+  - ./db_create.py
   - ./db_migrate.py stamp head
 
 script:
-  - pytest --cov=nyaa --cov-report=term tests
+  - ./dev.py test --cov=nyaa --cov-report=term tests
-  - ./lint.sh --check
+  - ./dev.py lint
 
 notifications:
   email: false

README.md

@@ -6,14 +6,14 @@ This guide also assumes you 1) are using Linux and 2) are somewhat capable with
 It's not impossible to run Nyaa on Windows, but this guide doesn't focus on that.
 
 ### Code Quality:
-- Before we get any deeper, remember to follow PEP8 style guidelines and run `./lint.sh` before committing.
+- Before we get any deeper, remember to follow PEP8 style guidelines and run `./dev.py lint` before committing to see a list of warnings/problems.
-    - You may also use `./lint.sh -c` to see a list of warnings/problems instead of having `lint.sh` making modifications for you
+    - You may also use `./dev.py fix && ./dev.py isort` to automatically fix some of the issues reported by the previous command.
 - Other than PEP8, try to keep your code clean and easy to understand, as well. It's only polite!
 
 ### Running Tests
 The `tests` folder contains tests for the the `nyaa` module and the webserver. To run the tests:
 - Make sure that you are in the python virtual environment.
-- Run `pytest tests` while in the repository directory.
+- Run `./dev.py test` while in the repository directory.
 
 ### Setting up Pyenv
 pyenv eases the use of different Python versions, and as not all Linux distros offer 3.6 packages, it's right up our alley.

dev.py

@@ -0,0 +1,159 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+This tool is designed to assist developers run common tasks, such as
+checking the code for lint issues, auto fixing some lint issues and running tests.
+It imports modules lazily (as-needed basis), so it runs faster!
+"""
+import sys
+
+LINT_PATHS = [
+    'nyaa/',
+    'utils/',
+]
+TEST_PATHS = ['tests']
+
+
+def print_cmd(cmd, args):
+    """ Prints the command and args as you would run them manually. """
+    print('Running: {0}\n'.format(
+        ' '.join([('\'' + a + '\'' if ' ' in a else a) for a in [cmd] + args])))
+    sys.stdout.flush()  # Make sure stdout is flushed before continuing.
+
+
+def check_config_values():
+    """ Verify that all max_line_length values match. """
+    import configparser
+    config = configparser.ConfigParser()
+    config.read('setup.cfg')
+
+    # Max line length:
+    flake8 = config.get('flake8', 'max_line_length', fallback=None)
+    autopep8 = config.get('pycodestyle', 'max_line_length', fallback=None)
+    isort = config.get('isort', 'line_length', fallback=None)
+
+    values = (v for v in (flake8, autopep8, isort) if v is not None)
+    found = next(values, False)
+    if not found:
+        print('Warning: No max line length setting set in setup.cfg.')
+        return False
+    elif any(v != found for v in values):
+        print('Warning: Max line length settings differ in setup.cfg.')
+        return False
+
+    return True
+
+
+def print_help():
+    print('Nyaa Development Helper')
+    print('=======================\n')
+    print('Usage: {0} command [different arguments]'.format(sys.argv[0]))
+    print('Command can be one of the following:\n')
+    print('  lint | check       : do a lint check (flake8 + flake8-isort)')
+    print('  fix  | autolint    : try and auto-fix lint (autopep8)')
+    print('  isort              : fix import sorting (isort)')
+    print('  test | pytest      : run tests (pytest)')
+    print('  help | -h | --help : show this help and exit')
+    print('')
+    print('You may pass different arguments to the script that is being run.')
+    print('For example: {0} test tests/ --verbose'.format(sys.argv[0]))
+    print('')
+    return 1
+
+
+if __name__ == '__main__':
+    assert sys.version_info >= (3, 6), "Python 3.6 is required"
+
+    check_config_values()
+
+    if len(sys.argv) < 2:
+        sys.exit(print_help())
+
+    cmd = sys.argv[1].lower()
+    if cmd in ('help', '-h', '--help'):
+        sys.exit(print_help())
+
+    args = sys.argv[2:]
+    run_default = not (args or set(('--version', '-h', '--help')).intersection(args))
+
+    # Flake8 - lint and common errors checker
+    # When combined with flake8-isort, also checks for unsorted imports.
+    if cmd in ('lint', 'check'):
+        if run_default:
+            # Putting format in the setup.cfg file breaks `pip install flake8`
+            settings = ['--format', '%(path)s [%(row)s:%(col)s] %(code)s: %(text)s',
+                        '--show-source']
+            args = LINT_PATHS + settings + args
+
+        print_cmd('flake8', args)
+        try:
+            from flake8.main.application import Application as Flake8
+        except ImportError as err:
+            print('Unable to load module: {0!r}'.format(err))
+            result = False
+        else:
+            f8 = Flake8()
+            f8.run(args)
+            result = f8.result_count == 0
+
+            if not result:
+                print("The code requires some changes.")
+            else:
+                print("Looks good!")
+        finally:
+            sys.exit(int(not result))
+
+    # AutoPEP8 - auto code linter for most simple errors.
+    if cmd in ('fix', 'autolint'):
+        if run_default:
+            args = LINT_PATHS + args
+
+        print_cmd('autopep8', args)
+        try:
+            from autopep8 import main as autopep8
+        except ImportError as err:
+            print('Unable to load module: {0!r}'.format(err))
+            result = False
+        else:
+            args = [''] + args  # Workaround
+            result = autopep8(args)
+        finally:
+            sys.exit(result)
+
+    # isort - automate import sorting.
+    if cmd in ('isort', ):
+        if run_default:
+            args = LINT_PATHS + ['-rc'] + args
+
+        print_cmd('isort', args)
+        try:
+            from isort.main import main as isort
+        except ImportError as err:
+            print('Unable to load module: {0!r}'.format(err))
+            result = False
+        else:
+            # Need to patch sys.argv for argparse in isort
+            sys.argv.remove(cmd)
+            sys.argv = [sys.argv[0] + ' ' + cmd] + args
+            result = isort()
+        finally:
+            sys.exit(result)
+
+    # py.test - test runner
+    if cmd in ('test', 'pytest'):
+        if run_default:
+            args = TEST_PATHS + args
+
+        print_cmd('pytest', args)
+        try:
+            from pytest import main as pytest
+        except ImportError as err:
+            print('Unable to load module: {0!r}'.format(err))
+            result = False
+        else:
+            result = pytest(args)
+            result = result == 0
+        finally:
+            sys.exit(int(not result))
+
+    sys.exit(print_help())

lint.sh

@@ -1,39 +1,27 @@
+#!/bin/bash
 # Lint checker/fixer
+# This script is deprecated, but still works.
 
-check_paths="nyaa/ utils/"
+function auto_fix() {
-isort_paths="nyaa/" # just nyaa/ for now
+  ./dev.py fix && ./dev.py isort
-max_line_length=100
-
-function auto_pep8() {
-  autopep8 ${check_paths} \
-    --recursive \
-    --in-place \
-    --pep8-passes 2000 \
-    --max-line-length ${max_line_length} \
-    --verbose \
-  && \
-  isort ${isort_paths} \
-    --recursive
 }
 
+
 function check_lint() {
-  pycodestyle ${check_paths} \
+  ./dev.py lint
-    --show-source \
-    --max-line-length=${max_line_length} \
-    --format '%(path)s [%(row)s:%(col)s] %(code)s: %(text)s' \
-  && \
-  isort ${isort_paths} \
-    --recursive \
-    --diff \
-    --check-only
 }
 
 # MAIN
-action=auto_pep8 # default action
+action=auto_fix # default action
 for arg in "$@"
 do
   case "$arg" in
   "-h" | "--help")
+    echo "+ ========================= +"
+    echo "+ This script is deprecated +"
+    echo "+    Please use ./dev.py    +"
+    echo "+ ========================= +"
+    echo ""
     echo "Lint checker/fixer"
     echo ""
     echo "Usage: $0 [-c|--check] [-h|--help]"
@@ -49,14 +37,4 @@ do
 done
 
 ${action} # run selected action
-result=$?
+if [[ $? -ne 0 ]]; then exit 1; fi
-
-if [[ ${action} == check_lint ]]; then
-  if [[ ${result} == 0 ]]; then
-    echo "Looks good!"
-  else
-    echo "The code requires some changes."
-  fi
-fi
-
-if [[ ${result} -ne 0 ]]; then exit 1; fi

requirements.txt

@@ -8,6 +8,8 @@ click==6.7
 dominate==2.3.1
 elasticsearch==5.3.0
 elasticsearch-dsl==5.2.0
+flake8==3.3.0
+flake8-isort==2.2.1
 Flask==0.12.2
 Flask-Assets==0.12
 Flask-DebugToolbar==0.10.1
@@ -18,8 +20,8 @@ Flask-SQLAlchemy==2.2
 Flask-WTF==0.14.2
 gevent==1.2.1
 greenlet==0.4.12
-itsdangerous==0.24
 isort==4.2.15
+itsdangerous==0.24
 Jinja2==2.9.6
 libsass==0.12.3
 Mako==1.0.6

setup.cfg

@@ -1,3 +1,16 @@
+[flake8]
+max_line_length = 100
+# The following line raises an exception on `pip install flake8`
+# So we're using the command line argument instead.
+# format = %(path)s [%(row)s:%(col)s] %(code)s: %(text)s
+
+[pep8]
+max_line_length = 100
+pep8_passes = 2000
+in_place = 1
+recursive = 1
+verbose = 1
+
 [isort]
 line_length = 100
 not_skip = __init__.py