Fix RFC822 filters + More tests (#257)

* Make rfc822 filters compatible with Windows systems.

.strftime() is relative to the system it's being run on.
UNIX has '%s' for seconds since the EPOCH, Windows doesn't (ValueError).
Solution: use .timestamp() to achieve the same result on both platforms.
This also allows us to drop the float() around it, since it returns a float.

* Start testing filters

* Add placeholders for more tests

* Make 'tests' folder a Python package

Now you can run tests with just `pytest tests`

* Update readme and travis config

* Test timesince()

* Update and organize .gitignore

Deleted: (nothing)
Added: Coverage files, .idea\

* Test filter_truthy, category_name

* Tests for backend.py

* Tests for bencode.py

* Move (empty) test_models.py to tests package

* Tests for utils.py

* Fixes for flattenDict

* Change name to `flatten_dict`
* `newkey` was assigned but never used

* Add a helper class for testing

* Show coverage on Travis

(only Travis for now...)

* Remove IDE

* Use correct assert functions

* Update README.md

.gitignore

@@ -1,12 +1,29 @@
-*.sql
-/.vscode
-/venv
-*.swp
+# Cache
 __pycache__
 /nyaa/static/.webassets-cache
+
+# Virtual Environments
+/venv
+
+# Coverage
+.coverage
+/htmlcov
+
+# Editors
+/.vscode
+
+# Databases
+*.sql
 test.db
-install/*
+
+# Webserver
 uwsgi.sock
-/test_torrent_batch
+
+# Application
+install/*
 config.py
+/test_torrent_batch
 torrents
+
+# Other
+*.swp

.travis.yml

@@ -22,12 +22,13 @@ before_install:
 
 install:
   - pip install -r requirements.txt
+  - pip install pytest-cov
   - sed "s/mysql:\/\/test:test123@/mysql:\/\/root:@/" config.example.py > config.py
   - python db_create.py
   - ./db_migrate.py stamp head
 
 script:
-  - python -m pytest tests/
+  - pytest --cov=nyaa --cov-report=term tests
   - ./lint.sh --check
 
 notifications:

README.md

@@ -11,9 +11,9 @@ It's not impossible to run Nyaa on Windows, but this guide doesn't focus on that
 - Other than PEP8, try to keep your code clean and easy to understand, as well. It's only polite!
 
 ### Running Tests
-We have some basic tests that check if each page can render correctly. To run the tests:
+The `tests` folder contains tests for the the `nyaa` module and the webserver. To run the tests:
 - Make sure that you are in the python virtual environment.
-- Run `python -m pytest tests/` while in the repository directory.
+- Run `pytest tests` while in the repository directory.
 
 ### Setting up Pyenv
 pyenv eases the use of different Python versions, and as not all Linux distros offer 3.6 packages, it's right up our alley.

nyaa/routes.py

@@ -434,12 +434,12 @@ def view_user(user_name):
 
 @app.template_filter('rfc822')
 def _jinja2_filter_rfc822(date, fmt=None):
-    return formatdate(float(date.strftime('%s')))
+    return formatdate(date.timestamp())
 
 
 @app.template_filter('rfc822_es')
-def _jinja2_filter_rfc822(datestr, fmt=None):
-    return formatdate(float(datetime.strptime(datestr, '%Y-%m-%dT%H:%M:%S').strftime('%s')))
+def _jinja2_filter_rfc822_es(datestr, fmt=None):
+    return formatdate(datetime.strptime(datestr, '%Y-%m-%dT%H:%M:%S').timestamp())
 
 
 def render_rss(label, query, use_elastic, magnet_links=False):

nyaa/utils.py

@@ -35,7 +35,7 @@ def cached_function(f):
     return decorator
 
 
-def flattenDict(d, result=None):
+def flatten_dict(d, result=None):
     if result is None:
         result = {}
     for key in d:
@@ -44,7 +44,7 @@ def flattenDict(d, result=None):
             value1 = {}
             for keyIn in value:
                 value1["/".join([key, keyIn])] = value[keyIn]
-            flattenDict(value1, result)
+            flatten_dict(value1, result)
         elif isinstance(value, (list, tuple)):
             for indexB, element in enumerate(value):
                 if isinstance(element, dict):
@@ -52,10 +52,10 @@ def flattenDict(d, result=None):
                     index = 0
                     for keyIn in element:
                         newkey = "/".join([key, keyIn])
-                        value1["/".join([key, keyIn])] = value[indexB][keyIn]
+                        value1[newkey] = value[indexB][keyIn]
                         index += 1
                     for keyA in value1:
-                        flattenDict(value1, result)
+                        flatten_dict(value1, result)
         else:
             result[key] = value
     return result

tests/__init__.py

@@ -0,0 +1,36 @@
+""" Sets up helper class for testing """
+
+import os
+import unittest
+
+from nyaa import app
+
+USE_MYSQL = True
+
+
+class NyaaTestCase(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        app.config['TESTING'] = True
+        cls.app_context = app.app_context()
+
+        # Use a seperate database for testing
+        # if USE_MYSQL:
+        #     cls.db_name = 'nyaav2_tests'
+        #     db_uri = 'mysql://root:@localhost/{}?charset=utf8mb4'.format(cls.db_name)
+        # else:
+        #     cls.db_name = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'test.db')
+        #     db_uri = 'sqlite:///{}?check_same_thread=False'.format(cls.db_name)
+
+        # if not os.environ.get('TRAVIS'):  # Travis doesn't need a seperate DB
+        #     app.config['USE_MYSQL'] = USE_MYSQL
+        #     app.config['SQLALCHEMY_DATABASE_URI'] = db_uri
+
+        with cls.app_context:
+            cls.app = app.test_client()
+
+    @classmethod
+    def tearDownClass(cls):
+        with cls.app_context:
+            pass

tests/test_api_handler.py

@@ -0,0 +1,34 @@
+import unittest
+import json
+
+from nyaa import api_handler, models
+from tests import NyaaTestCase
+from pprint import pprint
+
+
+class ApiHandlerTests(NyaaTestCase):
+
+    # @classmethod
+    # def setUpClass(cls):
+    #     super(ApiHandlerTests, cls).setUpClass()
+
+    # @classmethod
+    # def tearDownClass(cls):
+    #     super(ApiHandlerTests, cls).tearDownClass()
+
+    def test_no_authorization(self):
+        """ Test that API is locked unless you're logged in """
+        rv = self.app.get('/api/info/1')
+        data = json.loads(rv.get_data())
+        self.assertDictEqual({'errors': ['Bad authorization']}, data)
+
+    @unittest.skip('Not yet implemented')
+    def test_bad_credentials(self):
+        """ Test that API is locked unless you're logged in """
+        rv = self.app.get('/api/info/1')
+        data = json.loads(rv.get_data())
+        self.assertDictEqual({'errors': ['Bad authorization']}, data)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_backend.py

@@ -0,0 +1,46 @@
+import unittest
+
+from nyaa import backend
+
+
+class TestBackend(unittest.TestCase):
+
+    # def setUp(self):
+    #     self.db, nyaa.app.config['DATABASE'] = tempfile.mkstemp()
+    #     nyaa.app.config['TESTING'] = True
+    #     self.app = nyaa.app.test_client()
+    #     with nyaa.app.app_context():
+    #         nyaa.db.create_all()
+    #
+    # def tearDown(self):
+    #     os.close(self.db)
+    #     os.unlink(nyaa.app.config['DATABASE'])
+
+    def test_replace_utf8_values(self):
+        test_dict = {
+            'hash': '2346ad27d7568ba9896f1b7da6b5991251debdf2',
+            'title.utf-8': '¡hola! ¿qué tal?',
+            'filelist.utf-8': [
+                'Español 101.mkv',
+                'ру́сский 202.mp4'
+            ]
+        }
+        expected_dict = {
+            'hash': '2346ad27d7568ba9896f1b7da6b5991251debdf2',
+            'title': '¡hola! ¿qué tal?',
+            'filelist': [
+                'Español 101.mkv',
+                'ру́сский 202.mp4'
+            ]
+        }
+
+        self.assertTrue(backend._replace_utf8_values(test_dict))
+        self.assertDictEqual(test_dict, expected_dict)
+
+    @unittest.skip('Not yet implemented')
+    def test_handle_torrent_upload(self):
+        pass
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_bencode.py

@@ -0,0 +1,90 @@
+import unittest
+
+from nyaa import bencode
+
+
+class TestBencode(unittest.TestCase):
+
+    def test_pairwise(self):
+        # test list with even length
+        initial = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+        expected = [(0, 1), (2, 3), (4, 5), (6, 7), (8, 9)]
+
+        for index, values in enumerate(bencode._pairwise(initial)):
+            self.assertEqual(values, expected[index])
+
+        # test list with odd length
+        initial = [0, 1, 2, 3, 4]
+        expected = [(0, 1), (2, 3), 4]
+
+        for index, values in enumerate(bencode._pairwise(initial)):
+            self.assertEqual(values, expected[index])
+
+        # test non-iterable
+        initial = b'012345'
+        expected = [(48, 49), (50, 51), (52, 53)]  # decimal ASCII
+        for index, values in enumerate(bencode._pairwise(initial)):
+            self.assertEqual(values, expected[index])
+
+    def test_encode(self):
+        exception_test_cases = [  # (raw, raised_exception, expected_result_regexp)
+            # test unsupported type
+            (None, bencode.BencodeException,
+                r'Unsupported type'),
+            (1.6, bencode.BencodeException,
+                r'Unsupported type'),
+        ]
+
+        test_cases = [  # (raw, expected_result)
+            (100, b'i100e'),  # int
+            (-5, b'i-5e'),  # int
+            ('test', b'4:test'),  # str
+            (b'test', b'4:test'),  # byte
+            (['test', 100], b'l4:testi100ee'),  # list
+            ({'numbers': [1, 2], 'hello': 'world'}, b'd5:hello5:world7:numbersli1ei2eee')  # dict
+        ]
+
+        for raw, raised_exception, expected_result_regexp in exception_test_cases:
+            self.assertRaisesRegexp(raised_exception, expected_result_regexp, bencode.encode, raw)
+
+        for raw, expected_result in test_cases:
+            self.assertEqual(bencode.encode(raw), expected_result)
+
+    def test_decode(self):
+        exception_test_cases = [  # (raw, raised_exception, expected_result_regexp)
+            # test malformed bencode
+            (b'l4:hey', bencode.MalformedBencodeException,
+                r'Read only \d+ bytes, \d+ wanted'),
+            (b'ie', bencode.MalformedBencodeException,
+                r'Unable to parse int'),
+            (b'i64', bencode.MalformedBencodeException,
+                r'Unexpected end while reading an integer'),
+            (b'i6-4', bencode.MalformedBencodeException,
+                r'Unexpected input while reading an integer'),
+            (b'4#string', bencode.MalformedBencodeException,
+                r'Unexpected input while reading string length'),
+            (b'$:string', bencode.MalformedBencodeException,
+                r'Unexpected data type'),
+            (b'd5:world7:numbersli1ei2eee', bencode.MalformedBencodeException,
+                r'Uneven amount of key/value pairs'),
+        ]
+
+        test_cases = [  # (raw, expected_result)
+            (b'i100e', 100),  # int
+            (b'i-5e', -5),  # int
+            ('4:test', b'test'),  # str
+            (b'4:test', b'test'),  # byte
+            (b'15:thisisalongone!', b'thisisalongone!'),  # big byte
+            (b'l4:testi100ee', [b'test', 100]),  # list
+            (b'd5:hello5:world7:numbersli1ei2eee', {'hello': b'world', 'numbers': [1, 2]})  # dict
+        ]
+
+        for raw, raised_exception, expected_result_regexp in exception_test_cases:
+            self.assertRaisesRegexp(raised_exception, expected_result_regexp, bencode.decode, raw)
+
+        for raw, expected_result in test_cases:
+            self.assertEqual(bencode.decode(raw), expected_result)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_filters.py

@@ -0,0 +1,98 @@
+import unittest
+import datetime
+
+from email.utils import formatdate
+
+from tests import NyaaTestCase
+from nyaa.routes import (_jinja2_filter_rfc822, _jinja2_filter_rfc822_es, get_utc_timestamp,
+                         get_display_time, timesince, filter_truthy, category_name)
+
+
+class TestFilters(NyaaTestCase):
+
+    # def setUp(self):
+    #     self.db, nyaa.app.config['DATABASE'] = tempfile.mkstemp()
+    #     nyaa.app.config['TESTING'] = True
+    #     self.app = nyaa.app.test_client()
+    #     with nyaa.app.app_context():
+    #         nyaa.db.create_all()
+    #
+    # def tearDown(self):
+    #     os.close(self.db)
+    #     os.unlink(nyaa.app.config['DATABASE'])
+
+    def test_filter_rfc822(self):
+        # test with timezone UTC
+        test_date = datetime.datetime(2017, 2, 15, 11, 15, 34, 100, datetime.timezone.utc)
+        self.assertEqual(_jinja2_filter_rfc822(test_date), 'Wed, 15 Feb 2017 11:15:34 -0000')
+
+    def test_filter_rfc822_es(self):
+        # test with local timezone
+        test_date_str = '2017-02-15T11:15:34'
+        # this is in order to get around local time zone issues
+        expected = formatdate(float(datetime.datetime(2017, 2, 15, 11, 15, 34, 100).timestamp()))
+        self.assertEqual(_jinja2_filter_rfc822_es(test_date_str), expected)
+
+    def test_get_utc_timestamp(self):
+        # test with local timezone
+        test_date_str = '2017-02-15T11:15:34'
+        self.assertEqual(get_utc_timestamp(test_date_str), 1487157334)
+
+    def test_get_display_time(self):
+        # test with local timezone
+        test_date_str = '2017-02-15T11:15:34'
+        self.assertEqual(get_display_time(test_date_str), '2017-02-15 11:15')
+
+    def test_timesince(self):
+        now = datetime.datetime.utcnow()
+        self.assertEqual(timesince(now), 'just now')
+        self.assertEqual(timesince(now - datetime.timedelta(seconds=5)), '5 seconds ago')
+        self.assertEqual(timesince(now - datetime.timedelta(minutes=1)), '1 minute ago')
+        self.assertEqual(
+            timesince(now - datetime.timedelta(minutes=38, seconds=43)), '38 minutes ago')
+        self.assertEqual(
+            timesince(now - datetime.timedelta(hours=2, minutes=38, seconds=51)), '2 hours ago')
+        bigger = now - datetime.timedelta(days=3)
+        self.assertEqual(timesince(bigger), bigger.strftime('%Y-%m-%d %H:%M UTC'))
+
+    @unittest.skip('Not yet implemented')
+    def test_static_cachebuster(self):
+        pass
+
+    @unittest.skip('Not yet implemented')
+    def test_modify_query(self):
+        pass
+
+    def test_filter_truthy(self):
+        my_list = [
+            True, False,  # booleans
+            'hello!', '',  # strings
+            1, 0, -1,  # integers
+            1.0, 0.0, -1.0,  # floats
+            ['test'], [],  # lists
+            {'marco': 'polo'}, {},  # dictionaries
+            None
+        ]
+        expected_result = [
+            True,
+            'hello!',
+            1, -1,
+            1.0, -1.0,
+            ['test'],
+            {'marco': 'polo'}
+        ]
+        self.assertListEqual(filter_truthy(my_list), expected_result)
+
+    def test_category_name(self):
+        with self.app_context:
+            # Nyaa categories only
+            self.assertEqual(category_name('1_0'), 'Anime')
+            self.assertEqual(category_name('1_2'), 'Anime - English-translated')
+            # Unknown category ids
+            self.assertEqual(category_name('100_0'), '???')
+            self.assertEqual(category_name('1_100'), '???')
+            self.assertEqual(category_name('0_0'), '???')
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_utils.py

@@ -0,0 +1,115 @@
+import unittest
+from collections import OrderedDict
+
+from hashlib import sha1
+from nyaa import utils
+
+
+class TestUtils(unittest.TestCase):
+
+    def test_sha1_hash(self):
+        bencoded_test_data = b'd5:hello5:world7:numbersli1ei2eee'
+        self.assertEqual(
+            utils.sha1_hash(bencoded_test_data),
+            sha1(bencoded_test_data).digest())
+
+    def test_sorted_pathdict(self):
+        initial = {
+            'api_handler.py': 11805,
+            'routes.py': 34247,
+            '__init__.py': 6499,
+            'torrents.py': 11948,
+            'static': {
+                'img': {
+                    'nyaa.png': 1200,
+                    'sukebei.png': 1100,
+                },
+                'js': {
+                    'main.js': 3000,
+                },
+            },
+            'search.py': 5148,
+            'models.py': 24293,
+            'templates': {
+                'upload.html': 3000,
+                'home.html': 1200,
+                'layout.html': 23000,
+            },
+            'utils.py': 14700,
+        }
+        expected = OrderedDict({
+            'static': OrderedDict({
+                'img': OrderedDict({
+                    'nyaa.png': 1200,
+                    'sukebei.png': 1100,
+                }),
+                'js': OrderedDict({
+                    'main.js': 3000,
+                }),
+            }),
+            'templates': OrderedDict({
+                'home.html': 1200,
+                'layout.html': 23000,
+                'upload.html': 3000,
+            }),
+            '__init__.py': 6499,
+            'api_handler.py': 11805,
+            'models.py': 24293,
+            'routes.py': 34247,
+            'search.py': 5148,
+            'torrents.py': 11948,
+            'utils.py': 14700,
+        })
+        self.assertDictEqual(utils.sorted_pathdict(initial), expected)
+
+    @unittest.skip('Not yet implemented')
+    def test_cached_function(self):
+        # TODO: Test with a function that generates something random?
+        pass
+
+    def test_flatten_dict(self):
+        initial = OrderedDict({
+            'static': OrderedDict({
+                'img': OrderedDict({
+                    'nyaa.png': 1200,
+                    'sukebei.png': 1100,
+                }),
+                'js': OrderedDict({
+                    'main.js': 3000,
+                }),
+                'favicon.ico': 1000,
+            }),
+            'templates': [
+                {'home.html': 1200},
+                {'layout.html': 23000},
+                {'upload.html': 3000},
+            ],
+            '__init__.py': 6499,
+            'api_handler.py': 11805,
+            'models.py': 24293,
+            'routes.py': 34247,
+            'search.py': 5148,
+            'torrents.py': 11948,
+            'utils.py': 14700,
+        })
+        expected = {
+            'static/img/nyaa.png': 1200,
+            'static/img/sukebei.png': 1100,
+            'static/js/main.js': 3000,
+            'static/favicon.ico': 1000,
+            'templates/home.html': 1200,
+            'templates/layout.html': 23000,
+            'templates/upload.html': 3000,
+            '__init__.py': 6499,
+            'api_handler.py': 11805,
+            'models.py': 24293,
+            'routes.py': 34247,
+            'search.py': 5148,
+            'utils.py': 14700,
+            'torrents.py': 11948,
+        }
+        self.assertDictEqual(utils.flatten_dict(initial), expected)
+
+
+if __name__ == '__main__':
+    unittest.main()