From 152e547ac5b7aa027e6dd634c695a631943a7020 Mon Sep 17 00:00:00 2001 From: nyaadev Date: Sun, 21 May 2017 17:47:16 +0200 Subject: [PATCH] Add flask-Migrate + alembic for automated database migrations. Update some dependencies to their latest version. Make executable scripts executable (chmod +x). --- README.md | 6 +++ WSGI.py | 0 db_create.py | 6 --- db_migrate.py | 13 ++++++ import_to_es.py | 0 migrations/README | 1 + migrations/alembic.ini | 45 ++++++++++++++++++++ migrations/env.py | 87 +++++++++++++++++++++++++++++++++++++++ migrations/script.py.mako | 24 +++++++++++ requirements.txt | 28 ++++++++----- run.py | 0 sync_es.py | 0 12 files changed, 194 insertions(+), 16 deletions(-) mode change 100644 => 100755 WSGI.py mode change 100644 => 100755 db_create.py create mode 100755 db_migrate.py mode change 100644 => 100755 import_to_es.py create mode 100755 migrations/README create mode 100644 migrations/alembic.ini create mode 100755 migrations/env.py create mode 100755 migrations/script.py.mako mode change 100644 => 100755 run.py mode change 100644 => 100755 sync_es.py diff --git a/README.md b/README.md index 843d0ac..c2471a8 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,12 @@ - Set up `sync_es.py` as a service and run it, preferably as the system/root - Make sure `sync_es.py` runs within venv with the right dependencies +## Database migrations +- Uses [flask-Migrate](https://flask-migrate.readthedocs.io/) +- Run `./db_migrate.py db migrate` to generate the migration script after database model changes. +- Take a look at the result in `migrations/versions/...` to make sure nothing went wrong. +- Run `./db_migrate.py db upgrade` to upgrade your database. + ## Good to go! - After that, enable the `USE_ELASTIC_SEARCH` flag and restart the webapp and you're good to go diff --git a/WSGI.py b/WSGI.py old mode 100644 new mode 100755 diff --git a/db_create.py b/db_create.py old mode 100644 new mode 100755 index 99c4a85..d881377 --- a/db_create.py +++ b/db_create.py @@ -33,9 +33,3 @@ if not existing_cats: db.session.add(main_cat) db.session.commit() - -# Create fulltext index - -if app.config['USE_MYSQL']: - db.engine.execute('ALTER TABLE ' + app.config['TABLE_PREFIX'] + 'torrents ADD FULLTEXT KEY (display_name)') - diff --git a/db_migrate.py b/db_migrate.py new file mode 100755 index 0000000..8d4f8f0 --- /dev/null +++ b/db_migrate.py @@ -0,0 +1,13 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +from nyaa import app, db +from flask_script import Manager +from flask_migrate import Migrate, MigrateCommand + +migrate = Migrate(app, db) + +manager = Manager(app) +manager.add_command("db", MigrateCommand) + +if __name__ == "__main__": + manager.run() diff --git a/import_to_es.py b/import_to_es.py old mode 100644 new mode 100755 diff --git a/migrations/README b/migrations/README new file mode 100755 index 0000000..98e4f9c --- /dev/null +++ b/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/migrations/alembic.ini b/migrations/alembic.ini new file mode 100644 index 0000000..f8ed480 --- /dev/null +++ b/migrations/alembic.ini @@ -0,0 +1,45 @@ +# A generic, single database configuration. + +[alembic] +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/migrations/env.py b/migrations/env.py new file mode 100755 index 0000000..4593816 --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,87 @@ +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig +import logging + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) +logger = logging.getLogger('alembic.env') + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +from flask import current_app +config.set_main_option('sqlalchemy.url', + current_app.config.get('SQLALCHEMY_DATABASE_URI')) +target_metadata = current_app.extensions['migrate'].db.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure(url=url) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + # this callback is used to prevent an auto-migration from being generated + # when there are no changes to the schema + # reference: http://alembic.readthedocs.org/en/latest/cookbook.html + def process_revision_directives(context, revision, directives): + if getattr(config.cmd_opts, 'autogenerate', False): + script = directives[0] + if script.upgrade_ops.is_empty(): + directives[:] = [] + logger.info('No changes in schema detected.') + + engine = engine_from_config(config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + connection = engine.connect() + context.configure(connection=connection, + target_metadata=target_metadata, + process_revision_directives=process_revision_directives, + **current_app.extensions['migrate'].configure_args) + + try: + with context.begin_transaction(): + context.run_migrations() + finally: + connection.close() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100755 index 0000000..2c01563 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/requirements.txt b/requirements.txt index 6e23eca..2a184ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +alembic==0.9.2 appdirs==1.4.3 argon2-cffi==16.3.0 autopep8==1.3.1 @@ -5,9 +6,14 @@ blinker==1.4 cffi==1.10.0 click==6.7 dominate==2.3.1 -Flask==0.12.1 +elasticsearch==5.3.0 +elasticsearch-dsl==5.2.0 +Flask==0.12.2 Flask-Assets==0.12 Flask-DebugToolbar==0.10.1 +Flask-Migrate==2.0.3 +flask-paginate==0.4.5 +Flask-Script==2.0.5 Flask-SQLAlchemy==2.2 Flask-WTF==0.14.2 gevent==1.2.1 @@ -15,27 +21,29 @@ greenlet==0.4.12 itsdangerous==0.24 Jinja2==2.9.6 libsass==0.12.3 +Mako==1.0.6 MarkupSafe==1.0 +mysql-replication==0.13 mysqlclient==1.3.10 orderedset==2.0 packaging==16.8 passlib==1.7.1 +progressbar2==3.20.0 pycodestyle==2.3.1 pycparser==2.17 +PyMySQL==0.7.11 pyparsing==2.2.0 +python-dateutil==2.6.0 +python-editor==1.0.3 +python-utils==2.1.0 six==1.10.0 -SQLAlchemy==1.1.9 +SQLAlchemy==1.1.10 SQLAlchemy-FullText-Search==0.2.3 SQLAlchemy-Utils==0.32.14 +statsd==3.2.1 +urllib3==1.21.1 uWSGI==2.0.15 visitor==0.1.3 webassets==0.12.1 -Werkzeug==0.12.1 +Werkzeug==0.12.2 WTForms==2.1 -## elasticsearch dependencies -elasticsearch==5.3.0 -elasticsearch-dsl==5.2.0 -progressbar2==3.20.0 -mysql-replication==0.13 -flask-paginate==0.4.5 -statsd==3.2.1 diff --git a/run.py b/run.py old mode 100644 new mode 100755 diff --git a/sync_es.py b/sync_es.py old mode 100644 new mode 100755