diff --git a/.docker/Dockerfile b/.docker/Dockerfile new file mode 100644 index 0000000..1f503fa --- /dev/null +++ b/.docker/Dockerfile @@ -0,0 +1,17 @@ +FROM ubuntu:18.04 + +ENV LANG=en_US.utf-8 LC_ALL=en_US.utf-8 DEBIAN_FRONTEND=noninteractive +RUN apt-get -y update + +COPY ./ /nyaa/ +RUN cat /nyaa/config.example.py /nyaa/.docker/nyaa-config-partial.py > /nyaa/config.py + +# Requirements for running the Flask app +RUN apt-get -y install build-essential git python3 python3-pip libmysqlclient-dev curl +# Helpful stuff for the docker entrypoint.sh script +RUN apt-get -y install mariadb-client netcat + +WORKDIR /nyaa +RUN pip3 install -r requirements.txt + +CMD ["/nyaa/.docker/entrypoint.sh"] diff --git a/.docker/README.md b/.docker/README.md new file mode 100644 index 0000000..fb3c9e3 --- /dev/null +++ b/.docker/README.md @@ -0,0 +1,46 @@ +# Nyaa on Docker + +Docker infrastructure is provided to ease setting up a dev environment + +## Quickstart + +Get started by running (from the root of the project): + + docker-compose -f .docker/full-stack.yml -p nyaa build nyaa-flask + docker-compose -f .docker/full-stack.yml -p nyaa up -d + +This builds the Flask app container, then starts up the project. You can then go +to [localhost:8080](http://localhost:8080/) (note that some of the +services are somewhat slow to start so it may not be available for 30s or so). + +You can shut it down with: + + docker-compose -f .docker/full-stack.yml -p nyaa down + +## Details + +The environment includes: + - [nginx frontend](http://localhost:8080/) (on port 8080) + - uwsgi running the flask app + - the ES<>MariaDB sync process + - MariaDB + - ElasticSearch + - [Kibana](http://localhost:8080/kibana/) (at /kibana/) + +MariaDB, ElasticSearch, the sync process, and uploaded torrents will +persistently store their data in volumes which makes future start ups faster. + +To make it more useful to develop with, you can copy `.docker/full-stack.yml` and +edit the copy and uncomment the `- "${NYAA_SRC_DIR}:/nyaa"` line, then +`export NYAA_SRC_DIR=$(pwd)` and start up the environment using the new compose +file: + + cp -a .docker/full-stack.yml .docker/local-dev.yml + cat config.example.py .docker/nyaa-config-partial.py > ./config.py + $EDITOR .docker/local-dev.yml + export NYAA_SRC_DIR=$(pwd) + docker-compose -f .docker/local-dev.yml -p nyaa up -d + +This will mount the local copy of the project files into the Flask container, +which combined with live-reloading in uWSGI should let you make changes and see +them take effect immediately (technically with a ~2 second delay). diff --git a/.docker/entrypoint-sync.sh b/.docker/entrypoint-sync.sh new file mode 100755 index 0000000..b58b646 --- /dev/null +++ b/.docker/entrypoint-sync.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# set +x + +pushd /nyaa + +echo 'Waiting for MySQL to start up' +while ! echo HELO | nc mariadb 3306 &>/dev/null; do + sleep 1 +done +echo 'DONE' + +echo 'Waiting for ES to start up' +while ! echo HELO | nc elasticsearch 9200 &>/dev/null; do + sleep 1 +done +echo 'DONE' + +echo 'Waiting for ES to be ready' +while ! curl -s -XGET 'elasticsearch:9200/_cluster/health?pretty=true&wait_for_status=green' &>/dev/null; do + sleep 1 +done +echo 'DONE' + +echo 'Waiting for sync data file to exist' +while ! [ -f /elasticsearch-sync/pos.json ]; do + sleep 1 +done +echo 'DONE' + +echo 'Starting the sync process' +/usr/bin/python3 /nyaa/sync_es.py /nyaa/.docker/es_sync_config.json diff --git a/.docker/entrypoint.sh b/.docker/entrypoint.sh new file mode 100755 index 0000000..7688739 --- /dev/null +++ b/.docker/entrypoint.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# set +x + +pushd /nyaa + +echo 'Waiting for MySQL to start up' +while ! echo HELO | nc mariadb 3306 &>/dev/null; do + sleep 1 +done +echo 'DONE' + +if ! [ -f /elasticsearch-sync/flag-db_create ]; then + python3 ./db_create.py + touch /elasticsearch-sync/flag-db_create +fi + +if ! [ -f /elasticsearch-sync/flag-db_migrate ]; then + python3 ./db_migrate.py stamp head + touch /elasticsearch-sync/flag-db_migrate +fi + +echo 'Waiting for ES to start up' +while ! echo HELO | nc elasticsearch 9200 &>/dev/null; do + sleep 1 +done +echo 'DONE' + +echo 'Waiting for ES to be ready' +while ! curl -s -XGET 'elasticsearch:9200/_cluster/health?pretty=true&wait_for_status=green' &>/dev/null; do + sleep 1 +done +echo 'DONE' + +if ! [ -f /elasticsearch-sync/flag-create_es ]; then + # @source create_es.sh + # create indices named "nyaa" and "sukebei", these are hardcoded + curl -v -XPUT 'elasticsearch:9200/nyaa?pretty' -H"Content-Type: application/yaml" --data-binary @es_mapping.yml + curl -v -XPUT 'elasticsearch:9200/sukebei?pretty' -H"Content-Type: application/yaml" --data-binary @es_mapping.yml + touch /elasticsearch-sync/flag-create_es +fi + +if ! [ -f /elasticsearch-sync/flag-import_to_es ]; then + python3 ./import_to_es.py | tee /elasticsearch-sync/import.out + grep -A1 'Save the following' /elasticsearch-sync/import.out | tail -1 > /elasticsearch-sync/pos.json + touch /elasticsearch-sync/flag-import_to_es +fi + +echo 'Starting the Flask app' +/usr/local/bin/uwsgi /nyaa/.docker/uwsgi.config.ini diff --git a/.docker/es_sync_config.json b/.docker/es_sync_config.json new file mode 100644 index 0000000..2c4ec0c --- /dev/null +++ b/.docker/es_sync_config.json @@ -0,0 +1,11 @@ +{ + "save_loc": "/elasticsearch-sync/pos.json", + "mysql_host": "mariadb", + "mysql_port": 3306, + "mysql_user": "nyaadev", + "mysql_password": "ZmtB2oihHFvc39JaEDoF", + "database": "nyaav2", + "internal_queue_depth": 10000, + "es_chunk_size": 10000, + "flush_interval": 5 +} diff --git a/.docker/full-stack.yml b/.docker/full-stack.yml new file mode 100644 index 0000000..4de5bb1 --- /dev/null +++ b/.docker/full-stack.yml @@ -0,0 +1,71 @@ +--- + +version: "3" +services: + nginx: + image: nginx:1.15-alpine + ports: + - '8080:80' + volumes: + - './nginx.conf:/etc/nginx/nginx.conf:ro' + - '../nyaa/static:/nyaa-static:ro' + depends_on: + - nyaa-flask + - kibana + + nyaa-flask: + image: local/nyaa:devel + volumes: + - 'nyaa-torrents:/nyaa-torrents' + - 'nyaa-sync-data:/elasticsearch-sync' + ## Uncomment this line to have to mount the local dir to the running + ## instance for live changes (after setting NYAA_SRC_DIR env var) + # - "${NYAA_SRC_DIR}:/nyaa" + depends_on: + - mariadb + - elasticsearch + build: + context: ../ + dockerfile: ./.docker/Dockerfile + + nyaa-sync: + image: local/nyaa:devel + volumes: + - 'nyaa-sync-data:/elasticsearch-sync' + command: /nyaa/.docker/entrypoint-sync.sh + depends_on: + - mariadb + - elasticsearch + restart: on-failure + + mariadb: + image: mariadb:10.0 + volumes: + - './mariadb-init-sql:/docker-entrypoint-initdb.d:ro' + - '../configs/my.cnf:/etc/mysql/conf.d/50-binlog.cnf:ro' + - 'mariadb-data:/var/lib/mysql' + environment: + - MYSQL_RANDOM_ROOT_PASSWORD=yes + - MYSQL_USER=nyaadev + - MYSQL_PASSWORD=ZmtB2oihHFvc39JaEDoF + - MYSQL_DATABASE=nyaav2 + + elasticsearch: + image: elasticsearch:6.5.4 + volumes: + - elasticsearch-data:/usr/share/elasticsearch/data + depends_on: + - mariadb + + kibana: + image: kibana:6.5.4 + volumes: + - './kibana.config.yml:/usr/share/kibana/config/kibana.yml:ro' + depends_on: + - elasticsearch + +volumes: + nyaa-torrents: + nyaa-sync-data: + mariadb-data: + elasticsearch-data: diff --git a/.docker/kibana.config.yml b/.docker/kibana.config.yml new file mode 100644 index 0000000..da9dc09 --- /dev/null +++ b/.docker/kibana.config.yml @@ -0,0 +1,9 @@ +--- + +server.name: kibana +server.host: 'kibana' +server.basePath: /kibana +# server.rewriteBasePath: true +# server.defaultRoute: /kibana/app/kibana +elasticsearch.url: http://elasticsearch:9200 +xpack.monitoring.ui.container.elasticsearch.enabled: true diff --git a/.docker/mariadb-init-sql/.gitignore b/.docker/mariadb-init-sql/.gitignore new file mode 100644 index 0000000..aecee65 --- /dev/null +++ b/.docker/mariadb-init-sql/.gitignore @@ -0,0 +1 @@ +!*.sql diff --git a/.docker/mariadb-init-sql/50-grant-binlog-access.sql b/.docker/mariadb-init-sql/50-grant-binlog-access.sql new file mode 100644 index 0000000..81fac03 --- /dev/null +++ b/.docker/mariadb-init-sql/50-grant-binlog-access.sql @@ -0,0 +1,3 @@ +GRANT REPLICATION SLAVE ON *.* TO 'nyaadev'@'%'; +GRANT REPLICATION CLIENT ON *.* TO 'nyaadev'@'%'; +FLUSH PRIVILEGES; diff --git a/.docker/nginx.conf b/.docker/nginx.conf new file mode 100644 index 0000000..27f3757 --- /dev/null +++ b/.docker/nginx.conf @@ -0,0 +1,59 @@ + +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + charset utf-8; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + gzip on; + + server { + listen 80; + server_name localhost default; + + location /static { + alias /nyaa-static; + } + + # fix kibana redirecting to localhost/kibana (without the port) + rewrite ^/kibana$ http://$http_host/kibana/ permanent; + location /kibana/ { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_cache_bypass $http_upgrade; + + proxy_set_header Host 'kibana'; + proxy_set_header X-Real-IP $remote_addr; + + proxy_pass http://kibana:5601/; + } + + location / { + include /etc/nginx/uwsgi_params; + uwsgi_pass nyaa-flask:5000; + } + } +} diff --git a/.docker/nyaa-config-partial.py b/.docker/nyaa-config-partial.py new file mode 100644 index 0000000..f805c7c --- /dev/null +++ b/.docker/nyaa-config-partial.py @@ -0,0 +1,10 @@ +# This is only a partial config file that will be appended to the end of +# config.example.py to build the full config for the docker environment + +SITE_NAME = 'Nyaa [DEVEL]' +GLOBAL_SITE_NAME = 'nyaa.devel' +SQLALCHEMY_DATABASE_URI = ('mysql://nyaadev:ZmtB2oihHFvc39JaEDoF@mariadb/nyaav2?charset=utf8mb4') +# MAIN_ANNOUNCE_URL = 'http://chihaya:6881/announce' +# TRACKER_API_URL = 'http://chihaya:6881/api' +BACKUP_TORRENT_FOLDER = '/nyaa-torrents' +ES_HOSTS = ['elasticsearch:9200'] diff --git a/.docker/uwsgi.config.ini b/.docker/uwsgi.config.ini new file mode 100644 index 0000000..7e8d620 --- /dev/null +++ b/.docker/uwsgi.config.ini @@ -0,0 +1,34 @@ +[uwsgi] +# socket = [addr:port] +socket = 0.0.0.0:5000 +#chmod-socket = 664 + +die-on-term = true + +# logging +#disable-logging = True +#logger = file:uwsgi.log + +# Base application directory +chdir = /nyaa + +# WSGI module and callable +# module = [wsgi_module_name]:[application_callable_name] +module = WSGI:app + +# master = [master process (true of false)] +master = true + +# debugging +catch-exceptions = true + +# performance +processes = 4 +buffer-size = 8192 + +loop = gevent +socket-timeout = 10 +gevent = 1000 +gevent-monkey-patch = true + +py-autoreload = 2