feature: docker development environment (#544)

Add docker support

It's self-contained so should not interfere with anything else and can easily be thrown out if nobody wants it anymore.

.docker/Dockerfile

@@ -0,0 +1,17 @@
+FROM ubuntu:18.04
+
+ENV LANG=en_US.utf-8 LC_ALL=en_US.utf-8 DEBIAN_FRONTEND=noninteractive
+RUN apt-get -y update
+
+COPY ./ /nyaa/
+RUN cat /nyaa/config.example.py /nyaa/.docker/nyaa-config-partial.py > /nyaa/config.py
+
+# Requirements for running the Flask app
+RUN apt-get -y install build-essential git python3 python3-pip libmysqlclient-dev curl
+# Helpful stuff for the docker entrypoint.sh script
+RUN apt-get -y install mariadb-client netcat
+
+WORKDIR /nyaa
+RUN pip3 install -r requirements.txt
+
+CMD ["/nyaa/.docker/entrypoint.sh"]

.docker/README.md

@@ -0,0 +1,46 @@
+# Nyaa on Docker
+
+Docker infrastructure is provided to ease setting up a dev environment
+
+## Quickstart
+
+Get started by running (from the root of the project):
+
+	docker-compose -f .docker/full-stack.yml -p nyaa build nyaa-flask
+	docker-compose -f .docker/full-stack.yml -p nyaa up -d
+
+This builds the Flask app container, then starts up the project. You can then go
+to [localhost:8080](http://localhost:8080/) (note that some of the
+services are somewhat slow to start so it may not be available for 30s or so).
+
+You can shut it down with:
+
+	docker-compose -f .docker/full-stack.yml -p nyaa down
+
+## Details
+
+The environment includes:
+  - [nginx frontend](http://localhost:8080/) (on port 8080)
+  - uwsgi running the flask app
+  - the ES<>MariaDB sync process
+  - MariaDB
+  - ElasticSearch
+  - [Kibana](http://localhost:8080/kibana/) (at /kibana/)
+
+MariaDB, ElasticSearch, the sync process, and uploaded torrents will
+persistently store their data in volumes which makes future start ups faster.
+
+To make it more useful to develop with, you can copy `.docker/full-stack.yml` and
+edit the copy and uncomment the `- "${NYAA_SRC_DIR}:/nyaa"` line, then
+`export NYAA_SRC_DIR=$(pwd)` and start up the environment using the new compose
+file:
+
+	cp -a .docker/full-stack.yml .docker/local-dev.yml
+	cat config.example.py .docker/nyaa-config-partial.py > ./config.py
+	$EDITOR .docker/local-dev.yml
+	export NYAA_SRC_DIR=$(pwd)
+	docker-compose -f .docker/local-dev.yml -p nyaa up -d
+
+This will mount the local copy of the project files into the Flask container,
+which combined with live-reloading in uWSGI should let you make changes and see
+them take effect immediately (technically with a ~2 second delay).

.docker/entrypoint-sync.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# set +x
+
+pushd /nyaa
+
+echo 'Waiting for MySQL to start up'
+while ! echo HELO | nc mariadb 3306 &>/dev/null; do
+	sleep 1
+done
+echo 'DONE'
+
+echo 'Waiting for ES to start up'
+while ! echo HELO | nc elasticsearch 9200 &>/dev/null; do
+	sleep 1
+done
+echo 'DONE'
+
+echo 'Waiting for ES to be ready'
+while ! curl -s -XGET 'elasticsearch:9200/_cluster/health?pretty=true&wait_for_status=green' &>/dev/null; do
+	sleep 1
+done
+echo 'DONE'
+
+echo 'Waiting for sync data file to exist'
+while ! [ -f /elasticsearch-sync/pos.json ]; do
+	sleep 1
+done
+echo 'DONE'
+
+echo 'Starting the sync process'
+/usr/bin/python3 /nyaa/sync_es.py /nyaa/.docker/es_sync_config.json

.docker/entrypoint.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# set +x
+
+pushd /nyaa
+
+echo 'Waiting for MySQL to start up'
+while ! echo HELO | nc mariadb 3306 &>/dev/null; do
+	sleep 1
+done
+echo 'DONE'
+
+if ! [ -f /elasticsearch-sync/flag-db_create ]; then
+	python3 ./db_create.py
+	touch /elasticsearch-sync/flag-db_create
+fi
+
+if ! [ -f /elasticsearch-sync/flag-db_migrate ]; then
+	python3 ./db_migrate.py stamp head
+	touch /elasticsearch-sync/flag-db_migrate
+fi
+
+echo 'Waiting for ES to start up'
+while ! echo HELO | nc elasticsearch 9200 &>/dev/null; do
+	sleep 1
+done
+echo 'DONE'
+
+echo 'Waiting for ES to be ready'
+while ! curl -s -XGET 'elasticsearch:9200/_cluster/health?pretty=true&wait_for_status=green' &>/dev/null; do
+	sleep 1
+done
+echo 'DONE'
+
+if ! [ -f /elasticsearch-sync/flag-create_es ]; then
+	# @source create_es.sh
+	# create indices named "nyaa" and "sukebei", these are hardcoded
+	curl -v -XPUT 'elasticsearch:9200/nyaa?pretty' -H"Content-Type: application/yaml" --data-binary @es_mapping.yml
+	curl -v -XPUT 'elasticsearch:9200/sukebei?pretty' -H"Content-Type: application/yaml" --data-binary @es_mapping.yml
+	touch /elasticsearch-sync/flag-create_es
+fi
+
+if ! [ -f /elasticsearch-sync/flag-import_to_es ]; then
+	python3 ./import_to_es.py | tee /elasticsearch-sync/import.out
+	grep -A1 'Save the following' /elasticsearch-sync/import.out | tail -1 > /elasticsearch-sync/pos.json
+	touch /elasticsearch-sync/flag-import_to_es
+fi
+
+echo 'Starting the Flask app'
+/usr/local/bin/uwsgi /nyaa/.docker/uwsgi.config.ini

.docker/es_sync_config.json

@@ -0,0 +1,11 @@
+{
+    "save_loc": "/elasticsearch-sync/pos.json",
+    "mysql_host": "mariadb",
+    "mysql_port": 3306,
+    "mysql_user": "nyaadev",
+    "mysql_password": "ZmtB2oihHFvc39JaEDoF",
+    "database": "nyaav2",
+    "internal_queue_depth": 10000,
+    "es_chunk_size": 10000,
+    "flush_interval": 5
+}

.docker/full-stack.yml

@@ -0,0 +1,71 @@
+---
+
+version: "3"
+services:
+  nginx:
+    image: nginx:1.15-alpine
+    ports:
+      - '8080:80'
+    volumes:
+      - './nginx.conf:/etc/nginx/nginx.conf:ro'
+      - '../nyaa/static:/nyaa-static:ro'
+    depends_on:
+      - nyaa-flask
+      - kibana
+
+  nyaa-flask:
+    image: local/nyaa:devel
+    volumes:
+      - 'nyaa-torrents:/nyaa-torrents'
+      - 'nyaa-sync-data:/elasticsearch-sync'
+      ## Uncomment this line to have to mount the local dir to the running
+      ## instance for live changes (after setting NYAA_SRC_DIR env var)
+      # - "${NYAA_SRC_DIR}:/nyaa"
+    depends_on:
+      - mariadb
+      - elasticsearch
+    build:
+      context: ../
+      dockerfile: ./.docker/Dockerfile
+
+  nyaa-sync:
+    image: local/nyaa:devel
+    volumes:
+      - 'nyaa-sync-data:/elasticsearch-sync'
+    command: /nyaa/.docker/entrypoint-sync.sh
+    depends_on:
+      - mariadb
+      - elasticsearch
+    restart: on-failure
+
+  mariadb:
+    image: mariadb:10.0
+    volumes:
+      - './mariadb-init-sql:/docker-entrypoint-initdb.d:ro'
+      - '../configs/my.cnf:/etc/mysql/conf.d/50-binlog.cnf:ro'
+      - 'mariadb-data:/var/lib/mysql'
+    environment:
+      - MYSQL_RANDOM_ROOT_PASSWORD=yes
+      - MYSQL_USER=nyaadev
+      - MYSQL_PASSWORD=ZmtB2oihHFvc39JaEDoF
+      - MYSQL_DATABASE=nyaav2
+
+  elasticsearch:
+    image: elasticsearch:6.5.4
+    volumes:
+      - elasticsearch-data:/usr/share/elasticsearch/data
+    depends_on:
+      - mariadb
+
+  kibana:
+    image: kibana:6.5.4
+    volumes:
+      - './kibana.config.yml:/usr/share/kibana/config/kibana.yml:ro'
+    depends_on:
+      - elasticsearch
+
+volumes:
+  nyaa-torrents:
+  nyaa-sync-data:
+  mariadb-data:
+  elasticsearch-data:

.docker/kibana.config.yml

@@ -0,0 +1,9 @@
+---
+
+server.name: kibana
+server.host: 'kibana'
+server.basePath: /kibana
+# server.rewriteBasePath: true
+# server.defaultRoute: /kibana/app/kibana
+elasticsearch.url: http://elasticsearch:9200
+xpack.monitoring.ui.container.elasticsearch.enabled: true

.docker/mariadb-init-sql/.gitignore

@@ -0,0 +1 @@
+!*.sql

.docker/mariadb-init-sql/50-grant-binlog-access.sql

@@ -0,0 +1,3 @@
+GRANT REPLICATION SLAVE ON *.* TO 'nyaadev'@'%';
+GRANT REPLICATION CLIENT ON *.* TO 'nyaadev'@'%';
+FLUSH PRIVILEGES;

.docker/nginx.conf

@@ -0,0 +1,59 @@
+
+user  nginx;
+worker_processes  1;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+    charset       utf-8;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+
+    gzip  on;
+
+    server {
+        listen      80;
+        server_name localhost default;
+
+        location /static {
+            alias /nyaa-static;
+        }
+
+        # fix kibana redirecting to localhost/kibana (without the port)
+        rewrite ^/kibana$ http://$http_host/kibana/ permanent;
+        location /kibana/ {
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection 'upgrade';
+            proxy_cache_bypass $http_upgrade;            
+
+            proxy_set_header Host 'kibana';
+            proxy_set_header X-Real-IP $remote_addr;
+
+            proxy_pass http://kibana:5601/;
+        }
+
+        location / {
+            include    /etc/nginx/uwsgi_params;
+            uwsgi_pass nyaa-flask:5000;
+        }
+    }
+}

.docker/nyaa-config-partial.py

@@ -0,0 +1,10 @@
+# This is only a partial config file that will be appended to the end of
+# config.example.py to build the full config for the docker environment
+
+SITE_NAME = 'Nyaa [DEVEL]'
+GLOBAL_SITE_NAME = 'nyaa.devel'
+SQLALCHEMY_DATABASE_URI = ('mysql://nyaadev:ZmtB2oihHFvc39JaEDoF@mariadb/nyaav2?charset=utf8mb4')
+# MAIN_ANNOUNCE_URL = 'http://chihaya:6881/announce'
+# TRACKER_API_URL = 'http://chihaya:6881/api'
+BACKUP_TORRENT_FOLDER = '/nyaa-torrents'
+ES_HOSTS = ['elasticsearch:9200']

.docker/uwsgi.config.ini

@@ -0,0 +1,34 @@
+[uwsgi]
+# socket = [addr:port]
+socket = 0.0.0.0:5000
+#chmod-socket = 664
+
+die-on-term = true
+
+# logging
+#disable-logging = True
+#logger = file:uwsgi.log
+
+# Base application directory
+chdir = /nyaa
+
+# WSGI module and callable
+# module = [wsgi_module_name]:[application_callable_name]
+module = WSGI:app
+
+# master = [master process (true of false)]
+master = true
+
+# debugging
+catch-exceptions = true
+
+# performance
+processes = 4
+buffer-size = 8192
+
+loop = gevent
+socket-timeout = 10
+gevent = 1000
+gevent-monkey-patch = true
+
+py-autoreload = 2