a3x/nyaa
1
0
Fork 0
mirror of https://gitlab.com/SIGBUS/nyaa.git synced 2024-12-22 19:19:59 +00:00
Code Issues Projects Releases Wiki Activity

elasticsearch integration (untested)

Browse source
This commit is contained in:
nyaadev 2018-04-08 04:57:01 +02:00
parent 268ca12321
commit 847800c23e
1 changed files with 6 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
sync_es.py
View file

@ -34,7 +34,6 @@ from elasticsearch.helpers import bulk, BulkIndexError
from pymysqlreplication import BinLogStreamReader from pymysqlreplication import BinLogStreamReader
from pymysqlreplication.row_event import UpdateRowsEvent, DeleteRowsEvent, WriteRowsEvent from pymysqlreplication.row_event import UpdateRowsEvent, DeleteRowsEvent, WriteRowsEvent
from datetime import datetime from datetime import datetime
from nyaa.models import TorrentFlags
import sys import sys
import json import json
import time import time
@ -79,7 +78,6 @@ def pad_bytes(in_bytes, size):
def reindex_torrent(t, index_name): def reindex_torrent(t, index_name):
# XXX annoyingly different from import_to_es, and # XXX annoyingly different from import_to_es, and
# you need to keep them in sync manually. # you need to keep them in sync manually.
f = t['flags']
doc = { doc = {
"id": t['id'], "id": t['id'],
"display_name": t['display_name'], "display_name": t['display_name'],
@ -94,16 +92,15 @@ def reindex_torrent(t, index_name):
"main_category_id": t['main_category_id'], "main_category_id": t['main_category_id'],
"sub_category_id": t['sub_category_id'], "sub_category_id": t['sub_category_id'],
"comment_count": t['comment_count'], "comment_count": t['comment_count'],
# XXX all the bitflags are numbers "anonymous": bool(t['anonymous']),
"anonymous": bool(f & TorrentFlags.ANONYMOUS), "trusted": bool(t['trusted']),
"trusted": bool(f & TorrentFlags.TRUSTED), "remake": bool(t['remake']),
"remake": bool(f & TorrentFlags.REMAKE), "complete": bool(t['complete']),
"complete": bool(f & TorrentFlags.COMPLETE),
# TODO instead of indexing and filtering later # TODO instead of indexing and filtering later
# could delete from es entirely. Probably won't matter # could delete from es entirely. Probably won't matter
# for at least a few months. # for at least a few months.
"hidden": bool(f & TorrentFlags.HIDDEN), "hidden": bool(t['hidden']),
"deleted": bool(f & TorrentFlags.DELETED), "deleted": bool(t['deleted']),
"has_torrent": bool(t['has_torrent']), "has_torrent": bool(t['has_torrent']),
} }
# update, so we don't delete the stats if present # update, so we don't delete the stats if present