Merge pull request #22 from moyamo/yowsup-2

Remove MySQL server in favour of using spectrum
This commit is contained in:
Steffen Vogel 2015-12-03 14:00:52 +02:00
commit da6ae5fd65
11 changed files with 210 additions and 262 deletions

View File

@ -78,23 +78,15 @@ Checkout the latest version of transWhat from GitHub:
Install required dependencies:
$ pip install --pre e4u protobuf mysql python-dateutil
$ pip install --pre e4u protobuf python-dateutil
- **e4u**: is a simple emoji4unicode python bindings
- [**yowsup**](https://github.com/tgalal/yowsup): is a python library that enables you build application which use WhatsApp service.
- **mysqldb**: MySQL client python bindings
##### Configuration
First create a new mySQL database named `transwhat` and fill it with the [schema](https://raw.githubusercontent.com/stv0g/transwhat/yowsup-2/conf/schema.sql) provided in the repo.
Then create a new file called `constants.py` in the newly checked out transWhat Git repository:
DB_HOST = "localhost"
DB_USER = "**fillin**"
DB_PASS = "**fillin**"
DB_TABLE = "transwhat"
BASE_PATH = "/location/to/transwhat"
TOKEN_FILE = BASE_PATH + "/logs/tokens"

View File

@ -19,6 +19,7 @@ class SpectrumBackend:
@param host: Host where Spectrum2 NetworkPluginServer runs.
@param port: Port.
"""
def __init__(self):
self.m_pingReceived = False
self.m_data = ""
@ -344,6 +345,14 @@ class SpectrumBackend:
groups = [g for g in payload.group]
self.handleBuddyRemovedRequest(payload.userName, payload.buddyName, groups);
def handleBuddiesPayload(self, data):
payload = protocol_pb2.Buddies()
if (payload.ParseFromString(data) == False):
#TODO: ERROR
return
self.handleBuddies(payload);
def handleChatStatePayload(self, data, msgType):
payload = protocol_pb2.Buddy()
if (payload.ParseFromString(data) == False):
@ -428,6 +437,8 @@ class SpectrumBackend:
self.handleConvMessageAckPayload(wrapper.payload)
elif wrapper.type == protocol_pb2.WrapperMessage.TYPE_RAW_XML:
self.handleRawXmlRequest(wrapper.payload)
elif wrapper.type == protocol_pb2.WrapperMessage.TYPE_BUDDIES:
self.handleBuddiesPayload(wrapper.payload)
def send(self, data):
header = struct.pack('!I',len(data))
@ -488,6 +499,9 @@ class SpectrumBackend:
raise NotImplementedError, "Implement me"
def handleBuddies(self, buddies):
pass
def handleLogoutRequest(self, user, legacyName):
"""
Called when XMPP user wants to disconnect legacy network.
@ -505,7 +519,7 @@ class SpectrumBackend:
@param legacyName: Legacy network name of buddy or room.
@param message: Plain text message.
@param xhtml: XHTML message.
@param ID: message ID
@param ID: message ID
"""
raise NotImplementedError, "Implement me"

View File

@ -63,6 +63,10 @@ message Buddy {
optional bool blocked = 8;
}
message Buddies {
repeated Buddy buddy = 1;
}
message ConversationMessage {
required string userName = 1;
required string buddyName = 2;
@ -182,6 +186,7 @@ message WrapperMessage {
TYPE_ROOM_LIST = 32;
TYPE_CONV_MESSAGE_ACK = 33;
TYPE_RAW_XML = 34;
TYPE_BUDDIES = 35;
}
required Type type = 1;
optional bytes payload = 2;

File diff suppressed because one or more lines are too long

199
buddy.py
View File

@ -25,34 +25,10 @@ __status__ = "Prototype"
from Spectrum2 import protocol_pb2
import logging
import threading
class Number():
def __init__(self, number, state, db):
self.number = number
self.db = db
self.state = state
cur = self.db.cursor()
cur.execute("SELECT id FROM numbers WHERE number = %s AND state = %s", (self.number, self.state))
if (cur.rowcount):
self.id = cur.fetchone()[0]
else:
cur.execute("REPLACE numbers (number, state) VALUES (%s, %s)", (self.number, self.state))
self.db.commit()
self.id = cur.lastrowid
def __str__(self):
return "%s (id=%s)" % (self.number, self.id)
class Buddy():
def __init__(self, owner, number, nick, groups, image_hash, id, db):
self.id = id
self.db = db
def __init__(self, owner, number, nick, statusMsg, groups, image_hash):
self.nick = nick
self.owner = owner
self.number = number
@ -69,121 +45,100 @@ class Buddy():
if image_hash is not None:
self.image_hash = image_hash
groups = u",".join(groups).encode("latin-1")
cur = self.db.cursor()
cur.execute("UPDATE buddies SET nick = %s, groups = %s, image_hash = %s WHERE owner_id = %s AND buddy_id = %s", (self.nick, groups, self.image_hash, self.owner.id, self.number.id))
self.db.commit()
def delete(self):
cur = self.db.cursor()
cur.execute("DELETE FROM buddies WHERE owner_id = %s AND buddy_id = %s", (self.owner.id, self.number.id))
self.db.commit()
self.id = None
@staticmethod
def create(owner, number, nick, groups, image_hash, db):
groups = u",".join(groups).encode("latin-1")
cur = db.cursor()
cur.execute("REPLACE buddies (owner_id, buddy_id, nick, groups, image_hash) VALUES (%s, %s, %s, %s, %s)", (owner.id, number.id, nick, groups, image_hash))
db.commit()
return Buddy(owner, number, nick, groups, image_hash, cur.lastrowid, db)
def __str__(self):
return "%s (nick=%s, id=%s)" % (self.number, self.nick, self.id)
return "%s (nick=%s)" % (self.number, self.nick)
class BuddyList(dict):
def __init__(self, owner, db):
self.db = db
self.owner = Number(owner, 1, db)
self.lock = threading.Lock()
def __init__(self, owner, backend, user, session):
self.owner = owner
self.backend = backend
self.session = session
self.user = user
self.logger = logging.getLogger(self.__class__.__name__)
self.synced = False
def _load(self, buddies):
for buddy in buddies:
number = buddy.buddyName
nick = buddy.alias
statusMsg = buddy.statusMessage
groups = [g for g in buddy.group]
image_hash = buddy.iconHash
self[number] = Buddy(self.owner, number, nick, statusMsg,
groups, image_hash)
self.logger.debug("Update roster")
# old = self.buddies.keys()
# self.buddies.load()
# new = self.buddies.keys()
# contacts = new
contacts = self.keys()
if self.synced == False:
self.session.sendSync(contacts, delta = False, interactive = True)
self.synced = True
# add = set(new) - set(old)
# remove = set(old) - set(new)
# self.logger.debug("Roster remove: %s", str(list(remove)))
self.logger.debug("Roster add: %s", str(list(contacts)))
# for number in remove:
# self.backend.handleBuddyChanged(self.user, number, "", [],
# protocol_pb2.STATUS_NONE)
# self.backend.handleBuddyRemoved(self.user, number)
# self.unsubscribePresence(number)
#
for number in contacts:
buddy = self[number]
if number != 'bot':
self.backend.handleBuddyChanged(self.user, number, buddy.nick,
buddy.groups, protocol_pb2.STATUS_NONE,
iconHash = buddy.image_hash if buddy.image_hash is not None else "")
self.session.subscribePresence(number)
def load(self):
self.clear()
self.lock.acquire()
cur = self.db.cursor()
cur.execute("""SELECT
b.id AS id,
n.number AS number,
b.nick AS nick,
b.groups AS groups,
n.state AS state,
b.image_hash AS image_hash
FROM buddies AS b
LEFT JOIN numbers AS n
ON b.buddy_id = n.id
WHERE
b.owner_id IN (%s, 0)
AND n.state >= 1
ORDER BY b.owner_id DESC""", self.owner.id)
for i in range(cur.rowcount):
id, number, nick, groups, state, image_hash = cur.fetchone()
self[number] = Buddy(self.owner, Number(number, state, self.db), nick.decode('latin1'), groups.split(","), image_hash, id, self.db)
self.lock.release()
def load(self, buddies):
if self.session.loggedIn:
self._load(buddies)
else:
self.session.loginQueue.append(lambda: self._load(buddies))
def update(self, number, nick, groups, image_hash):
self.lock.acquire()
if number in self:
buddy = self[number]
buddy.update(nick, groups, image_hash)
else:
buddy = self.add(number, nick, groups, 1, image_hash)
self.lock.release()
self.session.sendSync([number], delta = True, interactive = True)
self.session.subscribePresence(number)
buddy = Buddy(self.owner, number, nick, "", groups, image_hash)
self[number] = buddy
self.logger.debug("Roster add: %s", buddy)
if buddy.presence == 0:
status = protocol_pb2.STATUS_NONE
elif buddy.presence == 'unavailable':
status = protocol_pb2.STATUS_AWAY
else:
status = protocol_pb2.STATUS_ONLINE
self.backend.handleBuddyChanged(self.user, number, buddy.nick,
buddy.groups, status,
iconHash = buddy.image_hash if buddy.image_hash is not None else "")
return buddy
def add(self, number, nick, groups = [], state = 0, image_hash = ""):
return Buddy.create(self.owner, Number(number, state, self.db), nick, groups, image_hash, self.db)
def remove(self, number):
try:
buddy = self[number]
self.lock.acquire()
buddy.delete()
self.lock.release()
del self[number]
self.backend.handleBuddyChanged(self.user, number, "", [],
protocol_pb2.STATUS_NONE)
self.backend.handleBuddyRemoved(self.user, number)
self.session.unsubscribePresence(number)
# TODO Sync remove
return buddy
except KeyError:
return None
def prune(self):
self.lock.acquire()
cur = self.db.cursor()
cur.execute("DELETE FROM buddies WHERE owner_id = %s", self.owner.id)
self.db.commit()
self.lock.release()
def sync(self, user, password):
self.lock.acquire()
cur = self.db.cursor()
cur.execute("""SELECT
n.number AS number,
n.state AS state
FROM buddies AS r
LEFT JOIN numbers AS n
ON r.buddy_id = n.id
WHERE
r.owner_id = %s""", self.owner.id)
# prefix every number with leading 0 to force internation format
numbers = dict([("+" + number, state) for number, state in cur.fetchall()])
if len(numbers) == 0:
return 0
result = WAContactsSyncRequest(user, password, numbers.keys()).send()
using = 0
for number in result['c']:
cur = self.db.cursor()
cur.execute("UPDATE numbers SET state = %s WHERE number = %s", (number['w'], number['n']))
self.db.commit()
using += number['w']
self.lock.release()
return using

View File

@ -1,31 +0,0 @@
SET SQL_MODE="NO_AUTO_VALUE_ON_ZERO";
SET time_zone = "+00:00";
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
CREATE TABLE IF NOT EXISTS `buddies` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`owner_id` int(11) NOT NULL,
`buddy_id` int(11) NOT NULL,
`nick` varchar(255) NOT NULL,
`groups` varchar(255) NOT NULL,
`image_hash` varchar(40),
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
CREATE TABLE IF NOT EXISTS `numbers` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`number` varchar(32) NOT NULL,
`picture` blob NOT NULL,
`state` int(11) NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `number` (`number`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;

View File

@ -22,13 +22,8 @@ __status__ = "Prototype"
along with transWhat. If not, see <http://www.gnu.org/licenses/>.
"""
DB_HOST = "localhost"
DB_USER = ""
DB_PASS = ""
DB_TABLE = "transwhat"
BASE_PATH = "/opt/transwhat"
TOKEN_FILE = BASE_PATH + "/logs/tokens"
MOTD_FILE = BASE_PATH + "/conf/motd"
REQUESTS_FILE = BASE_PATH + "/logs/requests"
REQUESTS_FILE = BASE_PATH + "/logs/requests"

View File

@ -28,7 +28,6 @@ import urllib
import time
from PIL import Image
import MySQLdb
import sys
import os
@ -57,14 +56,11 @@ class MsgIDs:
class Session(YowsupApp):
def __init__(self, backend, user, legacyName, extra, db):
def __init__(self, backend, user, legacyName, extra):
super(Session, self).__init__()
self.logger = logging.getLogger(self.__class__.__name__)
self.logger.info("Created: %s", legacyName)
#self.db = db
self.db = MySQLdb.connect(DB_HOST, DB_USER, DB_PASS, DB_TABLE)
self.backend = backend
self.user = user
self.legacyName = legacyName
@ -74,12 +70,14 @@ class Session(YowsupApp):
self.groups = {}
self.gotGroupList = False
# Functions to exectute when logged in via yowsup
self.loginQueue = []
self.joinRoomQueue = []
self.presenceRequested = []
self.offlineQueue = []
self.msgIDs = { }
self.groupOfflineQueue = { }
self.shouldBeConnected = False
self.loggedIn = False
self.timer = None
self.password = None
@ -87,7 +85,7 @@ class Session(YowsupApp):
self.lastMsgId = None
self.synced = False
self.buddies = BuddyList(self.legacyName, self.db)
self.buddies = BuddyList(self.legacyName, self.backend, self.user, self)
self.bot = Bot(self)
self.imgMsgId = None
@ -102,6 +100,7 @@ class Session(YowsupApp):
def logout(self):
self.logger.info("%s logged out", self.user)
super(Session, self).logout()
self.loggedIn = False
def login(self, password):
self.logger.info("%s attempting login", self.user)
@ -132,39 +131,6 @@ class Session(YowsupApp):
'\n'.join(text) + '\nIf you do not join them you will lose messages'
#self.bot.send(message)
def updateRoster(self):
self.logger.debug("Update roster")
old = self.buddies.keys()
self.buddies.load()
new = self.buddies.keys()
contacts = new
if self.synced == False:
self.sendSync(contacts, delta = False, interactive = True)
self.synced = True
add = set(new) - set(old)
remove = set(old) - set(new)
self.logger.debug("Roster remove: %s", str(list(remove)))
self.logger.debug("Roster add: %s", str(list(add)))
for number in remove:
self.backend.handleBuddyChanged(self.user, number, "", [],
protocol_pb2.STATUS_NONE)
self.backend.handleBuddyRemoved(self.user, number)
self.unsubscribePresence(number)
for number in add:
buddy = self.buddies[number]
self.subscribePresence(number)
self.backend.handleBuddyChanged(self.user, number, buddy.nick,
buddy.groups, protocol_pb2.STATUS_NONE,
iconHash = buddy.image_hash if buddy.image_hash is not None else "")
#self.requestLastSeen(number, self._lastSeen)
def _updateGroups(self, response, request):
self.logger.debug('Received groups list %s', response)
groups = response.getGroups()
@ -278,17 +244,19 @@ class Session(YowsupApp):
#self.bot.call("welcome")
self.initialized = True
self.sendPresence(True)
self.updateRoster()
for func in self.loginQueue:
func()
self.logger.debug('Requesting groups list')
self.requestGroupsList(self._updateGroups)
self.loggedIn = True
# Called by superclass
def onAuthFailed(self, reason):
self.logger.info("Auth failed: %s (%s)", self.user, reason)
self.backend.handleDisconnected(self.user, 0, reason)
self.password = None
self.shouldBeConnected = False
self.loggedIn = False
# Called by superclass
def onDisconnect(self):
@ -305,7 +273,7 @@ class Session(YowsupApp):
buddy = self.buddies[_from.split('@')[0]]
#self.backend.handleBuddyChanged(self.user, buddy.number.number,
# buddy.nick, buddy.groups, protocol_pb2.STATUS_ONLINE)
self.backend.handleMessageAck(self.user, buddy.number.number, self.msgIDs[_id].xmppId)
self.backend.handleMessageAck(self.user, buddy.number, self.msgIDs[_id].xmppId)
self.msgIDs[_id].cnt = self.msgIDs[_id].cnt +1
if self.msgIDs[_id].cnt == 2:
del self.msgIDs[_id]
@ -523,8 +491,8 @@ class Session(YowsupApp):
if (lastseen == str(buddy.lastseen)) and (_type == buddy.presence):
return
if ((lastseen != "deny") and (lastseen != None) and (lastseen != "none")):
if ((lastseen != "deny") and (lastseen != None) and (lastseen != "none")):
buddy.lastseen = int(lastseen)
if (_type == None):
buddy.lastseen = time.time()
@ -533,22 +501,22 @@ class Session(YowsupApp):
timestamp = time.localtime(buddy.lastseen)
statusmsg = buddy.statusMsg + time.strftime("\n Last seen: %a, %d %b %Y %H:%M:%S", timestamp)
if _type == "unavailable":
self.onPresenceUnavailable(buddy, statusmsg)
else:
self.onPresenceAvailable(buddy, statusmsg)
def onPresenceAvailable(self, buddy, statusmsg):
def onPresenceAvailable(self, buddy, statusmsg):
self.logger.info("Is available: %s", buddy)
self.backend.handleBuddyChanged(self.user, buddy.number.number,
self.backend.handleBuddyChanged(self.user, buddy.number,
buddy.nick, buddy.groups, protocol_pb2.STATUS_ONLINE, statusmsg, buddy.image_hash)
def onPresenceUnavailable(self, buddy, statusmsg):
self.logger.info("Is unavailable: %s", buddy)
self.backend.handleBuddyChanged(self.user, buddy.number.number,
self.backend.handleBuddyChanged(self.user, buddy.number,
buddy.nick, buddy.groups, protocol_pb2.STATUS_AWAY, statusmsg, buddy.image_hash)
# spectrum RequestMethods
@ -789,19 +757,19 @@ class Session(YowsupApp):
msg = self.offlineQueue.pop(0)
self.backend.handleMessage(self.user, msg[0], msg[1], "", "", msg[2])
# Called when user logs in to initialize the roster
def loadBuddies(self, buddies):
self.buddies.load(buddies)
# also for adding a new buddy
def updateBuddy(self, buddy, nick, groups, image_hash = None):
if buddy != "bot":
self.buddies.update(buddy, nick, groups, image_hash)
if self.initialized == True:
self.updateRoster()
def removeBuddy(self, buddy):
if buddy != "bot":
self.logger.info("Buddy removed: %s", buddy)
self.buddies.remove(buddy)
self.updateRoster()
def requestVCard(self, buddy, ID=None):
def onSuccess(response, request):
@ -828,7 +796,7 @@ class Session(YowsupApp):
name=os.path.basename(self.imgPath)
self.logger.info("Buddy %s",self.imgBuddy)
self.image_send(self.imgBuddy, self.imgPath)
#self.logger.info("Sending picture %s of size %s with name %s",self.imgPath, statinfo.st_size, name)
#mtype = "image"

View File

@ -29,7 +29,6 @@ import traceback
import logging
import asyncore
import sys, os
import MySQLdb
import e4u
import threading
import Queue
@ -75,10 +74,11 @@ def connectionClosed():
closed = True
# Main
db = MySQLdb.connect(DB_HOST, DB_USER, DB_PASS, DB_TABLE)
io = IOChannel(args.host, args.port, handleTransportData, connectionClosed)
plugin = WhatsAppBackend(io, db, args.j)
plugin = WhatsAppBackend(io, args.j)
plugin.handleBackendConfig('features', 'send_buddies_on_login', 1)
while True:
try:

View File

@ -30,11 +30,10 @@ from session import Session
import logging
class WhatsAppBackend(SpectrumBackend):
def __init__(self, io, db, spectrum_jid):
def __init__(self, io, spectrum_jid):
SpectrumBackend.__init__(self)
self.logger = logging.getLogger(self.__class__.__name__)
self.io = io
self.db = db
self.sessions = { }
self.spectrum_jid = spectrum_jid
# Used to prevent duplicate messages
@ -46,7 +45,7 @@ class WhatsAppBackend(SpectrumBackend):
def handleLoginRequest(self, user, legacyName, password, extra):
self.logger.debug("handleLoginRequest(user=%s, legacyName=%s)", user, legacyName)
if user not in self.sessions:
self.sessions[user] = Session(self, user, legacyName, extra, self.db)
self.sessions[user] = Session(self, user, legacyName, extra)
if user not in self.lastMessage:
self.lastMessage[user] = {}
@ -87,6 +86,13 @@ class WhatsAppBackend(SpectrumBackend):
self.sessions[user].changeStatusMessage(statusMessage)
self.sessions[user].changeStatus(status)
def handleBuddies(self, buddies):
"""Called when user logs in. Used to initialize roster."""
self.logger.debug("handleBuddies(buddies=%s)", buddies)
buddies = [b for b in buddies.buddy]
user = buddies[0].userName
self.sessions[user].loadBuddies(buddies)
def handleBuddyUpdatedRequest(self, user, buddy, nick, groups):
self.logger.debug("handleBuddyUpdatedRequest(user=%s, buddy=%s, nick=%s, groups=%s)", user, buddy, nick, str(groups))
self.sessions[user].updateBuddy(buddy, nick, groups)
@ -139,7 +145,7 @@ class WhatsAppBackend(SpectrumBackend):
pass
def handleMessageAckRequest(self, user, legacyName, ID = 0):
self.logger.info("Meassage ACK request for %s !!",leagcyName)
self.logger.info("Meassage ACK request for %s !!",legacyName)
def sendData(self, data):
self.io.sendData(data)

View File

@ -216,6 +216,7 @@ class YowsupApp(object):
- phone_number: (str) The cellphone number of the person to
subscribe to
"""
self.logger.debug("Subscribing to Presence updates from %s", (phone_number))
jid = phone_number + '@s.whatsapp.net'
entity = SubscribePresenceProtocolEntity(jid)
self.sendEntity(entity)