msg can work

.gitignore

@@ -0,0 +1,2 @@
+*.pyc
+*.db

dump.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python2
 # -*- coding: UTF-8 -*-
 # File: dump.py
-# Date: Fri Nov 21 13:07:58 2014 +0800
+# Date: Fri Nov 21 14:08:45 2014 +0800
 # Author: Yuxin Wu <[email protected]>
 
 import sqlite3
@@ -10,6 +10,7 @@ from pprint import PrettyPrinter
 pp = PrettyPrinter()
 def log(x): print repr(x).decode('unicode-escape')
 from lib.Msg import WeChatMsg
+from lib.utils import ensure_unicode
 
 """ tables in concern:
 emojiinfo
@@ -28,42 +29,48 @@ class WeChatDBParser(object):
         self.db_conn = sqlite3.connect(self.db_fname)
         self.cc = self.db_conn.cursor()
         self.contacts = {}
+        self.msgs_by_talker = defaultdict(list)
 
     def _parse_contact(self):
         contacts = self.cc.execute(
 """
-SELECT username,conRemark,nickname
-FROM rcontact
+SELECT username,conRemark,nickname FROM rcontact
 """)
         for row in contacts:
             username, remark, nickname = row
             if remark:
-                self.contacts[username] = remark
+                self.contacts[username] = ensure_unicode(remark)
             else:
-                self.contacts[username] = nickname
+                self.contacts[username] = ensure_unicode(nickname)
 
     def _parse_msg(self):
-        msgs_by_talker = defaultdict(list)
         db_msgs = self.cc.execute(
 """
-SELECT {}
-FROM message ORDER BY createTime
+SELECT {} FROM message
 """.format(','.join(WeChatMsg.FIELDS)))
         for row in db_msgs:
             msg = WeChatMsg(row)
-            msgs_by_talker[msg.talker].append(msg)
-        msgs_by_talker = dict([
+            if msg.type not in WeChatMsg.FILTER_TYPES:
+                self.msgs_by_talker[msg.talker].append(msg)
+        self.msgs_by_talker = dict([
             (self.contacts[k], sorted(v, key=lambda x: x.createTime))
-                           for k, v in msgs_by_talker.iteritems()])
-        for k, v in msgs_by_talker.iteritems():
+                           for k, v in self.msgs_by_talker.iteritems()])
+        for k, v in self.msgs_by_talker.iteritems():
             for msg in v:
                 msg.talker = k
-        pp.pprint(msgs_by_talker.items()[0])
+
+    def _find_msg_by_type(self):
+        ret = []
+        for v in self.msgs_by_talker.itervalues():
+            for msg in v:
+                if msg.type == 34:
+                    print msg
+                    print
+        return ret
 
     def parse(self):
         self._parse_contact()
-        msg = self._parse_msg()
-        pass
+        self._parse_msg()
 
 if __name__ == '__main__':
     parser = WeChatDBParser('./decrypted_database.db')

lib/Msg.py

@@ -1,14 +1,33 @@
 #!/usr/bin/env python2
 # -*- coding: UTF-8 -*-
 # File: Msg.py
-# Date: Fri Nov 21 12:15:27 2014 +0800
+# Date: Fri Nov 21 14:08:33 2014 +0800
 # Author: Yuxin Wu <[email protected]>
 
 from datetime import datetime
+from bs4 import BeautifulSoup
+from .utils import ensure_bin_str, ensure_unicode
+
+TYPE_MSG = 1
+TYPE_IMG = 3
+TYPE_SPEAK = 34
+TYPE_VIDEO = 43
+TYPE_EMOJI = 47
+TYPE_LOCATION = 48
+TYPE_LINK = 49
+TYPE_VOIP = 50
+TYPE_SYSTEM = 10000
 
 class WeChatMsg(object):
     """ fields in concern"""
     FIELDS = ["msgSvrId","type","isSend","createTime","talker","content","imgPath"]
+    FILTER_TYPES = [TYPE_SYSTEM]
+
+    @staticmethod
+    def filter_types(tp):
+        if tp in WeChatMsg.FILTER_TYPES or tp > 10000 or tp < 0:
+            return True
+        return False
 
     def __init__(self, row):
         """ row: a tuple corresponding to FIELDS"""
@@ -17,7 +36,33 @@ class WeChatMsg(object):
             setattr(self, f, v)
         self.createTime = datetime.fromtimestamp(self.createTime / 1000)
         if self.content:
-            self.content = self.content.encode('utf-8')
+            self.content = ensure_unicode(self.content)
+
+    def msg_str(self):
+        # TODO: fix more types
+        if self.type == TYPE_LOCATION:
+            soup = BeautifulSoup(self.content)
+            loc = soup.find('location')
+            label = loc['label']
+            ret = label
+            try:
+                poiname = loc['poiname']
+                if poiname:
+                    ret = poiname
+            except:
+                pass
+            return ret + " ({},{})".format(loc['x'], loc['y'])
+        else:
+            return self.content
 
     def __repr__(self):
-        return "{}:{}:{}".format(self.talker.encode('utf-8') if not self.isSend else 'me', self.createTime, self.content)
+        ret = u"{}|{}:{}:{}".format(
+            self.type,
+            ensure_unicode(self.talker) if not self.isSend else 'me',
+            self.createTime,
+            ensure_unicode(self.msg_str())).encode('utf-8')
+        if self.imgPath:
+            ret = u"{}|img:{}".format(ensure_unicode(ret), self.imgPath)
+            return ret.encode('utf-8')
+        else:
+            return ret

lib/utils.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python2
+# -*- coding: UTF-8 -*-
+# File: utils.py
+# Date: Fri Nov 21 13:42:56 2014 +0800
+# Author: Yuxin Wu <[email protected]>
+
+def ensure_bin_str(s):
+    if type(s) == str:
+        return s
+    if type(s) == unicode:
+        return s.encode('utf-8')
+
+def ensure_unicode(s):
+    if type(s) == str:
+        return s.decode('utf-8')
+    if type(s) == unicode:
+        return s