commit 68c0e92aaccb9b9b1b9391f9f0ee3251f41de339 Author: ppwwyyxx Date: Fri Nov 21 13:09:01 2014 +0800 init diff --git a/README.md b/README.md new file mode 100644 index 0000000..1df6cc3 --- /dev/null +++ b/README.md @@ -0,0 +1,97 @@ +## Render WeChat Messages from Android + +## 安卓微信消息记录分析工具 + +WeChat, as the most popular mobile IM app in China, doesn't provide any methods to read its message history. +Customers are not able to analyze their own chat messages or interact with other data analysis tools. + +We provide this tool that can parse WeChat messages from a rooted android phone. +This is necessary to provide interoperability between WeChat messages and other message analysis tools. +As examples, we provide sample scripts to obtain statistics of the message history and +render the messages into self-contained html files including voice messages, images, emojis, videos, etc. +Users can also write custom programs based on this tool to manage their chat messages. + +The tool is last verified to work with latest version of WeChat on 2025/01/01. +If the tool works for you, please take a moment to add your phone/OS to [the wiki](https://github.com/ppwwyyxx/wechat-dump/wiki). + +## How to use: + +#### Dependencies: ++ adb and rooted android phone connected to a Linux/Mac OSX/Win10+Bash. ++ Python >= 3.8 ++ sox (command line tools) ++ Silk audio decoder (included; build it with `./third-party/compile_silk.sh`) ++ Other python dependencies: `pip install -r requirements.txt`. + +#### Get Necessary Data: + +1. Pull database file and (for older WeChat versions) avatar index: + + Automatic: `./android-interact.sh db`. It may use an incorrect userid. + + Manual: + + Figure out your `${userid}` by inspecting the contents of `/data/data/com.tencent.mm/MicroMsg` on the __root__ filesystem of the device. + It should be a 32-character-long name consisting of hexadecimal digits. + + Get `/data/data/com.tencent.mm/MicroMsg/${userid}/EnMicroMsg.db` from the device. +2. Decode `EnMicroMsg.db`. We do not provide instructions to do that. +3. Copy the unencrypted WeChat user resource directory `/data/data/com.tencent.mm/MicroMsg/${userid}/{avatar,emoji,image2,sfs,video,voice2}` from the phone to the `resource` directory: + + `./android-interact.sh res` + + Change `RES_DIR` in the script if the location of these directories is different on your phone. + For older version of WeChat, the directory may be `/mnt/sdcard/tencent/MicroMsg/` + + This can take a while. It can be faster to first archive it with `tar` with or without compression, and then copy the archive, + `busybox tar` is recommended as the Android system's `tar` may choke on long paths. + + In the end, we need a `resource` directory with the following subdir: `avatar,emoji,image2,sfs,video,voice2`. + +4. (Optional) Install and start a WXGF decoder server on an android device. Without this, certain WXGF images will not be rendered or will be rendered in low resolution. + See [WXGFDecoder](WXGFDecoder) for instructions. + +4. (Optional) Download the emoji cache from [here](https://github.com/ppwwyyxx/wechat-dump/releases/download/0.1/emoji.cache.tar.bz2) + and decompress it under `wechat-dump`. This will avoid downloading too many emojis during rendering. + + wget -c https://github.com/ppwwyyxx/wechat-dump/releases/download/0.1/emoji.cache.tar.bz2 + tar xf emoji.cache.tar.bz2 + +#### Run: ++ Parse and dump text messages of __every__ chat (requires decoded database): + + ``` + ./dump-msg.py decoded.db output_dir + ``` + ++ List all chats (required decoded database): + + ``` + ./list-chats.py decoded.db + ``` + ++ Generate statistics report on text messages (requires `output_dir` from `./dump-msg.py`): + + ``` + ./count-message.sh output_dir + ``` + ++ Dump messages of one contact to html, containing voice messages, emojis, and images (requires decoded database and `resource`): + + ``` + ./dump-html.py "" + ``` + + * The output file is `output.html`. Check `./dump-html.py -h` to use different input/output paths. + * Add `--wxgf-server ws://xx.xx.xx.xx:xxxx` to use a WXGF decoder server. + +### Examples: +Screenshots of generated html: + +![byvoid](https://github.com/ppwwyyxx/wechat-dump/raw/master/screenshots/byvoid.jpg) + +See [here](http://ppwwyyxx.com/static/wechat/example.html) for an example html. + +### TODO List (help needed!) +* After chat history migration, some emojis in the `EmojiInfo` table don't have corresponding URLs but only a md5 - + they are not downloaded by WeChat until the message needs to be displayed. We don't know how to manually download these emojis. +* Decoding WXGF images using an android app is too complex. Looking for an easier way (e.g. qemu). +* Fix rare unhandled message types: > 10000 and < 0 +* Better user experiences... see `grep 'TODO' wechat -R` + +### Donate! + +[paypal] + diff --git a/decrypt_db.sh b/decrypt_db.sh new file mode 100755 index 0000000..a5d02a3 --- /dev/null +++ b/decrypt_db.sh @@ -0,0 +1,23 @@ +#!/bin/bash -e +# File: decrypt_db.sh +# Date: Fri Nov 21 13:07:33 2014 +0800 +# Author: Yuxin Wu + +MMSGDB=$1 +imei=$2 +uin=$3 + +if [[ -z "$1" || -z "$2" || -z "$3" ]]; then + echo "Usage: $0 " + exit +fi + +KEY=$(echo -n "$imei$uin" | md5sum | cut -b 1-7) + +LD_LIBRARY_PATH=./lib ./lib/sqlcipher $MMSGDB << EOF +PRAGMA key='$KEY'; +PRAGMA cipher_use_hmac = off; +ATTACH DATABASE "decoded_database.db" AS decoded_database KEY ""; +SELECT sqlcipher_export("decoded_database"); +DETACH DATABASE decoded_database; +EOF diff --git a/dump.py b/dump.py new file mode 100755 index 0000000..3e97f37 --- /dev/null +++ b/dump.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python2 +# -*- coding: UTF-8 -*- +# File: dump.py +# Date: Fri Nov 21 13:07:58 2014 +0800 +# Author: Yuxin Wu + +import sqlite3 +from collections import defaultdict +from pprint import PrettyPrinter +pp = PrettyPrinter() +def log(x): print repr(x).decode('unicode-escape') +from lib.Msg import WeChatMsg + +""" tables in concern: +emojiinfo +imginfo2 +addr_upload2 +chatroom +message +rcontact + +""" + +class WeChatDBParser(object): + def __init__(self, db_fname): + """ db_fname: EnMicroMsg.db""" + self.db_fname = db_fname + self.db_conn = sqlite3.connect(self.db_fname) + self.cc = self.db_conn.cursor() + self.contacts = {} + + def _parse_contact(self): + contacts = self.cc.execute( +""" +SELECT username,conRemark,nickname +FROM rcontact +""") + for row in contacts: + username, remark, nickname = row + if remark: + self.contacts[username] = remark + else: + self.contacts[username] = nickname + + def _parse_msg(self): + msgs_by_talker = defaultdict(list) + db_msgs = self.cc.execute( +""" +SELECT {} +FROM message ORDER BY createTime +""".format(','.join(WeChatMsg.FIELDS))) + for row in db_msgs: + msg = WeChatMsg(row) + msgs_by_talker[msg.talker].append(msg) + msgs_by_talker = dict([ + (self.contacts[k], sorted(v, key=lambda x: x.createTime)) + for k, v in msgs_by_talker.iteritems()]) + for k, v in msgs_by_talker.iteritems(): + for msg in v: + msg.talker = k + pp.pprint(msgs_by_talker.items()[0]) + + def parse(self): + self._parse_contact() + msg = self._parse_msg() + pass + +if __name__ == '__main__': + parser = WeChatDBParser('./decoded_database.db') + parser.parse() diff --git a/lib/Msg.py b/lib/Msg.py new file mode 100644 index 0000000..40c3714 --- /dev/null +++ b/lib/Msg.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python2 +# -*- coding: UTF-8 -*- +# File: Msg.py +# Date: Fri Nov 21 12:15:27 2014 +0800 +# Author: Yuxin Wu + +from datetime import datetime + +class WeChatMsg(object): + """ fields in concern""" + FIELDS = ["msgSvrId","type","isSend","createTime","talker","content","imgPath"] + + def __init__(self, row): + """ row: a tuple corresponding to FIELDS""" + assert len(row) == len(WeChatMsg.FIELDS) + for f, v in zip(WeChatMsg.FIELDS, row): + setattr(self, f, v) + self.createTime = datetime.fromtimestamp(self.createTime / 1000) + if self.content: + self.content = self.content.encode('utf-8') + + def __repr__(self): + return "{}:{}:{}".format(self.talker.encode('utf-8') if not self.isSend else 'me', self.createTime, self.content) diff --git a/lib/sqlcipher b/lib/sqlcipher new file mode 100755 index 0000000..4f3b3d4 Binary files /dev/null and b/lib/sqlcipher differ