From 99c5ed6d3355a63f7fc948f278c5185567019b4a Mon Sep 17 00:00:00 2001 From: Yuxin Wu Date: Sun, 28 Jun 2020 00:10:57 -0700 Subject: [PATCH] fix old avatar storage --- .gitignore | 1 + README.md | 14 ++++----- android-interact.sh | 48 ++++++++++++++++------------ decrypt-db.py | 2 +- plot-num-msg-by-time.py | 3 -- wechat/avatar.py | 69 +++++++++++++++++++++++------------------ 6 files changed, 76 insertions(+), 61 deletions(-) diff --git a/.gitignore b/.gitignore index 3dc2d95..81989f2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ output/ resource *.db +*.db.decrypted *2015* *2017* 2020* diff --git a/README.md b/README.md index c02d444..9d4f1f5 100644 --- a/README.md +++ b/README.md @@ -53,17 +53,15 @@ If it doesn't work, you probably have to investigate it as the behavior may be d If decryption doesn't work, you can also try the [password cracker](https://github.com/chg-hou/EnMicroMsg.db-Password-Cracker) to brute-force the password. -+ Copy the WeChat user resource directory `/mnt/sdcard/tencent/MicroMsg/${userid}/{avatar,emoji,image2,sfs,video,voice2}` from the phone to the `resource` directory: +3. Copy the WeChat user resource directory `/mnt/sdcard/tencent/MicroMsg/${userid}/{avatar,emoji,image2,sfs,video,voice2}` from the phone to the `resource` directory: + `./android-interact.sh res` + You might need to change `RES_DIR` in the script if the default is incorrect on your phone. - + This script needs tar and base64 command on your phone. - If they are not available, there is a slow fallback method in the script you can use. - + This can take a few minutes. One way to do it faster: - + If there's enough free space on your phone, you can log in and archive all required files via `tar` with or without compression, - and use `adb pull` to copy the archive. Note that `busybox` is needed as the Android system's `tar` may choke on long paths. - + What you'll need in the end is a `resource` directory with the following subdir: `avatar,emoji,image2,sfs,video,voice2`. + + This can take a while. One way that might be slightly faster: + + If there's enough free space on your phone, you can log in and archive all required files via `tar` with or without compression, + and use `adb pull` to copy the archive. Note that `busybox tar` is recommended as the Android system's `tar` may choke on long paths. + + What is needed in the end is a `resource` directory with the following subdir: `avatar,emoji,image2,sfs,video,voice2`. -+ (Optional) Download the emoji cache from [here](https://github.com/ppwwyyxx/wechat-dump/releases/download/0.1/emoji.cache.tar.bz2) +4. (Optional) Download the emoji cache from [here](https://github.com/ppwwyyxx/wechat-dump/releases/download/0.1/emoji.cache.tar.bz2) and decompress it under `wechat-dump`. This will avoid downloading too many emojis during rendering. wget -c https://github.com/ppwwyyxx/wechat-dump/releases/download/0.1/emoji.cache.tar.bz2 diff --git a/android-interact.sh b/android-interact.sh index 858cdc6..b812541 100755 --- a/android-interact.sh +++ b/android-interact.sh @@ -1,6 +1,5 @@ #!/bin/bash # File: android-interact.sh -# Date: Wed Nov 29 02:19:00 2017 -0800 PROG_NAME=`python -c "import os, sys; print(os.path.realpath(sys.argv[1]))" "$0"` PROG_DIR=`dirname "$PROG_NAME"` @@ -29,25 +28,36 @@ if [[ $1 == "db" || $1 == "res" ]]; then echo "Found $numUser user(s). User chosen: $chooseUser" if [[ $1 == "res" ]]; then - mkdir -p resource; cd resource - echo "Pulling resources... " - for d in avatar image2 voice2 emoji video sfs; do - adb shell "cd $RES_DIR/$chooseUser && - tar czf - $d 2>/dev/null | base64" | - base64 -di | tar xzf - + mkdir -p resource + ( + cd resource || exit + echo "Pulling resources... " + for d in avatar image2 voice2 emoji video sfs; do + echo "Trying to download $RES_DIR/$chooseUser/$d with busybox ..." + adb shell "cd $RES_DIR/$chooseUser && + busybox tar czf - $d 2>/dev/null | busybox base64" | + base64 -di | tar xzf - + [[ -d $d ]] && continue - # Old Slow Way: - # mkdir -p $d; cd $d - # adb pull "$RES_DIR/$chooseUser/$d" - # cd .. - [[ -d $d ]] || { - >&2 echo "Failed to download resource directory: $RES_DIR/$chooseUser/$d" - exit 1 - } - done - cd .. - echo "Resource pulled at ./resource" - echo "Total size: $(du -sh resource | cut -f1)" + echo "Trying to download $RES_DIR/$chooseUser/$d with tar & base64 ..." + adb shell "cd $RES_DIR/$chooseUser && + tar czf - $d 2>/dev/null | base64" | base64 -di | tar xzf - + [[ -d $d ]] && continue + + echo "Trying to download $RES_DIR/$chooseUser/$d with adb pull (slow) ..." + mkdir -p $d + ( + cd $d || exit + adb pull "$RES_DIR/$chooseUser/$d" + ) + + [[ -d $d ]] || { + echo "Failed to download $RES_DIR/$chooseUser/$d" + } + done + echo "Resource pulled at ./resource" + echo "Total size: $(du -sh | cut -f1)" + ) else echo "Pulling database and avatar index file..." adb pull $MM_DIR/MicroMsg/$chooseUser/EnMicroMsg.db diff --git a/decrypt-db.py b/decrypt-db.py index e6a20b4..07bf40c 100755 --- a/decrypt-db.py +++ b/decrypt-db.py @@ -103,8 +103,8 @@ def get_imei(): except: logger.warning("imei not found in CompatibleInfo.cfg") else: + candidates.append(imei) logger.info(f"found imei={imei} in CompatibleInfo.cfg") - candidates.append(imei) logger.info(f"Possible imei: {candidates}") return [x.encode('ascii') for x in set(candidates)] diff --git a/plot-num-msg-by-time.py b/plot-num-msg-by-time.py index ada4692..9727878 100755 --- a/plot-num-msg-by-time.py +++ b/plot-num-msg-by-time.py @@ -1,8 +1,5 @@ #!/usr/bin/env python3 # -*- coding: UTF-8 -*- -# File: plot-num-msg-by-time.py -# Date: Wed Mar 25 17:44:39 2015 +0800 -# Author: Yuxin Wu from wechat.parser import WeChatDBParser from wechat.common.textutil import ensure_unicode diff --git a/wechat/avatar.py b/wechat/avatar.py index 6bd7779..dde908e 100644 --- a/wechat/avatar.py +++ b/wechat/avatar.py @@ -13,6 +13,13 @@ logger = logging.getLogger(__name__) from .common.textutil import ensure_unicode, md5 +def _filename_priority(s): + if "_hd" in s and s.endswith(".png"): + return 10 + else: + return 1 + + class AvatarReader(object): def __init__(self, res_dir, avt_db="avatar.index"): self.sfs_dir = os.path.join(res_dir, 'sfs') @@ -22,14 +29,15 @@ class AvatarReader(object): self.avt_db = avt_db self._use_avt = True if os.path.isdir(self.avt_dir) and len(os.listdir(self.avt_dir)): - self.avt_use_db = False + # has resource/avatar + pass elif self.avt_db is not None \ and os.path.isfile(self.avt_db) \ and glob.glob(os.path.join(self.sfs_dir, 'avatar*')): - self.avt_use_db = True + # has sfs/avatar* + pass else: - logger.warn( - "Cannot find avatar files. Will not use avatar!") + logger.warn("Cannot find avatar storage. Will not use avatar!") self._use_avt = False def get_avatar(self, username): @@ -38,41 +46,39 @@ class AvatarReader(object): return None username = ensure_unicode(username) avtid = md5(username.encode('utf-8')) + try: + candidates = self.query_index(avtid) + candidates = sorted(candidates, key=lambda x: _filename_priority(x[0]), reverse=True) + for c in candidates: + path, offset, size = c + return self.read_img_from_block(path, offset, size) + except: + pass + dir1, dir2 = avtid[:2], avtid[2:4] candidates = glob.glob(os.path.join(self.avt_dir, dir1, dir2, f"*{avtid}*")) default_candidate = os.path.join(self.avt_dir, dir1, dir2, f"user_{avtid}.png") candidates.append(default_candidate) - def priority(s): - if "_hd" in s and s.endswith(".png"): - return 10 - else: - return 1 - - candidates = sorted(set(candidates), key=priority, reverse=True) - + candidates = sorted(set(candidates), key=_filename_priority, reverse=True) for cand in candidates: try: - if self.avt_use_db: - pos, size = self.query_index(cand) - return self.read_img(pos, size) - else: - if os.path.exists(cand): - if cand.endswith(".bm"): - return self.read_bm_file(cand) - else: - return Image.open(cand) + if os.path.exists(cand): + if cand.endswith(".bm"): + return self.read_bm_file(cand) + else: + return Image.open(cand) except Exception: - logger.exception("HHH") + logger.exception("") pass - logger.warning("Avatar for {} not found in avatar database.".format(username)) + logger.warning("Avatar for {} not found anywhere.".format(username)) - def read_img(self, pos, size): + def read_img_from_block(self, filename, pos, size): file_idx = pos >> 32 fname = os.path.join(self.sfs_dir, 'avatar.block.' + '{:05d}'.format(file_idx)) - # a 64-byte offset of each block file - start_pos = pos - file_idx * (2**32) + 64 + # offset of each block file: 17 + len(path) + start_pos = pos - file_idx * (2**32) + 16 + len(filename) + 1 try: with open(fname, 'rb') as f: f.seek(start_pos) @@ -95,11 +101,14 @@ class AvatarReader(object): img[i,j] = (r, g, b) return Image.fromarray(img, mode="RGB") - def query_index(self, filename): + def query_index(self, avtid): conn = sqlite3.connect(self.avt_db) - cursor = conn.execute("select Offset,Size from Index_avatar where FileName='{}'".format(filename)) - pos, size = cursor.fetchone() - return pos, size + cursor = conn.execute("select FileName,Offset,Size from Index_avatar") + candidates = [] + for path, offset, size in cursor: + if avtid in path: + candidates.append((path, offset, size)) + return candidates if __name__ == '__main__': import sys