diff --git a/README.md b/README.md index 09ab7fd..92bf89a 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,6 @@ __NEWS__: WeChat 6.0+ use silk to encode audio. The code is updated. + python-PIL + [PyQuery](https://pypi.python.org/pypi/pyquery/1.2.1) + [pysox](https://pypi.python.org/pypi/pysox/0.3.6.alpha) -+ [dill](https://pypi.python.org/pypi/dill) + numpy + python-csscompressor(optional) + adb and rooted android phone connected to PC diff --git a/common/__init__.py b/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/common/progress.py b/common/progress.py new file mode 100644 index 0000000..cdd9021 --- /dev/null +++ b/common/progress.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python2 +# -*- coding: UTF-8 -*- +# File: progress.py +# Date: Wed Jun 17 23:59:52 2015 +0800 +# Author: Yuxin Wu + +import time +import sys + +class ProgressReporter(object): + """report progress of long-term jobs""" + _start_time = None + _prev_report_time = 0 + _cnt = 0 + _name = None + _total = None + + def __init__(self, name, total=0, fout=sys.stderr): + self._start_time = time.time() + self._name = name + self._total = int(total) + self._fout = fout + + @property + def total_time(self): + return time.time() - self._start_time + + def trigger(self, delta=1, extra_msg='', target_cnt=None): + if target_cnt is None: + self._cnt += int(delta) + else: + self._cnt = int(target_cnt) + now = time.time() + if now - self._prev_report_time < 0.5: + return + self._prev_report_time = now + dt = now - self._start_time + if self._total and self._cnt > 0: + eta_msg = '{}/{} ETA: {:.2f}'.format(self._cnt, self._total, + (self._total-self._cnt)*dt/self._cnt) + else: + eta_msg = '{} done'.format(self._cnt) + self._fout.write(u'{}: avg {:.3f}/sec' + u', passed {:.3f}sec, {} {} \r'.format( + self._name, self._cnt / dt, dt, eta_msg, extra_msg)) + self._fout.flush() + + def finish(self): + """:return: total time""" + self._fout.write('\n') + self._fout.flush() + return self.total_time + diff --git a/common/textutil.py b/common/textutil.py new file mode 100644 index 0000000..be71849 --- /dev/null +++ b/common/textutil.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python2 +# -*- coding: UTF-8 -*- +# File: utils.py +# Date: Wed Jun 17 23:59:25 2015 +0800 +# Author: Yuxin Wu + +import hashlib +import base64 + +def ensure_bin_str(s): + if type(s) == str: + return s + if type(s) == unicode: + return s.encode('utf-8') + +def ensure_unicode(s): + if type(s) == str: + return s.decode('utf-8') + if type(s) == unicode: + return s + + +def md5(s): + m = hashlib.md5() + m.update(s) + return m.hexdigest() + +def get_file_b64(fname): + data = open(fname, 'rb').read() + return base64.b64encode(data) + +def safe_filename(fname): + filename = ensure_unicode(fname) + return "".join( + [c for c in filename if c.isalpha() or c.isdigit() or c==' ']).rstrip() diff --git a/common/timer.py b/common/timer.py new file mode 100644 index 0000000..914d55d --- /dev/null +++ b/common/timer.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python2 +# -*- coding: UTF-8 -*- +# File: timer.py +# Date: Wed Jun 17 23:25:54 2015 +0800 +# Author: Yuxin Wu + +import time, functools +from collections import defaultdict +import logging +logger = logging.getLogger(__name__) + +class TotalTimer(object): + def __init__(self): + self.times = defaultdict(float) + + def add(self, name, t): + self.times[name] += t + + def reset(self): + self.times = defaultdict(float) + + def __del__(self): + for k, v in self.times.iteritems(): + logger.info("{} took {} seconds in total.".format(k, v)) + +_total_timer = TotalTimer() +class timing(object): + def __init__(self, total=False): + self.total = total + + def __call__(self, func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + start_time = time.time() + ret = func(*args, **kwargs) + duration = time.time() - start_time + + if hasattr(func, '__name__'): + func_name = func.__name__ + else: + func_name = 'function in module {}'.format(func.__module__) + if self.total: + _total_timer.add(func_name, duration) + else: + logger.info('Duration for `{}\': {}'.format( + func_name, duration)) + return ret + return wrapper + diff --git a/wechat/audio.py b/wechat/audio.py index 93034a9..90ffb5f 100644 --- a/wechat/audio.py +++ b/wechat/audio.py @@ -1,7 +1,7 @@ #!/usr/bin/env python2 # -*- coding: UTF-8 -*- # File: audio.py -# Date: Tue Jun 16 23:16:11 2015 +0800 +# Date: Thu Jun 18 00:01:49 2015 +0800 # Author: Yuxin Wu import os @@ -11,7 +11,7 @@ logger = logging.getLogger(__name__) import pysox -from .utils import get_file_b64 +from common.textutil import get_file_b64 SILK_DECODER = os.path.join(os.path.dirname(__file__), '../third-party/silk/decoder') diff --git a/wechat/avatar.py b/wechat/avatar.py index 578ae95..68776b0 100644 --- a/wechat/avatar.py +++ b/wechat/avatar.py @@ -1,7 +1,7 @@ #!/usr/bin/env python2 # -*- coding: UTF-8 -*- # File: avatar.py -# Date: Fri Jan 09 22:41:55 2015 +0800 +# Date: Thu Jun 18 00:02:07 2015 +0800 # Author: Yuxin Wu import os @@ -9,7 +9,7 @@ import numpy as np import logging logger = logging.getLogger(__name__) -from .utils import ensure_bin_str, md5 +from common.textutil import ensure_bin_str, md5 class AvatarReader(object): def __init__(self, avt_dir): diff --git a/wechat/libchathelper.py b/wechat/libchathelper.py index 80fb646..cfe9014 100644 --- a/wechat/libchathelper.py +++ b/wechat/libchathelper.py @@ -1,7 +1,7 @@ #!/usr/bin/env python2 # -*- coding: UTF-8 -*- # File: libchathelper.py -# Date: Fri Mar 27 22:25:14 2015 +0800 +# Date: Thu Jun 18 00:02:35 2015 +0800 # Author: Yuxin Wu import base64 @@ -12,7 +12,8 @@ logger = logging.getLogger(__name__) from libchat.libchat import SqliteLibChat, ChatMsg from .msg import * -from .utils import timing, ProgressReporter +from common.timer import timing +from common.progress import ProgressReporter class LibChatHelper(object): """ Build LibChat messages from WeChat Msg""" diff --git a/wechat/msg.py b/wechat/msg.py index d5f6e69..96c2bfa 100644 --- a/wechat/msg.py +++ b/wechat/msg.py @@ -1,7 +1,7 @@ #!/usr/bin/env python2 # -*- coding: UTF-8 -*- # File: msg.py -# Date: Wed May 06 10:47:57 2015 +0800 +# Date: Thu Jun 18 00:01:00 2015 +0800 # Author: Yuxin Wu TYPE_MSG = 1 TYPE_IMG = 3 @@ -26,7 +26,7 @@ from pyquery import PyQuery import logging logger = logging.getLogger(__name__) -from .utils import ensure_unicode +from common.textutil import ensure_unicode class WeChatMsg(object): diff --git a/wechat/parser.py b/wechat/parser.py index a32bd69..a447d7a 100644 --- a/wechat/parser.py +++ b/wechat/parser.py @@ -1,7 +1,7 @@ #!/usr/bin/env python2 # -*- coding: UTF-8 -*- # File: parser.py -# Date: Wed Jan 07 23:29:50 2015 +0800 +# Date: Thu Jun 18 00:03:53 2015 +0800 # Author: Yuxin Wu import sqlite3 @@ -11,7 +11,7 @@ import logging logger = logging.getLogger(__name__) from .msg import WeChatMsg -from .utils import ensure_unicode +from common.textutil import ensure_unicode """ tables in concern: emojiinfo diff --git a/wechat/render.py b/wechat/render.py index 98e674e..673e609 100644 --- a/wechat/render.py +++ b/wechat/render.py @@ -1,7 +1,7 @@ #!/usr/bin/env python2 # -*- coding: UTF-8 -*- # File: render.py -# Date: Tue Jun 16 23:47:52 2015 +0800 +# Date: Thu Jun 18 00:03:10 2015 +0800 # Author: Yuxin Wu import os @@ -23,7 +23,9 @@ except ImportError: css_compress = lambda x: x from .msg import * -from .utils import ensure_unicode, ProgressReporter, pmap, timing +from common.textutil import ensure_unicode +from common.progress import ProgressReporter +from common.timer import timing from .smiley import SmileyProvider from .msgslice import MessageSlicerByTime, MessageSlicerBySize diff --git a/wechat/res.py b/wechat/res.py index 0afebbb..72a9e4a 100644 --- a/wechat/res.py +++ b/wechat/res.py @@ -1,7 +1,7 @@ #!/usr/bin/env python2 # -*- coding: UTF-8 -*- # File: res.py -# Date: Tue Jun 16 22:30:08 2015 +0800 +# Date: Thu Jun 18 00:02:21 2015 +0800 # Author: Yuxin Wu import glob @@ -18,7 +18,8 @@ from multiprocessing import Pool from .avatar import AvatarReader -from .utils import timing, md5, get_file_b64 +from common.textutil import md5, get_file_b64 +from common.timer import timing from .msg import TYPE_SPEAK from .audio import parse_wechat_audio_file diff --git a/wechat/smiley.py b/wechat/smiley.py index 69f0559..d58d911 100755 --- a/wechat/smiley.py +++ b/wechat/smiley.py @@ -1,7 +1,7 @@ #!/usr/bin/env python2 # -*- coding: UTF-8 -*- # File: smiley.py -# Date: Sun Feb 01 17:47:07 2015 +0800 +# Date: Thu Jun 18 00:02:43 2015 +0800 # Author: Yuxin Wu import os @@ -9,7 +9,7 @@ import re import json import struct -from .utils import get_file_b64 +from common.textutil import get_file_b64 STATIC_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'static') diff --git a/wechat/utils.py b/wechat/utils.py deleted file mode 100644 index e04572c..0000000 --- a/wechat/utils.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding: UTF-8 -*- -# File: utils.py -# Date: Mon May 25 15:21:06 2015 +0800 -# Author: Yuxin Wu - -import sys -import time -import logging -logger = logging.getLogger(__name__) - -import time, functools -from collections import defaultdict - -def ensure_bin_str(s): - if type(s) == str: - return s - if type(s) == unicode: - return s.encode('utf-8') - -def ensure_unicode(s): - if type(s) == str: - return s.decode('utf-8') - if type(s) == unicode: - return s - -class ProgressReporter(object): - """report progress of long-term jobs""" - _start_time = None - _prev_report_time = 0 - _cnt = 0 - _name = None - _total = None - - def __init__(self, name, total=0, fout=sys.stderr): - self._start_time = time.time() - self._name = name - self._total = int(total) - self._fout = fout - - @property - def total_time(self): - return time.time() - self._start_time - - def trigger(self, delta=1, extra_msg='', target_cnt=None): - if target_cnt is None: - self._cnt += int(delta) - else: - self._cnt = int(target_cnt) - now = time.time() - if now - self._prev_report_time < 0.5: - return - self._prev_report_time = now - dt = now - self._start_time - if self._total and self._cnt > 0: - eta_msg = '{}/{} ETA: {:.2f}'.format(self._cnt, self._total, - (self._total-self._cnt)*dt/self._cnt) - else: - eta_msg = '{} done'.format(self._cnt) - self._fout.write(u'{}: avg {:.3f}/sec' - u', passed {:.3f}sec, {} {} \r'.format( - self._name, self._cnt / dt, dt, eta_msg, extra_msg)) - self._fout.flush() - - def finish(self): - """:return: total time""" - self._fout.write('\n') - self._fout.flush() - return self.total_time - - - -import multiprocessing -from multiprocessing import Process, Queue -from collections import deque -import inspect -import dill -class PickleableMethodProxy(object): - def __init__(self, func): - self.data = dill.dumps(func) - return - assert inspect.ismethod(func) - self.im_self = func.im_self - self.method_name = func.__name__ - - def __call__(self, *args, **kwargs): - return dill.loads(self.data)(*args, **kwargs) - return getattr(self.im_self, self.method_name)(*args, **kwargs) - - -def ensure_pickleable_func(func): - if inspect.ismethod(func): - return PickleableMethodProxy(func) - return func - -def pimap(map_func, iterator, nr_proc=None, nr_precompute=None): - '''parallel imap''' - map_func = ensure_pickleable_func(map_func) - if nr_proc is None: - nr_proc = multiprocessing.cpu_count() - if nr_precompute is None: - nr_precompute = nr_proc * 2 - - pool = multiprocessing.Pool(nr_proc) - results = deque() - for i in iterator: - results.append(pool.apply_async(map_func, [i])) - if len(results) == nr_precompute: - yield results.popleft().get() - for r in results: - yield r.get() - pool.close() - pool.join() - pool.terminate() - -def pmap(map_func, iterator, nr_proc=None, nr_precompute=None): - '''parallel map''' - return list(pimap(map_func, iterator, nr_proc, nr_precompute)) - - -class TotalTimer(object): - def __init__(self): - self.times = defaultdict(float) - - def add(self, name, t): - self.times[name] += t - - def reset(self): - self.times = defaultdict(float) - - def __del__(self): - for k, v in self.times.iteritems(): - logger.info("{} took {} seconds in total.".format(k, v)) - -_total_timer = TotalTimer() -class timing(object): - def __init__(self, total=False): - self.total = total - - def __call__(self, func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - start_time = time.time() - ret = func(*args, **kwargs) - duration = time.time() - start_time - - if hasattr(func, '__name__'): - func_name = func.__name__ - else: - func_name = 'function in module {}'.format(func.__module__) - if self.total: - _total_timer.add(func_name, duration) - else: - logger.info('Duration for `{}\': {}'.format( - func_name, duration)) - return ret - return wrapper - - -import hashlib -import base64 -def md5(s): - m = hashlib.md5() - m.update(s) - return m.hexdigest() - -def get_file_b64(fname): - data = open(fname, 'rb').read() - return base64.b64encode(data) - -def safe_filename(fname): - filename = ensure_unicode(fname) - return "".join( - [c for c in filename if c.isalpha() or c.isdigit() or c==' ']).rstrip()