ZSCGR/wechat-dump
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

move utils to common

Browse Source
This commit is contained in:
ppwwyyxx
2015-06-18 00:07:15 +08:00
parent a55f8ec341
commit 43cda58e71
14 changed files with 157 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
-1
View File
@@ -13,7 +13,6 @@ __NEWS__: WeChat 6.0+ use silk to encode audio. The code is updated.
+ python-PIL
+ [PyQuery](https://pypi.python.org/pypi/pyquery/1.2.1)
+ [pysox](https://pypi.python.org/pypi/pysox/0.3.6.alpha)
+ [dill](https://pypi.python.org/pypi/dill)
+ numpy
+ python-csscompressor(optional)
+ adb and rooted android phone connected to PC
common/__init__.py
View File
common/progress.py
+53
View File
@@ -0,0 +1,53 @@
#!/usr/bin/env python2
# -*- coding: UTF-8 -*-
# File: progress.py
# Date: Wed Jun 17 23:59:52 2015 +0800
# Author: Yuxin Wu <[email protected]>
import time
import sys
class ProgressReporter(object):
"""report progress of long-term jobs"""
_start_time = None
_prev_report_time = 0
_cnt = 0
_name = None
_total = None
def __init__(self, name, total=0, fout=sys.stderr):
self._start_time = time.time()
self._name = name
self._total = int(total)
self._fout = fout
@property
def total_time(self):
return time.time() - self._start_time
def trigger(self, delta=1, extra_msg='', target_cnt=None):
if target_cnt is None:
self._cnt += int(delta)
else:
self._cnt = int(target_cnt)
now = time.time()
if now - self._prev_report_time < 0.5:
return
self._prev_report_time = now
dt = now - self._start_time
if self._total and self._cnt > 0:
eta_msg = '{}/{} ETA: {:.2f}'.format(self._cnt, self._total,
(self._total-self._cnt)*dt/self._cnt)
else:
eta_msg = '{} done'.format(self._cnt)
self._fout.write(u'{}: avg {:.3f}/sec'
u', passed {:.3f}sec, {} {} \r'.format(
self._name, self._cnt / dt, dt, eta_msg, extra_msg))
self._fout.flush()
def finish(self):
""":return: total time"""
self._fout.write('\n')
self._fout.flush()
return self.total_time
common/textutil.py
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env python2
# -*- coding: UTF-8 -*-
# File: utils.py
# Date: Wed Jun 17 23:59:25 2015 +0800
# Author: Yuxin Wu <[email protected]>
import hashlib
import base64
def ensure_bin_str(s):
if type(s) == str:
return s
if type(s) == unicode:
return s.encode('utf-8')
def ensure_unicode(s):
if type(s) == str:
return s.decode('utf-8')
if type(s) == unicode:
return s
def md5(s):
m = hashlib.md5()
m.update(s)
return m.hexdigest()
def get_file_b64(fname):
data = open(fname, 'rb').read()
return base64.b64encode(data)
def safe_filename(fname):
filename = ensure_unicode(fname)
return "".join(
[c for c in filename if c.isalpha() or c.isdigit() or c==' ']).rstrip()
common/timer.py
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env python2
# -*- coding: UTF-8 -*-
# File: timer.py
# Date: Wed Jun 17 23:25:54 2015 +0800
# Author: Yuxin Wu <[email protected]>
import time, functools
from collections import defaultdict
import logging
logger = logging.getLogger(__name__)
class TotalTimer(object):
def __init__(self):
self.times = defaultdict(float)
def add(self, name, t):
self.times[name] += t
def reset(self):
self.times = defaultdict(float)
def __del__(self):
for k, v in self.times.iteritems():
logger.info("{} took {} seconds in total.".format(k, v))
_total_timer = TotalTimer()
class timing(object):
def __init__(self, total=False):
self.total = total
def __call__(self, func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
start_time = time.time()
ret = func(*args, **kwargs)
duration = time.time() - start_time
if hasattr(func, '__name__'):
func_name = func.__name__
else:
func_name = 'function in module {}'.format(func.__module__)
if self.total:
_total_timer.add(func_name, duration)
else:
logger.info('Duration for `{}\': {}'.format(
func_name, duration))
return ret
return wrapper
wechat/audio.py
+2 -2
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env python2
# -*- coding: UTF-8 -*-
# File: audio.py
# Date: Tue Jun 16 23:16:11 2015 +0800
# Date: Thu Jun 18 00:01:49 2015 +0800
# Author: Yuxin Wu <[email protected]>
import os
@@ -11,7 +11,7 @@ logger = logging.getLogger(__name__)
import pysox
from .utils import get_file_b64
from common.textutil import get_file_b64
SILK_DECODER = os.path.join(os.path.dirname(__file__),
'../third-party/silk/decoder')
wechat/avatar.py
+2 -2
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env python2
# -*- coding: UTF-8 -*-
# File: avatar.py
# Date: Fri Jan 09 22:41:55 2015 +0800
# Date: Thu Jun 18 00:02:07 2015 +0800
# Author: Yuxin Wu <[email protected]>
import os
@@ -9,7 +9,7 @@ import numpy as np
import logging
logger = logging.getLogger(__name__)
from .utils import ensure_bin_str, md5
from common.textutil import ensure_bin_str, md5
class AvatarReader(object):
def __init__(self, avt_dir):
wechat/libchathelper.py
+3 -2
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env python2
# -*- coding: UTF-8 -*-
# File: libchathelper.py
# Date: Fri Mar 27 22:25:14 2015 +0800
# Date: Thu Jun 18 00:02:35 2015 +0800
# Author: Yuxin Wu <[email protected]>
import base64
@@ -12,7 +12,8 @@ logger = logging.getLogger(__name__)
from libchat.libchat import SqliteLibChat, ChatMsg
from .msg import *
from .utils import timing, ProgressReporter
from common.timer import timing
from common.progress import ProgressReporter
class LibChatHelper(object):
""" Build LibChat messages from WeChat Msg"""
wechat/msg.py
+2 -2
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env python2
# -*- coding: UTF-8 -*-
# File: msg.py
# Date: Wed May 06 10:47:57 2015 +0800
# Date: Thu Jun 18 00:01:00 2015 +0800
# Author: Yuxin Wu <[email protected]>
TYPE_MSG = 1
TYPE_IMG = 3
@@ -26,7 +26,7 @@ from pyquery import PyQuery
import logging
logger = logging.getLogger(__name__)
from .utils import ensure_unicode
from common.textutil import ensure_unicode
class WeChatMsg(object):
wechat/parser.py
+2 -2
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env python2
# -*- coding: UTF-8 -*-
# File: parser.py
# Date: Wed Jan 07 23:29:50 2015 +0800
# Date: Thu Jun 18 00:03:53 2015 +0800
# Author: Yuxin Wu <[email protected]>
import sqlite3
@@ -11,7 +11,7 @@ import logging
logger = logging.getLogger(__name__)
from .msg import WeChatMsg
from .utils import ensure_unicode
from common.textutil import ensure_unicode
""" tables in concern:
emojiinfo
wechat/render.py
+4 -2
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env python2
# -*- coding: UTF-8 -*-
# File: render.py
# Date: Tue Jun 16 23:47:52 2015 +0800
# Date: Thu Jun 18 00:03:10 2015 +0800
# Author: Yuxin Wu <[email protected]>
import os
@@ -23,7 +23,9 @@ except ImportError:
css_compress = lambda x: x
from .msg import *
from .utils import ensure_unicode, ProgressReporter, pmap, timing
from common.textutil import ensure_unicode
from common.progress import ProgressReporter
from common.timer import timing
from .smiley import SmileyProvider
from .msgslice import MessageSlicerByTime, MessageSlicerBySize
wechat/res.py
+3 -2
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env python2
# -*- coding: UTF-8 -*-
# File: res.py
# Date: Tue Jun 16 22:30:08 2015 +0800
# Date: Thu Jun 18 00:02:21 2015 +0800
# Author: Yuxin Wu <[email protected]>
import glob
@@ -18,7 +18,8 @@ from multiprocessing import Pool
from .avatar import AvatarReader
from .utils import timing, md5, get_file_b64
from common.textutil import md5, get_file_b64
from common.timer import timing
from .msg import TYPE_SPEAK
from .audio import parse_wechat_audio_file
wechat/smiley.py
+2 -2
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env python2
# -*- coding: UTF-8 -*-
# File: smiley.py
# Date: Sun Feb 01 17:47:07 2015 +0800
# Date: Thu Jun 18 00:02:43 2015 +0800
# Author: Yuxin Wu <[email protected]>
import os
@@ -9,7 +9,7 @@ import re
import json
import struct
from .utils import get_file_b64
from common.textutil import get_file_b64
STATIC_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'static')
wechat/utils.py
-174
View File
@@ -1,174 +0,0 @@
#!/usr/bin/env python2
# -*- coding: UTF-8 -*-
# File: utils.py
# Date: Mon May 25 15:21:06 2015 +0800
# Author: Yuxin Wu <[email protected]>
import sys
import time
import logging
logger = logging.getLogger(__name__)
import time, functools
from collections import defaultdict
def ensure_bin_str(s):
if type(s) == str:
return s
if type(s) == unicode:
return s.encode('utf-8')
def ensure_unicode(s):
if type(s) == str:
return s.decode('utf-8')
if type(s) == unicode:
return s
class ProgressReporter(object):
"""report progress of long-term jobs"""
_start_time = None
_prev_report_time = 0
_cnt = 0
_name = None
_total = None
def __init__(self, name, total=0, fout=sys.stderr):
self._start_time = time.time()
self._name = name
self._total = int(total)
self._fout = fout
@property
def total_time(self):
return time.time() - self._start_time
def trigger(self, delta=1, extra_msg='', target_cnt=None):
if target_cnt is None:
self._cnt += int(delta)
else:
self._cnt = int(target_cnt)
now = time.time()
if now - self._prev_report_time < 0.5:
return
self._prev_report_time = now
dt = now - self._start_time
if self._total and self._cnt > 0:
eta_msg = '{}/{} ETA: {:.2f}'.format(self._cnt, self._total,
(self._total-self._cnt)*dt/self._cnt)
else:
eta_msg = '{} done'.format(self._cnt)
self._fout.write(u'{}: avg {:.3f}/sec'
u', passed {:.3f}sec, {} {} \r'.format(
self._name, self._cnt / dt, dt, eta_msg, extra_msg))
self._fout.flush()
def finish(self):
""":return: total time"""
self._fout.write('\n')
self._fout.flush()
return self.total_time
import multiprocessing
from multiprocessing import Process, Queue
from collections import deque
import inspect
import dill
class PickleableMethodProxy(object):
def __init__(self, func):
self.data = dill.dumps(func)
return
assert inspect.ismethod(func)
self.im_self = func.im_self
self.method_name = func.__name__
def __call__(self, *args, **kwargs):
return dill.loads(self.data)(*args, **kwargs)
return getattr(self.im_self, self.method_name)(*args, **kwargs)
def ensure_pickleable_func(func):
if inspect.ismethod(func):
return PickleableMethodProxy(func)
return func
def pimap(map_func, iterator, nr_proc=None, nr_precompute=None):
'''parallel imap'''
map_func = ensure_pickleable_func(map_func)
if nr_proc is None:
nr_proc = multiprocessing.cpu_count()
if nr_precompute is None:
nr_precompute = nr_proc * 2
pool = multiprocessing.Pool(nr_proc)
results = deque()
for i in iterator:
results.append(pool.apply_async(map_func, [i]))
if len(results) == nr_precompute:
yield results.popleft().get()
for r in results:
yield r.get()
pool.close()
pool.join()
pool.terminate()
def pmap(map_func, iterator, nr_proc=None, nr_precompute=None):
'''parallel map'''
return list(pimap(map_func, iterator, nr_proc, nr_precompute))
class TotalTimer(object):
def __init__(self):
self.times = defaultdict(float)
def add(self, name, t):
self.times[name] += t
def reset(self):
self.times = defaultdict(float)
def __del__(self):
for k, v in self.times.iteritems():
logger.info("{} took {} seconds in total.".format(k, v))
_total_timer = TotalTimer()
class timing(object):
def __init__(self, total=False):
self.total = total
def __call__(self, func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
start_time = time.time()
ret = func(*args, **kwargs)
duration = time.time() - start_time
if hasattr(func, '__name__'):
func_name = func.__name__
else:
func_name = 'function in module {}'.format(func.__module__)
if self.total:
_total_timer.add(func_name, duration)
else:
logger.info('Duration for `{}\': {}'.format(
func_name, duration))
return ret
return wrapper
import hashlib
import base64
def md5(s):
m = hashlib.md5()
m.update(s)
return m.hexdigest()
def get_file_b64(fname):
data = open(fname, 'rb').read()
return base64.b64encode(data)
def safe_filename(fname):
filename = ensure_unicode(fname)
return "".join(
[c for c in filename if c.isalpha() or c.isdigit() or c==' ']).rstrip()