move utils to common

common/progress.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python2
+# -*- coding: UTF-8 -*-
+# File: progress.py
+# Date: Wed Jun 17 23:59:52 2015 +0800
+# Author: Yuxin Wu <[email protected]>
+
+import time
+import sys
+
+class ProgressReporter(object):
+    """report progress of long-term jobs"""
+    _start_time = None
+    _prev_report_time = 0
+    _cnt = 0
+    _name = None
+    _total = None
+
+    def __init__(self, name, total=0, fout=sys.stderr):
+        self._start_time = time.time()
+        self._name = name
+        self._total = int(total)
+        self._fout = fout
+
+    @property
+    def total_time(self):
+        return time.time() - self._start_time
+
+    def trigger(self, delta=1, extra_msg='', target_cnt=None):
+        if target_cnt is None:
+            self._cnt += int(delta)
+        else:
+            self._cnt = int(target_cnt)
+        now = time.time()
+        if now - self._prev_report_time < 0.5:
+            return
+        self._prev_report_time = now
+        dt = now - self._start_time
+        if self._total and self._cnt > 0:
+            eta_msg = '{}/{} ETA: {:.2f}'.format(self._cnt, self._total,
+                    (self._total-self._cnt)*dt/self._cnt)
+        else:
+            eta_msg = '{} done'.format(self._cnt)
+        self._fout.write(u'{}: avg {:.3f}/sec'
+                         u', passed {:.3f}sec, {}  {} \r'.format(
+            self._name, self._cnt / dt, dt, eta_msg, extra_msg))
+        self._fout.flush()
+
+    def finish(self):
+        """:return: total time"""
+        self._fout.write('\n')
+        self._fout.flush()
+        return self.total_time
+

common/textutil.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python2
+# -*- coding: UTF-8 -*-
+# File: utils.py
+# Date: Wed Jun 17 23:59:25 2015 +0800
+# Author: Yuxin Wu <[email protected]>
+
+import hashlib
+import base64
+
+def ensure_bin_str(s):
+    if type(s) == str:
+        return s
+    if type(s) == unicode:
+        return s.encode('utf-8')
+
+def ensure_unicode(s):
+    if type(s) == str:
+        return s.decode('utf-8')
+    if type(s) == unicode:
+        return s
+
+
+def md5(s):
+    m = hashlib.md5()
+    m.update(s)
+    return m.hexdigest()
+
+def get_file_b64(fname):
+    data = open(fname, 'rb').read()
+    return base64.b64encode(data)
+
+def safe_filename(fname):
+    filename = ensure_unicode(fname)
+    return "".join(
+        [c for c in filename if c.isalpha() or c.isdigit() or c==' ']).rstrip()

common/timer.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python2
+# -*- coding: UTF-8 -*-
+# File: timer.py
+# Date: Wed Jun 17 23:25:54 2015 +0800
+# Author: Yuxin Wu <[email protected]>
+
+import time, functools
+from collections import defaultdict
+import logging
+logger = logging.getLogger(__name__)
+
+class TotalTimer(object):
+    def __init__(self):
+        self.times = defaultdict(float)
+
+    def add(self, name, t):
+        self.times[name] += t
+
+    def reset(self):
+        self.times = defaultdict(float)
+
+    def __del__(self):
+        for k, v in self.times.iteritems():
+            logger.info("{} took {} seconds in total.".format(k, v))
+
+_total_timer = TotalTimer()
+class timing(object):
+    def __init__(self, total=False):
+        self.total = total
+
+    def __call__(self, func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            start_time = time.time()
+            ret = func(*args, **kwargs)
+            duration = time.time() - start_time
+
+            if hasattr(func, '__name__'):
+                func_name = func.__name__
+            else:
+                func_name = 'function in module {}'.format(func.__module__)
+            if self.total:
+                _total_timer.add(func_name, duration)
+            else:
+                logger.info('Duration for `{}\': {}'.format(
+                    func_name, duration))
+            return ret
+        return wrapper
+

wechat/audio.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python2
 # -*- coding: UTF-8 -*-
 # File: audio.py
-# Date: Tue Jun 16 23:16:11 2015 +0800
+# Date: Thu Jun 18 00:01:49 2015 +0800
 # Author: Yuxin Wu <[email protected]>
 
 import os
@@ -11,7 +11,7 @@ logger = logging.getLogger(__name__)
 
 import pysox
 
-from .utils import get_file_b64
+from common.textutil import get_file_b64
 
 SILK_DECODER = os.path.join(os.path.dirname(__file__),
                             '../third-party/silk/decoder')

wechat/avatar.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python2
 # -*- coding: UTF-8 -*-
 # File: avatar.py
-# Date: Fri Jan 09 22:41:55 2015 +0800
+# Date: Thu Jun 18 00:02:07 2015 +0800
 # Author: Yuxin Wu <[email protected]>
 
 import os
@@ -9,7 +9,7 @@ import numpy as np
 import logging
 logger = logging.getLogger(__name__)
 
-from .utils import ensure_bin_str, md5
+from common.textutil import ensure_bin_str, md5
 
 class AvatarReader(object):
     def __init__(self, avt_dir):

wechat/libchathelper.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python2
 # -*- coding: UTF-8 -*-
 # File: libchathelper.py
-# Date: Fri Mar 27 22:25:14 2015 +0800
+# Date: Thu Jun 18 00:02:35 2015 +0800
 # Author: Yuxin Wu <[email protected]>
 
 import base64
@@ -12,7 +12,8 @@ logger = logging.getLogger(__name__)
 
 from libchat.libchat import SqliteLibChat, ChatMsg
 from .msg import *
-from .utils import timing, ProgressReporter
+from common.timer import timing
+from common.progress import ProgressReporter
 
 class LibChatHelper(object):
     """ Build LibChat messages from WeChat Msg"""

wechat/msg.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python2
 # -*- coding: UTF-8 -*-
 # File: msg.py
-# Date: Wed May 06 10:47:57 2015 +0800
+# Date: Thu Jun 18 00:01:00 2015 +0800
 # Author: Yuxin Wu <[email protected]>
 TYPE_MSG = 1
 TYPE_IMG = 3
@@ -26,7 +26,7 @@ from pyquery import PyQuery
 import logging
 logger = logging.getLogger(__name__)
 
-from .utils import ensure_unicode
+from common.textutil import ensure_unicode
 
 
 class WeChatMsg(object):

wechat/parser.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python2
 # -*- coding: UTF-8 -*-
 # File: parser.py
-# Date: Wed Jan 07 23:29:50 2015 +0800
+# Date: Thu Jun 18 00:03:53 2015 +0800
 # Author: Yuxin Wu <[email protected]>
 
 import sqlite3
@@ -11,7 +11,7 @@ import logging
 logger = logging.getLogger(__name__)
 
 from .msg import WeChatMsg
-from .utils import ensure_unicode
+from common.textutil import ensure_unicode
 
 """ tables in concern:
 emojiinfo

wechat/render.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python2
 # -*- coding: UTF-8 -*-
 # File: render.py
-# Date: Tue Jun 16 23:47:52 2015 +0800
+# Date: Thu Jun 18 00:03:10 2015 +0800
 # Author: Yuxin Wu <[email protected]>
 
 import os
@@ -23,7 +23,9 @@ except ImportError:
     css_compress = lambda x: x
 
 from .msg import *
-from .utils import ensure_unicode, ProgressReporter, pmap, timing
+from common.textutil import ensure_unicode
+from common.progress import ProgressReporter
+from common.timer import timing
 from .smiley import SmileyProvider
 from .msgslice import MessageSlicerByTime, MessageSlicerBySize
 

wechat/res.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python2
 # -*- coding: UTF-8 -*-
 # File: res.py
-# Date: Tue Jun 16 22:30:08 2015 +0800
+# Date: Thu Jun 18 00:02:21 2015 +0800
 # Author: Yuxin Wu <[email protected]>
 
 import glob
@@ -18,7 +18,8 @@ from multiprocessing import Pool
 
 
 from .avatar import AvatarReader
-from .utils import timing, md5, get_file_b64
+from common.textutil import md5, get_file_b64
+from common.timer import timing
 from .msg import TYPE_SPEAK
 from .audio import parse_wechat_audio_file
 

wechat/smiley.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python2
 # -*- coding: UTF-8 -*-
 # File: smiley.py
-# Date: Sun Feb 01 17:47:07 2015 +0800
+# Date: Thu Jun 18 00:02:43 2015 +0800
 # Author: Yuxin Wu <[email protected]>
 
 import os
@@ -9,7 +9,7 @@ import re
 import json
 import struct
 
-from .utils import get_file_b64
+from common.textutil import get_file_b64
 
 STATIC_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'static')
 

wechat/utils.py

@@ -1,174 +0,0 @@
-#!/usr/bin/env python2
-# -*- coding: UTF-8 -*-
-# File: utils.py
-# Date: Mon May 25 15:21:06 2015 +0800
-# Author: Yuxin Wu <[email protected]>
-
-import sys
-import time
-import logging
-logger = logging.getLogger(__name__)
-
-import time, functools
-from collections import defaultdict
-
-def ensure_bin_str(s):
-    if type(s) == str:
-        return s
-    if type(s) == unicode:
-        return s.encode('utf-8')
-
-def ensure_unicode(s):
-    if type(s) == str:
-        return s.decode('utf-8')
-    if type(s) == unicode:
-        return s
-
-class ProgressReporter(object):
-    """report progress of long-term jobs"""
-    _start_time = None
-    _prev_report_time = 0
-    _cnt = 0
-    _name = None
-    _total = None
-
-    def __init__(self, name, total=0, fout=sys.stderr):
-        self._start_time = time.time()
-        self._name = name
-        self._total = int(total)
-        self._fout = fout
-
-    @property
-    def total_time(self):
-        return time.time() - self._start_time
-
-    def trigger(self, delta=1, extra_msg='', target_cnt=None):
-        if target_cnt is None:
-            self._cnt += int(delta)
-        else:
-            self._cnt = int(target_cnt)
-        now = time.time()
-        if now - self._prev_report_time < 0.5:
-            return
-        self._prev_report_time = now
-        dt = now - self._start_time
-        if self._total and self._cnt > 0:
-            eta_msg = '{}/{} ETA: {:.2f}'.format(self._cnt, self._total,
-                    (self._total-self._cnt)*dt/self._cnt)
-        else:
-            eta_msg = '{} done'.format(self._cnt)
-        self._fout.write(u'{}: avg {:.3f}/sec'
-                         u', passed {:.3f}sec, {}  {} \r'.format(
-            self._name, self._cnt / dt, dt, eta_msg, extra_msg))
-        self._fout.flush()
-
-    def finish(self):
-        """:return: total time"""
-        self._fout.write('\n')
-        self._fout.flush()
-        return self.total_time
-
-
-
-import multiprocessing
-from multiprocessing import Process, Queue
-from collections import deque
-import inspect
-import dill
-class PickleableMethodProxy(object):
-    def __init__(self, func):
-        self.data = dill.dumps(func)
-        return
-        assert inspect.ismethod(func)
-        self.im_self = func.im_self
-        self.method_name = func.__name__
-
-    def __call__(self, *args, **kwargs):
-        return dill.loads(self.data)(*args, **kwargs)
-        return getattr(self.im_self, self.method_name)(*args, **kwargs)
-
-
-def ensure_pickleable_func(func):
-    if inspect.ismethod(func):
-        return PickleableMethodProxy(func)
-    return func
-
-def pimap(map_func, iterator, nr_proc=None, nr_precompute=None):
-    '''parallel imap'''
-    map_func = ensure_pickleable_func(map_func)
-    if nr_proc is None:
-        nr_proc = multiprocessing.cpu_count()
-    if nr_precompute is None:
-        nr_precompute = nr_proc * 2
-
-    pool = multiprocessing.Pool(nr_proc)
-    results = deque()
-    for i in iterator:
-        results.append(pool.apply_async(map_func, [i]))
-        if len(results) == nr_precompute:
-            yield results.popleft().get()
-    for r in results:
-        yield r.get()
-    pool.close()
-    pool.join()
-    pool.terminate()
-
-def pmap(map_func, iterator, nr_proc=None, nr_precompute=None):
-    '''parallel map'''
-    return list(pimap(map_func, iterator, nr_proc, nr_precompute))
-
-
-class TotalTimer(object):
-    def __init__(self):
-        self.times = defaultdict(float)
-
-    def add(self, name, t):
-        self.times[name] += t
-
-    def reset(self):
-        self.times = defaultdict(float)
-
-    def __del__(self):
-        for k, v in self.times.iteritems():
-            logger.info("{} took {} seconds in total.".format(k, v))
-
-_total_timer = TotalTimer()
-class timing(object):
-    def __init__(self, total=False):
-        self.total = total
-
-    def __call__(self, func):
-        @functools.wraps(func)
-        def wrapper(*args, **kwargs):
-            start_time = time.time()
-            ret = func(*args, **kwargs)
-            duration = time.time() - start_time
-
-            if hasattr(func, '__name__'):
-                func_name = func.__name__
-            else:
-                func_name = 'function in module {}'.format(func.__module__)
-            if self.total:
-                _total_timer.add(func_name, duration)
-            else:
-                logger.info('Duration for `{}\': {}'.format(
-                    func_name, duration))
-            return ret
-        return wrapper
-
-
-import hashlib
-import base64
-def md5(s):
-    m = hashlib.md5()
-    m.update(s)
-    return m.hexdigest()
-
-def get_file_b64(fname):
-    data = open(fname, 'rb').read()
-    return base64.b64encode(data)
-
-def safe_filename(fname):
-    filename = ensure_unicode(fname)
-    return "".join(
-        [c for c in filename if c.isalpha() or c.isdigit() or c==' ']).rstrip()