multiple html output

dump_html.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python2
 # -*- coding: UTF-8 -*-
 # File: dump_html.py
-# Date: Sat Dec 20 15:58:35 2014 +0800
+# Date: Tue Dec 23 00:01:20 2014 +0800
 # Author: Yuxin Wu <[email protected]>
 
 import sys
@@ -22,7 +22,12 @@ parser = WeChatDBParser(db_file)
 msgs = parser.msgs_by_talker[name]
 
 render = HTMLRender(parser, res)
-html = render.render_msgs(msgs).encode('utf-8')
+htmls = render.render_msgs(msgs)
 
-with open(output_file, 'w') as f:
-    print >> f, html
+if len(htmls) == 1:
+    with open(output_file, 'w') as f:
+        print >> f, htmls[0].encode('utf-8')
+else:
+    for idx, html in enumerate(htmls):
+        with open(output_file + '.{}'.format(idx), 'w') as f:
+            print >> f, html.encode('utf-8')

lib/msgslice.py

@@ -1,11 +1,12 @@
 #!/usr/bin/env python2
 # -*- coding: UTF-8 -*-
 # File: msgslice.py
-# Date: Mon Dec 22 23:07:02 2014 +0800
+# Date: Mon Dec 22 23:56:55 2014 +0800
 # Author: Yuxin Wu <[email protected]>
 
-class MessageSlicer(object):
-    """ Separate messages into slices by time.
+class MessageSlicerByTime(object):
+    """ Separate messages into slices by time,
+        for time display in html.
         A new day always begins a new slice.
     """
     def __init__(self, diff_thres=5 * 60):
@@ -31,4 +32,29 @@ class MessageSlicer(object):
         assert len(msgs) == sum([len(k) for k in ret])
         return ret
 
+class MessageSlicerBySize(object):
+    """ Separate messages into slices by max slice size,
+        to avoid too large html.
+    """
+    def __init__(self, size=1000):
+        """ a slice will have <= 1.5 * cnt messages"""
+        self.size = size
+        assert self.size > 1
 
+    def slice(self, msgs):
+        ret = []
+        now = []
+        for m in msgs:
+            if len(now) >= self.size:
+                nowtime, lasttime = m.createTime, now[-1].createTime
+                if nowtime.date() != lasttime.date():
+                    ret.append(now)
+                    now = [m]
+                    continue
+            now.append(m)
+        if len(now) > self.size / 2:
+            ret.append(now)
+        else:
+            ret[-1].extend(now)
+        assert len(msgs) == sum([len(k) for k in ret])
+        return ret

lib/render.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python2
 # -*- coding: UTF-8 -*-
 # File: render.py
-# Date: Mon Dec 22 23:07:05 2014 +0800
+# Date: Tue Dec 23 00:01:11 2014 +0800
 # Author: Yuxin Wu <[email protected]>
 
 import os
@@ -20,9 +20,9 @@ except:
     css_compress = lambda x: x
 
 from .msg import *
-from .utils import ensure_unicode
+from .utils import ensure_unicode, ProgressReporter
 from .smiley import SmileyProvider
-from .msgslice import MessageSlicer
+from .msgslice import MessageSlicerByTime, MessageSlicerBySize
 
 TEMPLATES_FILES = {TYPE_MSG: "TP_MSG",
                    TYPE_IMG: "TP_IMG",
@@ -41,7 +41,6 @@ class HTMLRender(object):
         if self.res is None:
             logger.warn("Resource Directory not given. Images / Voice Message won't be displayed.")
         self.smiley = SmileyProvider()
-        self.slicer = MessageSlicer()
 
         csss = glob.glob(os.path.join(LIB_PATH, 'static/*.css'))
         css_string = []
@@ -125,13 +124,12 @@ class HTMLRender(object):
                 return template.format(**format_dict)
         return fallback()
 
-    def render_msgs(self, msgs):
-        """ render msgs of one friend"""
+    def _render_partial_msgs(self, msgs):
+        """ return single html"""
         talker_name = msgs[0].talker
-        logger.info(u"Rendering {} messages of {}({})".format(
-            len(msgs), self.parser.contacts[talker_name], talker_name))
         avatars = self.get_avatar_pair(talker_name)
-        slices = self.slicer.slice(msgs)
+        slicer = MessageSlicerByTime()
+        slices = slicer.slice(msgs)
 
         blocks = []
         for idx, slice in enumerate(slices):
@@ -143,6 +141,7 @@ class HTMLRender(object):
                 timestr = nowtime.strftime("%H:%M:%S")
             blocks.append(self.time_html.format(time=timestr))
             blocks.extend([self.render_msg(m) for m in slice])
+            self.prgs.trigger(len(slice))
 
         return self.html.format(extra_css=self.css_string,
                                 extra_js=self.js_string,
@@ -150,6 +149,19 @@ class HTMLRender(object):
                                 messages=u''.join(blocks),
                                 avatars=avatars)
 
+
+    def render_msgs(self, msgs):
+        """ render msgs of one friend, return a list of html"""
+        talker_name = msgs[0].talker
+        logger.info(u"Rendering {} messages of {}({})".format(
+            len(msgs), self.parser.contacts[talker_name], talker_name))
+
+        self.prgs = ProgressReporter("Render", total=len(msgs))
+        slice_by_size = MessageSlicerBySize().slice(msgs)
+        ret = [self._render_partial_msgs(s) for s in slice_by_size]
+        self.prgs.finish()
+        return ret
+
 if __name__ == '__main__':
     r = HTMLRender()
     with open('/tmp/a.html', 'w') as f:

lib/utils.py

@@ -1,9 +1,12 @@
 #!/usr/bin/env python2
 # -*- coding: UTF-8 -*-
 # File: utils.py
-# Date: Fri Nov 21 13:42:56 2014 +0800
+# Date: Mon Dec 22 23:24:02 2014 +0800
 # Author: Yuxin Wu <[email protected]>
 
+import sys
+import time
+
 def ensure_bin_str(s):
     if type(s) == str:
         return s
@@ -15,3 +18,47 @@ def ensure_unicode(s):
         return s.decode('utf-8')
     if type(s) == unicode:
         return s
+
+class ProgressReporter(object):
+    """report progress of long-term jobs"""
+    _start_time = None
+    _prev_report_time = 0
+    _cnt = 0
+    _name = None
+    _total = None
+
+    def __init__(self, name, total=0, fout=sys.stderr):
+        self._start_time = time.time()
+        self._name = name
+        self._total = int(total)
+        self._fout = fout
+
+    @property
+    def total_time(self):
+        return time.time() - self._start_time
+
+    def trigger(self, delta=1, extra_msg='', target_cnt=None):
+        if target_cnt is None:
+            self._cnt += int(delta)
+        else:
+            self._cnt = int(target_cnt)
+        now = time.time()
+        if now - self._prev_report_time < 0.5:
+            return
+        self._prev_report_time = now
+        dt = now - self._start_time
+        if self._total and self._cnt > 0:
+            eta_msg = '{}/{} ETA: {:.2f}'.format(self._cnt, self._total,
+                    (self._total-self._cnt)*dt/self._cnt)
+        else:
+            eta_msg = '{} done'.format(self._cnt)
+        self._fout.write(u'{}: avg {:.3f}/sec'
+                         u', passed {:.3f}sec, {}  {} \r'.format(
+            self._name, self._cnt / dt, dt, eta_msg, extra_msg))
+        self._fout.flush()
+
+    def finish(self):
+        """:return: total time"""
+        self._fout.write('\n')
+        self._fout.flush()
+        return self.total_time