From bcaa7dcae6a37bbdbb698c9d84d166f635c9d3f2 Mon Sep 17 00:00:00 2001 From: ppwwyyxx Date: Tue, 23 Dec 2014 00:03:40 +0800 Subject: [PATCH] multiple html output --- dump_html.py | 13 +++++++++---- lib/msgslice.py | 32 +++++++++++++++++++++++++++++--- lib/render.py | 30 +++++++++++++++++++++--------- lib/utils.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 107 insertions(+), 17 deletions(-) diff --git a/dump_html.py b/dump_html.py index 0d74190..65adc34 100755 --- a/dump_html.py +++ b/dump_html.py @@ -1,7 +1,7 @@ #!/usr/bin/env python2 # -*- coding: UTF-8 -*- # File: dump_html.py -# Date: Sat Dec 20 15:58:35 2014 +0800 +# Date: Tue Dec 23 00:01:20 2014 +0800 # Author: Yuxin Wu import sys @@ -22,7 +22,12 @@ parser = WeChatDBParser(db_file) msgs = parser.msgs_by_talker[name] render = HTMLRender(parser, res) -html = render.render_msgs(msgs).encode('utf-8') +htmls = render.render_msgs(msgs) -with open(output_file, 'w') as f: - print >> f, html +if len(htmls) == 1: + with open(output_file, 'w') as f: + print >> f, htmls[0].encode('utf-8') +else: + for idx, html in enumerate(htmls): + with open(output_file + '.{}'.format(idx), 'w') as f: + print >> f, html.encode('utf-8') diff --git a/lib/msgslice.py b/lib/msgslice.py index c422448..ce74885 100644 --- a/lib/msgslice.py +++ b/lib/msgslice.py @@ -1,11 +1,12 @@ #!/usr/bin/env python2 # -*- coding: UTF-8 -*- # File: msgslice.py -# Date: Mon Dec 22 23:07:02 2014 +0800 +# Date: Mon Dec 22 23:56:55 2014 +0800 # Author: Yuxin Wu -class MessageSlicer(object): - """ Separate messages into slices by time. +class MessageSlicerByTime(object): + """ Separate messages into slices by time, + for time display in html. A new day always begins a new slice. """ def __init__(self, diff_thres=5 * 60): @@ -31,4 +32,29 @@ class MessageSlicer(object): assert len(msgs) == sum([len(k) for k in ret]) return ret +class MessageSlicerBySize(object): + """ Separate messages into slices by max slice size, + to avoid too large html. + """ + def __init__(self, size=1000): + """ a slice will have <= 1.5 * cnt messages""" + self.size = size + assert self.size > 1 + def slice(self, msgs): + ret = [] + now = [] + for m in msgs: + if len(now) >= self.size: + nowtime, lasttime = m.createTime, now[-1].createTime + if nowtime.date() != lasttime.date(): + ret.append(now) + now = [m] + continue + now.append(m) + if len(now) > self.size / 2: + ret.append(now) + else: + ret[-1].extend(now) + assert len(msgs) == sum([len(k) for k in ret]) + return ret diff --git a/lib/render.py b/lib/render.py index 7391700..a295729 100644 --- a/lib/render.py +++ b/lib/render.py @@ -1,7 +1,7 @@ #!/usr/bin/env python2 # -*- coding: UTF-8 -*- # File: render.py -# Date: Mon Dec 22 23:07:05 2014 +0800 +# Date: Tue Dec 23 00:01:11 2014 +0800 # Author: Yuxin Wu import os @@ -20,9 +20,9 @@ except: css_compress = lambda x: x from .msg import * -from .utils import ensure_unicode +from .utils import ensure_unicode, ProgressReporter from .smiley import SmileyProvider -from .msgslice import MessageSlicer +from .msgslice import MessageSlicerByTime, MessageSlicerBySize TEMPLATES_FILES = {TYPE_MSG: "TP_MSG", TYPE_IMG: "TP_IMG", @@ -41,7 +41,6 @@ class HTMLRender(object): if self.res is None: logger.warn("Resource Directory not given. Images / Voice Message won't be displayed.") self.smiley = SmileyProvider() - self.slicer = MessageSlicer() csss = glob.glob(os.path.join(LIB_PATH, 'static/*.css')) css_string = [] @@ -125,13 +124,12 @@ class HTMLRender(object): return template.format(**format_dict) return fallback() - def render_msgs(self, msgs): - """ render msgs of one friend""" + def _render_partial_msgs(self, msgs): + """ return single html""" talker_name = msgs[0].talker - logger.info(u"Rendering {} messages of {}({})".format( - len(msgs), self.parser.contacts[talker_name], talker_name)) avatars = self.get_avatar_pair(talker_name) - slices = self.slicer.slice(msgs) + slicer = MessageSlicerByTime() + slices = slicer.slice(msgs) blocks = [] for idx, slice in enumerate(slices): @@ -143,6 +141,7 @@ class HTMLRender(object): timestr = nowtime.strftime("%H:%M:%S") blocks.append(self.time_html.format(time=timestr)) blocks.extend([self.render_msg(m) for m in slice]) + self.prgs.trigger(len(slice)) return self.html.format(extra_css=self.css_string, extra_js=self.js_string, @@ -150,6 +149,19 @@ class HTMLRender(object): messages=u''.join(blocks), avatars=avatars) + + def render_msgs(self, msgs): + """ render msgs of one friend, return a list of html""" + talker_name = msgs[0].talker + logger.info(u"Rendering {} messages of {}({})".format( + len(msgs), self.parser.contacts[talker_name], talker_name)) + + self.prgs = ProgressReporter("Render", total=len(msgs)) + slice_by_size = MessageSlicerBySize().slice(msgs) + ret = [self._render_partial_msgs(s) for s in slice_by_size] + self.prgs.finish() + return ret + if __name__ == '__main__': r = HTMLRender() with open('/tmp/a.html', 'w') as f: diff --git a/lib/utils.py b/lib/utils.py index d38a5ed..58c26a0 100644 --- a/lib/utils.py +++ b/lib/utils.py @@ -1,9 +1,12 @@ #!/usr/bin/env python2 # -*- coding: UTF-8 -*- # File: utils.py -# Date: Fri Nov 21 13:42:56 2014 +0800 +# Date: Mon Dec 22 23:24:02 2014 +0800 # Author: Yuxin Wu +import sys +import time + def ensure_bin_str(s): if type(s) == str: return s @@ -15,3 +18,47 @@ def ensure_unicode(s): return s.decode('utf-8') if type(s) == unicode: return s + +class ProgressReporter(object): + """report progress of long-term jobs""" + _start_time = None + _prev_report_time = 0 + _cnt = 0 + _name = None + _total = None + + def __init__(self, name, total=0, fout=sys.stderr): + self._start_time = time.time() + self._name = name + self._total = int(total) + self._fout = fout + + @property + def total_time(self): + return time.time() - self._start_time + + def trigger(self, delta=1, extra_msg='', target_cnt=None): + if target_cnt is None: + self._cnt += int(delta) + else: + self._cnt = int(target_cnt) + now = time.time() + if now - self._prev_report_time < 0.5: + return + self._prev_report_time = now + dt = now - self._start_time + if self._total and self._cnt > 0: + eta_msg = '{}/{} ETA: {:.2f}'.format(self._cnt, self._total, + (self._total-self._cnt)*dt/self._cnt) + else: + eta_msg = '{} done'.format(self._cnt) + self._fout.write(u'{}: avg {:.3f}/sec' + u', passed {:.3f}sec, {} {} \r'.format( + self._name, self._cnt / dt, dt, eta_msg, extra_msg)) + self._fout.flush() + + def finish(self): + """:return: total time""" + self._fout.write('\n') + self._fout.flush() + return self.total_time