support latest smileys

wechat/render.py

@@ -171,7 +171,7 @@ class HTMLRender(object):
 
     def _render_partial_msgs(self, msgs):
         """ return single html"""
-        self.smiley.used_smiley_id.clear()
+        self.smiley.reset()
         slicer = MessageSlicerByTime()
         slices = slicer.slice(msgs)
 

wechat/smiley.py

@@ -2,6 +2,7 @@
 # -*- coding: UTF-8 -*-
 
 import os
+import functools
 import re
 import json
 import struct
@@ -9,10 +10,7 @@ import struct
 from common.textutil import get_file_b64
 
 STATIC_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'static')
-
-UNICODE_SMILEY_FILE = os.path.join(STATIC_PATH, 'unicode-smiley.json')
 TENCENT_SMILEY_FILE = os.path.join(STATIC_PATH, 'tencent-smiley.json')
-TENCENT_EXTRASMILEY_FILE = os.path.join(STATIC_PATH, 'tencent-smiley-extra.json')
 
 try:
     UNICODE_SMILEY_RE = re.compile(
@@ -37,10 +35,16 @@ HEAD = """.smiley {
 }
 """
 
-TEMPLATE = """.smiley{name} {{
+TEMPLATE = """.{name} {{
     background-image: url("data:image/png;base64,{b64}");
+    background-size: 24px 24px;
 }}"""
 
+def _css_class_name(s):
+    s = s.replace("/", "_")
+    s = s.replace(".", "_")
+    return "smiley_" + s
+
 class SmileyProvider(object):
     def __init__(self, html_replace=True):
         """ html_replace: replace smileycode by html.
@@ -50,74 +54,40 @@ class SmileyProvider(object):
         if not html_replace:
             raise NotImplementedError()
 
-        # [微笑] -> 0
+        # [微笑] -> smiley/0.png
         self.tencent_smiley = json.load(open(TENCENT_SMILEY_FILE))
+        self.used_smileys = set()
 
-        # some extra smiley from javascript on wx.qq.com
-        extra_smiley = json.load(open(TENCENT_EXTRASMILEY_FILE))
-        extra_smiley = {u'[' + k + u']': v for k, v in
-                            extra_smiley.items()}
-        self.tencent_smiley.update(extra_smiley)
+    def reset(self):
+        self.used_smileys.clear()
 
-        # 1f35c -> "\ue340"
-        #self.unicode_smiley_code = gUnicodeCodeMap
-
-        # u'\U0001f35c' -> "e340"   # for iphone
-        # u'\ue415' -> 'e415'       # for android
-        unicode_smiley_dict = json.load(open(UNICODE_SMILEY_FILE))
-        self.unicode_smiley = {(self.unichar(int(k, 16))): hex(ord(v))[2:] for k, v in
-                                unicode_smiley_dict.items()}
-        self.unicode_smiley.update({v: hex(ord(v))[2:] for _, v in
-                                unicode_smiley_dict.items()})
-        self.used_smiley_id = set()
-
-    def unichar(self, i):
-        try:
-            return chr(i)
-        except ValueError:
-            return struct.pack('i', i).decode('utf-32')
-
-    def gen_replace_elem(self, smiley_id):
-        self.used_smiley_id.add(str(smiley_id))
-        return '<span class="smiley smiley{}"></span>'.format(smiley_id)
-
-    def _replace_unicode(self, msg):
-        if not UNICODE_SMILEY_RE.findall(msg):
-        # didn't find the code
-            return msg
-        for k, v in self.unicode_smiley.items():
-            if k in msg:
-                msg = msg.replace(k, self.gen_replace_elem(v))
-        return msg
-
-    def _replace_tencent(self, msg):
-        if (not '[' in msg or not ']' in msg) \
-           and (not '/:' in msg) and (not '/' in msg):
-            return msg
-        for k, v in self.tencent_smiley.items():
-            if k in msg:
-                msg = msg.replace(k, self.gen_replace_elem(v))
-        return msg
+    def gen_replace_elem(self, smiley_path):
+        self.used_smileys.add(str(smiley_path))
+        return '<span class="smiley {}"></span>'.format(_css_class_name(smiley_path))
 
     def replace_smileycode(self, msg):
         """ replace the smiley code in msg
             return a html
         """
-        msg = self._replace_unicode(msg)
-        msg = self._replace_tencent(msg)
+        # pre-filter:
+        if ('[' not in msg) and ('/' not in msg) and not UNICODE_SMILEY_RE.findall(msg):
+            return msg
+        for k, v in self.tencent_smiley.items():
+            if k in msg:
+                msg = msg.replace(k, self.gen_replace_elem(v))
+        return msg
         return msg
 
     def gen_used_smiley_css(self):
         ret = HEAD
-        for sid in self.used_smiley_id:
-            fname = os.path.join(STATIC_PATH, 'smileys', '{}.png'.format(sid))
+        for path in self.used_smileys:
+            fname = os.path.join(STATIC_PATH, path)
             b64 = get_file_b64(fname)
-            ret = ret + TEMPLATE.format(name=sid, b64=b64)
+            ret = ret + TEMPLATE.format(name=_css_class_name(path), b64=b64)
         return ret
 
 if __name__ == '__main__':
     smiley = SmileyProvider()
     msg = u"[挥手]哈哈呵呵ｈｉｈｉ\U0001f684\u2728\u0001 /::<\ue415"
     msg = smiley.replace_smileycode(msg)
-    #print msg
     smiley.gen_used_smiley_css()

wechat/static/README.md

@@ -0,0 +1,10 @@
+
+Sources:
+
+* web wechat
+* /data/data/com.tencent.mm/files/public/emoji/newemoji
+
+
+Generated file:
+
+* tencent-smiley.json generated by  `parse_tencent_smiley.py`

wechat/static/newemoji/config.conf

@@ -0,0 +1,2 @@
+{
+	"version": 86}

wechat/static/newemoji/newemoji-config.xml

@@ -0,0 +1,275 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<newemoji> 
+	<emoji> 
+    <key><![CDATA[[Smirk]]]></key>  
+    <cn-value><![CDATA[[奸笑]]]></cn-value>  
+    <qq-value><![CDATA[[奸笑]]]></qq-value>  
+    <en-value><![CDATA[[Smirk]]]></en-value>  
+    <tw-value><![CDATA[[奸笑]]]></tw-value>  
+    <th-value><![CDATA[[Smirk]]]></th-value>  
+    <fileName>2_02.png</fileName>  
+    <eggIndex>2002</eggIndex> 
+  </emoji>  
+  <emoji> 
+    <key><![CDATA[[Hey]]]></key>  
+    <cn-value><![CDATA[[嘿哈]]]></cn-value>  
+    <qq-value><![CDATA[[嘿哈]]]></qq-value>  
+    <en-value><![CDATA[[Hey]]]></en-value>  
+    <tw-value><![CDATA[[吼嘿]]]></tw-value>  
+    <th-value><![CDATA[[Hey]]]></th-value>  
+    <fileName>2_04.png</fileName>  
+    <eggIndex>2004</eggIndex> 
+  </emoji>  
+  <emoji> 
+    <key><![CDATA[[Facepalm]]]></key>  
+    <cn-value><![CDATA[[捂脸]]]></cn-value>  
+    <qq-value><![CDATA[[捂脸]]]></qq-value>  
+    <en-value><![CDATA[[Facepalm]]]></en-value>  
+    <tw-value><![CDATA[[掩面]]]></tw-value>  
+    <th-value><![CDATA[[Facepalm]]]></th-value>  
+    <fileName>2_05.png</fileName>  
+    <eggIndex>2005</eggIndex> 
+  </emoji>  
+  <emoji> 
+    <key><![CDATA[[Smart]]]></key>  
+    <cn-value><![CDATA[[机智]]]></cn-value>  
+    <qq-value><![CDATA[[机智]]]></qq-value>  
+    <en-value><![CDATA[[Smart]]]></en-value>  
+    <tw-value><![CDATA[[機智]]]></tw-value>  
+    <th-value><![CDATA[[Smart]]]></th-value>  
+    <fileName>2_06.png</fileName>  
+    <eggIndex>2006</eggIndex> 
+  </emoji>  
+  <emoji> 
+    <key><![CDATA[[Tea]]]></key>  
+    <cn-value><![CDATA[[茶]]]></cn-value>  
+    <qq-value><![CDATA[[茶]]]></qq-value>  
+    <en-value><![CDATA[[Tea]]]></en-value>  
+    <tw-value><![CDATA[[茶]]]></tw-value>  
+    <th-value><![CDATA[[Tea]]]></th-value>  
+    <fileName>2_07.png</fileName>  
+    <eggIndex>2007</eggIndex> 
+  </emoji> 
+  <emoji> 
+    <key><![CDATA[[Packet]]]></key>
+    <cn-value><![CDATA[[红包]]]></cn-value>  
+    <qq-value><![CDATA[[红包]]]></qq-value>  
+    <en-value><![CDATA[[Packet]]]></en-value>  
+    <tw-value><![CDATA[[Packet]]]></tw-value>
+    <th-value><![CDATA[[Packet]]]></th-value>  
+    <fileName>2_09.png</fileName>  
+    <eggIndex>2009</eggIndex> 
+  </emoji>  
+  <emoji> 
+    <key><![CDATA[[Candle]]]></key>
+    <cn-value><![CDATA[[蜡烛]]]></cn-value>  
+    <qq-value><![CDATA[[蜡烛]]]></qq-value>  
+    <en-value><![CDATA[[Candle]]]></en-value>  
+    <tw-value><![CDATA[[蠟燭]]]></tw-value>
+    <th-value><![CDATA[[Candle]]]></th-value>  
+    <fileName>2_10.png</fileName>  
+    <eggIndex>2010</eggIndex> 
+  </emoji> 
+  <emoji> 
+    <key><![CDATA[[Yeah!]]]></key>
+    <cn-value><![CDATA[[耶]]]></cn-value>  
+    <qq-value><![CDATA[[耶]]]></qq-value>  
+    <en-value><![CDATA[[Yeah!]]]></en-value>  
+    <tw-value><![CDATA[[歐耶]]]></tw-value>
+    <th-value><![CDATA[[Yeah!]]]></th-value>  
+    <fileName>2_11.png</fileName>  
+    <eggIndex>2011</eggIndex> 
+  </emoji>   
+  <emoji> 
+    <key><![CDATA[[Concerned]]]></key>
+    <cn-value><![CDATA[[皱眉]]]></cn-value>  
+    <qq-value><![CDATA[[皱眉]]]></qq-value>  
+    <en-value><![CDATA[[Concerned]]]></en-value>  
+    <tw-value><![CDATA[[皺眉]]]></tw-value>
+    <th-value><![CDATA[[Concerned]]]></th-value>  
+    <fileName>2_12.png</fileName>  
+    <eggIndex>2018</eggIndex> 
+  </emoji> 
+  <emoji> 
+    <key><![CDATA[[Blush]]]></key>  
+    <cn-value><![CDATA[[囧]]]></cn-value>  
+    <qq-value><![CDATA[[囧]]]></qq-value>  
+    <en-value><![CDATA[[Blush]]]></en-value>  
+    <tw-value><![CDATA[[囧]]]></tw-value>  
+    <th-value><![CDATA[[Blush]]]></th-value>  
+    <fileName>smiley_17b.png</fileName>  
+    <eggIndex>2013</eggIndex> 
+  </emoji>  
+  <emoji> 
+    <key><![CDATA[[Salute]]]></key>  
+    <cn-value><![CDATA[[抱拳]]]></cn-value>  
+    <qq-value><![CDATA[[抱拳]]]></qq-value>  
+    <en-value><![CDATA[[Salute]]]></en-value>  
+    <tw-value><![CDATA[[抱拳]]]></tw-value>  
+    <th-value><![CDATA[[Salute]]]></th-value>  
+    <fileName>smiley_83b.png</fileName>  
+    <eggIndex>2014</eggIndex> 
+  </emoji>  
+  <emoji> 
+    <key><![CDATA[[Chick]]]></key>  
+    <cn-value><![CDATA[[鸡]]]></cn-value>  
+    <qq-value><![CDATA[[鸡]]]></qq-value>  
+    <en-value><![CDATA[[Chick]]]></en-value>  
+    <tw-value><![CDATA[[小雞]]]></tw-value>  
+    <th-value><![CDATA[[Chick]]]></th-value>  
+    <fileName>2_14.png</fileName>  
+    <eggIndex>2015</eggIndex> 
+  </emoji> 
+  <emoji> 
+    <key><![CDATA[[Blessing]]]></key>  
+    <cn-value><![CDATA[[福]]]></cn-value>  
+    <qq-value><![CDATA[[福]]]></qq-value>  
+    <en-value><![CDATA[[Blessing]]]></en-value>  
+    <tw-value><![CDATA[[福]]]></tw-value>  
+    <th-value><![CDATA[[Blessing]]]></th-value>  
+    <fileName>2_15.png</fileName>  
+    <eggIndex>2016</eggIndex> 
+  </emoji>  
+  <emoji> 
+    <key><![CDATA[[Bye]]]></key>  
+    <cn-value><![CDATA[[再见]]]></cn-value>  
+    <qq-value><![CDATA[[再见]]]></qq-value>  
+    <en-value><![CDATA[[Bye]]]></en-value>  
+    <tw-value><![CDATA[[再見]]]></tw-value>  
+    <th-value><![CDATA[[Bye]]]></th-value>  
+    <fileName>smiley_39b.png</fileName>  
+    <eggIndex>2017</eggIndex> 
+  </emoji>  
+ 
+  <emoji> 
+    <key><![CDATA[[Rich]]]></key>
+    <cn-value><![CDATA[[發]]]></cn-value>  
+    <qq-value><![CDATA[[發]]]></qq-value>  
+    <en-value><![CDATA[[Rich]]]></en-value>  
+    <tw-value><![CDATA[[發]]]></tw-value>
+    <th-value><![CDATA[[Rich]]]></th-value>  
+    <fileName>2_16.png</fileName>  
+    <eggIndex>2019</eggIndex> 
+  </emoji>  
+  <emoji> 
+    <key><![CDATA[[Pup]]]></key>
+    <cn-value><![CDATA[[小狗]]]></cn-value>  
+    <qq-value><![CDATA[[小狗]]]></qq-value>  
+    <en-value><![CDATA[[Pup]]]></en-value>  
+    <tw-value><![CDATA[[小狗]]]></tw-value>
+    <th-value><![CDATA[[Pup]]]></th-value>  
+    <fileName>2_17.png</fileName>  
+    <eggIndex>2020</eggIndex> 
+  </emoji> 
+  <emoji> 
+    <key><![CDATA[[Onlooker]]]></key>
+    <cn-value><![CDATA[[吃瓜]]]></cn-value>  
+    <qq-value><![CDATA[[吃瓜]]]></qq-value>  
+    <en-value><![CDATA[[Onlooker]]]></en-value>  
+    <tw-value><![CDATA[[吃西瓜]]]></tw-value>
+    <th-value><![CDATA[[Onlooker]]]></th-value>  
+    <fileName>Watermelon.png</fileName>  
+    <eggIndex>2021</eggIndex> 
+ </emoji>
+  <emoji> 
+    <key><![CDATA[[GoForIt]]]></key>
+    <cn-value><![CDATA[[加油]]]></cn-value>  
+    <qq-value><![CDATA[[加油]]]></qq-value>  
+    <en-value><![CDATA[[GoForIt]]]></en-value>  
+    <tw-value><![CDATA[[加油]]]></tw-value>
+    <th-value><![CDATA[[GoForIt]]]></th-value>  
+    <fileName>Addoil.png</fileName>  
+    <eggIndex>2022</eggIndex> 
+  </emoji>
+    <emoji> 
+    <key><![CDATA[[Sweats]]]></key>
+    <cn-value><![CDATA[[汗]]]></cn-value>  
+    <qq-value><![CDATA[[汗]]]></qq-value>  
+    <en-value><![CDATA[[Sweats]]]></en-value>  
+    <tw-value><![CDATA[[汗]]]></tw-value>
+    <th-value><![CDATA[[Sweats]]]></th-value>  
+    <fileName>Sweat.png</fileName>  
+    <eggIndex>2023</eggIndex> 
+  </emoji>
+    <emoji> 
+    <key><![CDATA[[OMG]]]></key>
+    <cn-value><![CDATA[[天啊]]]></cn-value>  
+    <qq-value><![CDATA[[天啊]]]></qq-value>  
+    <en-value><![CDATA[[OMG]]]></en-value>  
+    <tw-value><![CDATA[[天啊]]]></tw-value>
+    <th-value><![CDATA[[OMG]]]></th-value>  
+    <fileName>Shocked!.png</fileName>  
+    <eggIndex>2025</eggIndex> 
+  </emoji>  
+    <emoji> 
+    <key><![CDATA[[Emm]]]></key>
+    <cn-value><![CDATA[[Emm]]]></cn-value>  
+    <qq-value><![CDATA[[Emm]]]></qq-value>  
+    <en-value><![CDATA[[Emm]]]></en-value>  
+    <tw-value><![CDATA[[一言難盡]]]></tw-value>
+    <th-value><![CDATA[[Emm]]]></th-value>  
+    <fileName>Cold.png</fileName>  
+    <eggIndex>2027</eggIndex> 
+  </emoji>
+    <emoji> 
+    <key><![CDATA[[Respect]]]></key>
+    <cn-value><![CDATA[[社会社会]]]></cn-value>  
+    <qq-value><![CDATA[[社会社会]]]></qq-value>  
+    <en-value><![CDATA[[Respect]]]></en-value>  
+    <tw-value><![CDATA[[失敬失敬]]]></tw-value>
+    <th-value><![CDATA[[Respect]]]></th-value>  
+    <fileName>Social.png</fileName>  
+    <eggIndex>2028</eggIndex> 
+  </emoji>
+   <emoji> 
+    <key><![CDATA[[Doge]]]></key>
+    <cn-value><![CDATA[[旺柴]]]></cn-value>  
+    <qq-value><![CDATA[[旺柴]]]></qq-value>  
+    <en-value><![CDATA[[Doge]]]></en-value>  
+    <tw-value><![CDATA[[旺柴]]]></tw-value>
+    <th-value><![CDATA[[Doge]]]></th-value>  
+    <fileName>Yellowdog.png</fileName>  
+    <eggIndex>2030</eggIndex> 
+ </emoji>
+   <emoji> 
+    <key><![CDATA[[NoProb]]]></key>
+    <cn-value><![CDATA[[好的]]]></cn-value>  
+    <qq-value><![CDATA[[好的]]]></qq-value>  
+    <en-value><![CDATA[[NoProb]]]></en-value>  
+    <tw-value><![CDATA[[好的]]]></tw-value>
+    <th-value><![CDATA[[NoProb]]]></th-value>  
+    <fileName>NoProb.png</fileName>  
+    <eggIndex>2034</eggIndex> 
+ </emoji>
+    <emoji> 
+    <key><![CDATA[[MyBad]]]></key>
+    <cn-value><![CDATA[[打脸]]]></cn-value>  
+    <qq-value><![CDATA[[打脸]]]></qq-value>  
+    <en-value><![CDATA[[MyBad]]]></en-value>  
+    <tw-value><![CDATA[[打臉]]]></tw-value>
+    <th-value><![CDATA[[MyBad]]]></th-value>  
+    <fileName>Slap.png</fileName>  
+    <eggIndex>2036</eggIndex> 
+  </emoji>
+    <emoji> 
+    <key><![CDATA[[Wow]]]></key>
+    <cn-value><![CDATA[[哇]]]></cn-value>  
+    <qq-value><![CDATA[[哇]]]></qq-value>  
+    <en-value><![CDATA[[Wow]]]></en-value>  
+    <tw-value><![CDATA[[哇]]]></tw-value>
+    <th-value><![CDATA[[Wow]]]></th-value>  
+    <fileName>Wow!.png</fileName>  
+    <eggIndex>2037</eggIndex>
+  </emoji>	
+  <emoji> 
+    <key><![CDATA[[KeepFighting]]]></key>
+    <cn-value><![CDATA[[加油加油]]]></cn-value>  
+    <qq-value><![CDATA[[加油加油]]]></qq-value>  
+    <en-value><![CDATA[[KeepFighting]]]></en-value>  
+    <tw-value><![CDATA[[加油！]]]></tw-value>
+    <th-value><![CDATA[[KeepFighting]]]></th-value>  
+    <fileName>KeepFighting.png</fileName>  
+    <eggIndex>2038</eggIndex> 
+  </emoji> 
+</newemoji>

wechat/static/parse_tencent_smiley.py

@@ -5,14 +5,16 @@ import xml.etree.ElementTree as ET
 import os
 import json
 
-tree = ET.parse(os.path.join(
-    os.path.dirname(os.path.abspath(__file__)), 'smiley.xml'))
-root = tree.getroot()
 
-smileys = {}
-for child in root:
-    name = child.attrib['name']
-    if 'smiley_values' in name:
+def parse_smiley_xml():
+    ret = {}
+    tree = ET.parse('smiley.xml')
+    root = tree.getroot()
+
+    for child in root:
+        name = child.attrib['name']
+        if 'smiley_values' not in name:
+            continue
         if '_th' in name:
             continue        # ignore thailand language
         lst = [c.text for c in child]
@@ -23,9 +25,63 @@ for child in root:
                 v = v.strip('"')
                 v = v.replace('&lt;', '<')
                 v = v.replace('&amp;', '&')
-                v = v.decode('utf-8')
-            smileys[v] = idx
+            ret[v] = os.path.join("smileys", f"{idx}.png")
+    return ret
 
-with open('tencent-smiley.json', 'w') as f:
-    json.dump(smileys, f)
 
+def parse_extra_smiley():
+    # some extra smiley from javascript on wx.qq.com
+    with open("tencent-smiley-extra.json") as f:
+        obj = json.load(f)
+    extra = {'[' + k + ']': os.path.join("smileys", f"{v}.png") for k, v in obj.items()}
+    return extra
+
+def parse_new_emoji():
+    ret = {}
+    xmlfile = "newemoji/newemoji-config.xml"
+    tree = ET.parse(xmlfile)
+    root = tree.getroot()
+    for child in root:
+        assert child.tag == "emoji"
+        vals = {k.tag: k.text for k in child}
+        filename = os.path.join("newemoji", vals["fileName"])
+        for k, v in vals.items():
+            if "-value" in k:
+                ret[v] = filename
+    return ret
+
+def parse_unicode_smiley():
+    # 1f35c -> "\ue340"
+    #self.unicode_smiley_code = gUnicodeCodeMap
+
+    # u'\U0001f35c' -> "e340"   # for iphone
+    # u'\ue415' -> 'e415'       # for android
+    def unichar(i):
+        try:
+            return chr(i)
+        except ValueError:
+            return struct.pack('i', i).decode('utf-32')
+
+    ret = {}
+    with open("unicode-smiley.json") as f:
+        d = json.load(f)
+    for k, v in d.items():
+        fname = os.path.join("smileys", hex(ord(v))[2:] + ".png")
+        ret[unichar(int(k, 16))] = fname
+        ret[v] = fname
+    return ret
+
+if __name__ == "__main__":
+    # parse old smileys
+    smileys = {}
+    def add(dic, name):
+        smileys.update(dic)
+        print(f"Found {len(dic)} smileys from {name}. Total is {len(smileys)}")
+
+    add(parse_smiley_xml(), "smiley.xml")
+    add(parse_extra_smiley(), "tencent-smiley-extra.json")
+    add(parse_new_emoji(), "newemoji")
+    add(parse_unicode_smiley(), "unicode-smiley.json")
+
+    with open('tencent-smiley.json', 'w') as f:
+        json.dump(smileys, f)