From 6d9e4a00a9bac9e056cb9d61a629c69a1802f103 Mon Sep 17 00:00:00 2001
From: Yuxin Wu <ppwwyyxx@users.noreply.github.com>
Date: Wed, 22 Jan 2025 04:25:01 -0800
Subject: [PATCH] fix utf8 decode error

---
 requirements.txt |  1 +
 wechat/parser.py | 15 ++++++++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 6393861..4d7d41f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,3 +6,4 @@ requests
 pysqlcipher3>=1.0.3
 csscompressor
 numpy
+ipython
diff --git a/wechat/parser.py b/wechat/parser.py
index 1d59360..a593d04 100644
--- a/wechat/parser.py
+++ b/wechat/parser.py
@@ -26,7 +26,11 @@ class WeChatDBParser(object):
         """ db_fname: a decoded EnMicroMsg.db"""
         self.db_fname = db_fname
         self.db_conn = sqlite3.connect(self.db_fname)
+        self.db_conn_bytes = sqlite3.connect(self.db_fname)
+        # https://stackoverflow.com/questions/22751363/sqlite3-operationalerror-could-not-decode-to-utf-8-column
+        self.db_conn_bytes.text_factory = lambda b: b
         self.cc = self.db_conn.cursor()
+
         self.contacts = {}      # username -> nickname
         self.contacts_rev = defaultdict(list)
         self.msgs_by_chat = defaultdict(list)
@@ -53,7 +57,7 @@ SELECT username,conRemark,nickname FROM rcontact
 
     def _parse_msg(self):
         msgs_tot_cnt = 0
-        db_msgs = self.cc.execute(
+        db_msgs = self.db_conn_bytes.cursor().execute(
 """
 SELECT {} FROM message
 """.format(','.join(WeChatDBParser.FIELDS)))
@@ -136,11 +140,20 @@ SELECT {} FROM message
     def _parse_msg_row(self, row):
         """ parse a record of message into my format"""
         values = dict(zip(WeChatDBParser.FIELDS, row))
+        try:
+            values['content'].decode()
+        except:
+            logger.warning(f"Invalid byte sequence in message content (type={values['type']})")
+            values['content'] = 'FAILED TO DECODE'
         if values['content']:
             values['content'] = ensure_unicode(values['content'])
         else:
             values['content'] = ''
         values['createTime'] = datetime.fromtimestamp(values['createTime']/ 1000)
+
+        values['talker'] = values['talker'].decode()
+        if values['imgPath']:
+            values['imgPath'] = values['imgPath'].decode()
         values['chat'] = values['talker']
         try:
             if values['chat'].endswith('@chatroom'):