feat: real conversation threading at ingest + migration script
The IMAP parser only extracted Message-ID, so In-Reply-To / References were never persisted and threadId stayed empty -> conversations never grouped with real data (verified: 162/162 empty in user's DB). - ParsedMimeMessage: add inReplyTo + references fields - MimeStorage::parseMessage: extract In-Reply-To and References headers - ImapSynchronizer parseFetchResponseBytes: propagate inReplyTo/references, compute + persist threadId at ingest time (ThreadUtils::extractThreadId) - MimeStorage::storeMessage: propagate the same headers on MailItem - scripts/rethread.py: migrate existing DBs by re-reading .eml files (recovered In-Reply-To/References from raw MIME and recomputed threadId: 149/162 grouped, incl. Nacho x6, Bitar x4, Mariel x3, etc.) Verified against user's real DB copy.
This commit is contained in:
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Recupera In-Reply-To/References del .eml y recalcula threadId en Wino Mail.
|
||||
Uso: python3 rethread2.py <bd.sqlite> <carpeta_mails>
|
||||
"""
|
||||
import sys, re, sqlite3, os, email
|
||||
|
||||
bd = sys.argv[1]
|
||||
mails_dir = sys.argv[2]
|
||||
conn = sqlite3.connect(bd)
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("SELECT id, messageId FROM MailCopy")
|
||||
rows = cur.fetchall()
|
||||
|
||||
# indexar emls por message-id en nombre de archivo
|
||||
eml_by_mid = {}
|
||||
for root, _, files in os.walk(mails_dir):
|
||||
for f in files:
|
||||
if not f.endswith(".eml"):
|
||||
continue
|
||||
# nombre base contiene el message-id antes de .eml
|
||||
base = f[:-4]
|
||||
eml_by_mid[base] = os.path.join(root, f)
|
||||
|
||||
def parse_headers(p):
|
||||
try:
|
||||
msg = email.message_from_bytes(open(p, "rb").read())
|
||||
except Exception:
|
||||
return None
|
||||
mid = msg.get("Message-ID", "")
|
||||
irt = msg.get("In-Reply-To", "")
|
||||
refs = msg.get("References", "")
|
||||
return mid, irt, refs
|
||||
|
||||
def thread_id(mid, irt, refs):
|
||||
if refs:
|
||||
m = re.search(r"<([^>]+)>", refs)
|
||||
return m.group(1) if m else refs.strip("<>")
|
||||
if irt:
|
||||
m = re.search(r"<([^>]+)>", irt)
|
||||
return m.group(1) if m else irt.strip("<>")
|
||||
if mid:
|
||||
m = re.search(r"<([^>]+)>", mid)
|
||||
return m.group(1) if m else mid.strip("<>")
|
||||
return ""
|
||||
|
||||
upd_thread = 0
|
||||
upd_irt = 0
|
||||
upd_refs = 0
|
||||
not_found = 0
|
||||
no_headers = 0
|
||||
|
||||
for rid, mid in rows:
|
||||
mid = (mid or "").strip()
|
||||
# buscar eml: por nombre base == mid, o mid contenido
|
||||
p = None
|
||||
if mid in eml_by_mid:
|
||||
p = eml_by_mid[mid]
|
||||
if p is None:
|
||||
# intentar coincidencia parcial (dominio distinto)
|
||||
mid_nodot = mid.split("@")[0][:20]
|
||||
for base, path in eml_by_mid.items():
|
||||
if mid_nodot and mid_nodot in base:
|
||||
p = path
|
||||
break
|
||||
if p is None:
|
||||
not_found += 1
|
||||
continue
|
||||
|
||||
hdrs = parse_headers(p)
|
||||
if hdrs is None:
|
||||
not_found += 1
|
||||
continue
|
||||
emid, eirt, erefs = hdrs
|
||||
|
||||
tid = thread_id(emid or mid, eirt, erefs)
|
||||
# guardar headers recuperados
|
||||
cur.execute("UPDATE MailCopy SET inReplyTo=?, \"references\"=?, threadId=? WHERE id=?",
|
||||
(eirt.strip() if eirt else "", erefs.strip() if erefs else "", tid or "", rid))
|
||||
if tid: upd_thread += 1
|
||||
if eirt.strip(): upd_irt += 1
|
||||
if erefs.strip(): upd_refs += 1
|
||||
|
||||
conn.commit()
|
||||
print(f"correos: {len(rows)}")
|
||||
print(f"eml no encontrados: {not_found}")
|
||||
print(f"inReplyTo recuperado: {upd_irt}")
|
||||
print(f"references recuperado: {upd_refs}")
|
||||
print(f"threadId asignado: {upd_thread}")
|
||||
|
||||
# agrupar en hilos
|
||||
print("\n=== hilos detectados (threadId con >1 correo) ===")
|
||||
cur.execute("SELECT threadId, COUNT(*), MAX(sender) FROM MailCopy WHERE threadId!='' GROUP BY threadId HAVING COUNT(*)>1 ORDER BY COUNT(*) DESC LIMIT 15")
|
||||
for row in cur.fetchall():
|
||||
print(row)
|
||||
conn.close()
|
||||
Reference in New Issue
Block a user