feat: real conversation threading at ingest + migration script
The IMAP parser only extracted Message-ID, so In-Reply-To / References were never persisted and threadId stayed empty -> conversations never grouped with real data (verified: 162/162 empty in user's DB). - ParsedMimeMessage: add inReplyTo + references fields - MimeStorage::parseMessage: extract In-Reply-To and References headers - ImapSynchronizer parseFetchResponseBytes: propagate inReplyTo/references, compute + persist threadId at ingest time (ThreadUtils::extractThreadId) - MimeStorage::storeMessage: propagate the same headers on MailItem - scripts/rethread.py: migrate existing DBs by re-reading .eml files (recovered In-Reply-To/References from raw MIME and recomputed threadId: 149/162 grouped, incl. Nacho x6, Bitar x4, Mariel x3, etc.) Verified against user's real DB copy.
This commit is contained in:
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Recupera In-Reply-To/References del .eml y recalcula threadId en Wino Mail.
|
||||
Uso: python3 rethread2.py <bd.sqlite> <carpeta_mails>
|
||||
"""
|
||||
import sys, re, sqlite3, os, email
|
||||
|
||||
bd = sys.argv[1]
|
||||
mails_dir = sys.argv[2]
|
||||
conn = sqlite3.connect(bd)
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("SELECT id, messageId FROM MailCopy")
|
||||
rows = cur.fetchall()
|
||||
|
||||
# indexar emls por message-id en nombre de archivo
|
||||
eml_by_mid = {}
|
||||
for root, _, files in os.walk(mails_dir):
|
||||
for f in files:
|
||||
if not f.endswith(".eml"):
|
||||
continue
|
||||
# nombre base contiene el message-id antes de .eml
|
||||
base = f[:-4]
|
||||
eml_by_mid[base] = os.path.join(root, f)
|
||||
|
||||
def parse_headers(p):
|
||||
try:
|
||||
msg = email.message_from_bytes(open(p, "rb").read())
|
||||
except Exception:
|
||||
return None
|
||||
mid = msg.get("Message-ID", "")
|
||||
irt = msg.get("In-Reply-To", "")
|
||||
refs = msg.get("References", "")
|
||||
return mid, irt, refs
|
||||
|
||||
def thread_id(mid, irt, refs):
|
||||
if refs:
|
||||
m = re.search(r"<([^>]+)>", refs)
|
||||
return m.group(1) if m else refs.strip("<>")
|
||||
if irt:
|
||||
m = re.search(r"<([^>]+)>", irt)
|
||||
return m.group(1) if m else irt.strip("<>")
|
||||
if mid:
|
||||
m = re.search(r"<([^>]+)>", mid)
|
||||
return m.group(1) if m else mid.strip("<>")
|
||||
return ""
|
||||
|
||||
upd_thread = 0
|
||||
upd_irt = 0
|
||||
upd_refs = 0
|
||||
not_found = 0
|
||||
no_headers = 0
|
||||
|
||||
for rid, mid in rows:
|
||||
mid = (mid or "").strip()
|
||||
# buscar eml: por nombre base == mid, o mid contenido
|
||||
p = None
|
||||
if mid in eml_by_mid:
|
||||
p = eml_by_mid[mid]
|
||||
if p is None:
|
||||
# intentar coincidencia parcial (dominio distinto)
|
||||
mid_nodot = mid.split("@")[0][:20]
|
||||
for base, path in eml_by_mid.items():
|
||||
if mid_nodot and mid_nodot in base:
|
||||
p = path
|
||||
break
|
||||
if p is None:
|
||||
not_found += 1
|
||||
continue
|
||||
|
||||
hdrs = parse_headers(p)
|
||||
if hdrs is None:
|
||||
not_found += 1
|
||||
continue
|
||||
emid, eirt, erefs = hdrs
|
||||
|
||||
tid = thread_id(emid or mid, eirt, erefs)
|
||||
# guardar headers recuperados
|
||||
cur.execute("UPDATE MailCopy SET inReplyTo=?, \"references\"=?, threadId=? WHERE id=?",
|
||||
(eirt.strip() if eirt else "", erefs.strip() if erefs else "", tid or "", rid))
|
||||
if tid: upd_thread += 1
|
||||
if eirt.strip(): upd_irt += 1
|
||||
if erefs.strip(): upd_refs += 1
|
||||
|
||||
conn.commit()
|
||||
print(f"correos: {len(rows)}")
|
||||
print(f"eml no encontrados: {not_found}")
|
||||
print(f"inReplyTo recuperado: {upd_irt}")
|
||||
print(f"references recuperado: {upd_refs}")
|
||||
print(f"threadId asignado: {upd_thread}")
|
||||
|
||||
# agrupar en hilos
|
||||
print("\n=== hilos detectados (threadId con >1 correo) ===")
|
||||
cur.execute("SELECT threadId, COUNT(*), MAX(sender) FROM MailCopy WHERE threadId!='' GROUP BY threadId HAVING COUNT(*)>1 ORDER BY COUNT(*) DESC LIMIT 15")
|
||||
for row in cur.fetchall():
|
||||
print(row)
|
||||
conn.close()
|
||||
@@ -9,6 +9,7 @@
|
||||
#include "../../db/dao/folderdao.h"
|
||||
#include "../../db/dao/mailitemdao.h"
|
||||
#include "../../services/mimestorage.h"
|
||||
#include "../../utils/threadutils.h"
|
||||
#include <algorithm>
|
||||
|
||||
ImapSynchronizer::ImapSynchronizer(QObject* parent)
|
||||
@@ -526,6 +527,23 @@ QVector<MailItem> ImapSynchronizer::parseFetchResponseBytes(const QByteArray& re
|
||||
MailItem item;
|
||||
item.setUid(uidMatch.captured(1).toLongLong());
|
||||
item.setMessageId(parsed.messageId);
|
||||
item.setInReplyTo(parsed.inReplyTo);
|
||||
// Persist references so conversation grouping can reconstruct threads.
|
||||
if (!parsed.references.isEmpty()) {
|
||||
QStringList refs;
|
||||
QRegularExpression re("<([^>]+)>");
|
||||
auto it = re.globalMatch(parsed.references);
|
||||
while (it.hasNext()) refs.append(it.next().captured(1));
|
||||
item.setReferences(refs);
|
||||
}
|
||||
// Compute and store the thread id at ingest time.
|
||||
if (!item.threadId().isEmpty()) {
|
||||
// already set
|
||||
} else if (!item.inReplyTo().isEmpty() || !item.references().isEmpty()
|
||||
|| (!parsed.references.isEmpty())) {
|
||||
item.setThreadId(ThreadUtils::extractThreadId(
|
||||
item.messageId(), item.inReplyTo(), item.references()));
|
||||
}
|
||||
item.setSubject(parsed.subject.isEmpty() ? QStringLiteral("(No Subject)") : parsed.subject);
|
||||
item.setSender(parsed.from);
|
||||
item.setRecipient(parsed.to);
|
||||
|
||||
@@ -85,6 +85,14 @@ bool MimeStorageService::storeMessage(const QString &accountId, const QString &f
|
||||
if (!parseMessage(rawMime, parsed)) return false;
|
||||
|
||||
if (!parsed.messageId.isEmpty()) mail.setMessageId(parsed.messageId);
|
||||
if (!parsed.inReplyTo.isEmpty()) mail.setInReplyTo(parsed.inReplyTo);
|
||||
if (!parsed.references.isEmpty()) {
|
||||
QStringList refs;
|
||||
QRegularExpression re("<([^>]+)>");
|
||||
auto it = re.globalMatch(parsed.references);
|
||||
while (it.hasNext()) refs.append(it.next().captured(1));
|
||||
mail.setReferences(refs);
|
||||
}
|
||||
if (!parsed.subject.isEmpty()) mail.setSubject(parsed.subject);
|
||||
if (!parsed.from.isEmpty()) mail.setSender(parsed.from);
|
||||
if (!parsed.to.isEmpty()) { mail.setTo(parsed.to); mail.setRecipient(parsed.to); }
|
||||
@@ -339,6 +347,8 @@ bool MimeStorageService::parseMessage(const QByteArray &rawMime, ParsedMimeMessa
|
||||
};
|
||||
const QMap<QString, QString> headers = parseHeaders(rawMime);
|
||||
message.messageId = decodeHeaderValue(headers.value(QStringLiteral("message-id"))).remove('<').remove('>');
|
||||
message.inReplyTo = decodeHeaderValue(headers.value(QStringLiteral("in-reply-to"))).remove('<').remove('>');
|
||||
message.references = decodeHeaderValue(headers.value(QStringLiteral("references")));
|
||||
message.subject = decodeHeaderValue(headers.value(QStringLiteral("subject")));
|
||||
message.from = decodeHeaderValue(headers.value(QStringLiteral("from")));
|
||||
message.to = decodeHeaderValue(headers.value(QStringLiteral("to")));
|
||||
|
||||
@@ -27,6 +27,8 @@ struct ParsedMimeMessage
|
||||
QString bcc;
|
||||
QDateTime date;
|
||||
QString bodyHtml;
|
||||
QString inReplyTo;
|
||||
QString references;
|
||||
QVector<ParsedMimeAttachment> attachments;
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user