feat: real conversation threading at ingest + migration script
The IMAP parser only extracted Message-ID, so In-Reply-To / References were never persisted and threadId stayed empty -> conversations never grouped with real data (verified: 162/162 empty in user's DB). - ParsedMimeMessage: add inReplyTo + references fields - MimeStorage::parseMessage: extract In-Reply-To and References headers - ImapSynchronizer parseFetchResponseBytes: propagate inReplyTo/references, compute + persist threadId at ingest time (ThreadUtils::extractThreadId) - MimeStorage::storeMessage: propagate the same headers on MailItem - scripts/rethread.py: migrate existing DBs by re-reading .eml files (recovered In-Reply-To/References from raw MIME and recomputed threadId: 149/162 grouped, incl. Nacho x6, Bitar x4, Mariel x3, etc.) Verified against user's real DB copy.
This commit is contained in:
@@ -0,0 +1,96 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Recupera In-Reply-To/References del .eml y recalcula threadId en Wino Mail.
|
||||||
|
Uso: python3 rethread2.py <bd.sqlite> <carpeta_mails>
|
||||||
|
"""
|
||||||
|
import sys, re, sqlite3, os, email
|
||||||
|
|
||||||
|
bd = sys.argv[1]
|
||||||
|
mails_dir = sys.argv[2]
|
||||||
|
conn = sqlite3.connect(bd)
|
||||||
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
cur.execute("SELECT id, messageId FROM MailCopy")
|
||||||
|
rows = cur.fetchall()
|
||||||
|
|
||||||
|
# indexar emls por message-id en nombre de archivo
|
||||||
|
eml_by_mid = {}
|
||||||
|
for root, _, files in os.walk(mails_dir):
|
||||||
|
for f in files:
|
||||||
|
if not f.endswith(".eml"):
|
||||||
|
continue
|
||||||
|
# nombre base contiene el message-id antes de .eml
|
||||||
|
base = f[:-4]
|
||||||
|
eml_by_mid[base] = os.path.join(root, f)
|
||||||
|
|
||||||
|
def parse_headers(p):
|
||||||
|
try:
|
||||||
|
msg = email.message_from_bytes(open(p, "rb").read())
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
mid = msg.get("Message-ID", "")
|
||||||
|
irt = msg.get("In-Reply-To", "")
|
||||||
|
refs = msg.get("References", "")
|
||||||
|
return mid, irt, refs
|
||||||
|
|
||||||
|
def thread_id(mid, irt, refs):
|
||||||
|
if refs:
|
||||||
|
m = re.search(r"<([^>]+)>", refs)
|
||||||
|
return m.group(1) if m else refs.strip("<>")
|
||||||
|
if irt:
|
||||||
|
m = re.search(r"<([^>]+)>", irt)
|
||||||
|
return m.group(1) if m else irt.strip("<>")
|
||||||
|
if mid:
|
||||||
|
m = re.search(r"<([^>]+)>", mid)
|
||||||
|
return m.group(1) if m else mid.strip("<>")
|
||||||
|
return ""
|
||||||
|
|
||||||
|
upd_thread = 0
|
||||||
|
upd_irt = 0
|
||||||
|
upd_refs = 0
|
||||||
|
not_found = 0
|
||||||
|
no_headers = 0
|
||||||
|
|
||||||
|
for rid, mid in rows:
|
||||||
|
mid = (mid or "").strip()
|
||||||
|
# buscar eml: por nombre base == mid, o mid contenido
|
||||||
|
p = None
|
||||||
|
if mid in eml_by_mid:
|
||||||
|
p = eml_by_mid[mid]
|
||||||
|
if p is None:
|
||||||
|
# intentar coincidencia parcial (dominio distinto)
|
||||||
|
mid_nodot = mid.split("@")[0][:20]
|
||||||
|
for base, path in eml_by_mid.items():
|
||||||
|
if mid_nodot and mid_nodot in base:
|
||||||
|
p = path
|
||||||
|
break
|
||||||
|
if p is None:
|
||||||
|
not_found += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
hdrs = parse_headers(p)
|
||||||
|
if hdrs is None:
|
||||||
|
not_found += 1
|
||||||
|
continue
|
||||||
|
emid, eirt, erefs = hdrs
|
||||||
|
|
||||||
|
tid = thread_id(emid or mid, eirt, erefs)
|
||||||
|
# guardar headers recuperados
|
||||||
|
cur.execute("UPDATE MailCopy SET inReplyTo=?, \"references\"=?, threadId=? WHERE id=?",
|
||||||
|
(eirt.strip() if eirt else "", erefs.strip() if erefs else "", tid or "", rid))
|
||||||
|
if tid: upd_thread += 1
|
||||||
|
if eirt.strip(): upd_irt += 1
|
||||||
|
if erefs.strip(): upd_refs += 1
|
||||||
|
|
||||||
|
conn.commit()
|
||||||
|
print(f"correos: {len(rows)}")
|
||||||
|
print(f"eml no encontrados: {not_found}")
|
||||||
|
print(f"inReplyTo recuperado: {upd_irt}")
|
||||||
|
print(f"references recuperado: {upd_refs}")
|
||||||
|
print(f"threadId asignado: {upd_thread}")
|
||||||
|
|
||||||
|
# agrupar en hilos
|
||||||
|
print("\n=== hilos detectados (threadId con >1 correo) ===")
|
||||||
|
cur.execute("SELECT threadId, COUNT(*), MAX(sender) FROM MailCopy WHERE threadId!='' GROUP BY threadId HAVING COUNT(*)>1 ORDER BY COUNT(*) DESC LIMIT 15")
|
||||||
|
for row in cur.fetchall():
|
||||||
|
print(row)
|
||||||
|
conn.close()
|
||||||
@@ -9,6 +9,7 @@
|
|||||||
#include "../../db/dao/folderdao.h"
|
#include "../../db/dao/folderdao.h"
|
||||||
#include "../../db/dao/mailitemdao.h"
|
#include "../../db/dao/mailitemdao.h"
|
||||||
#include "../../services/mimestorage.h"
|
#include "../../services/mimestorage.h"
|
||||||
|
#include "../../utils/threadutils.h"
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
|
|
||||||
ImapSynchronizer::ImapSynchronizer(QObject* parent)
|
ImapSynchronizer::ImapSynchronizer(QObject* parent)
|
||||||
@@ -526,6 +527,23 @@ QVector<MailItem> ImapSynchronizer::parseFetchResponseBytes(const QByteArray& re
|
|||||||
MailItem item;
|
MailItem item;
|
||||||
item.setUid(uidMatch.captured(1).toLongLong());
|
item.setUid(uidMatch.captured(1).toLongLong());
|
||||||
item.setMessageId(parsed.messageId);
|
item.setMessageId(parsed.messageId);
|
||||||
|
item.setInReplyTo(parsed.inReplyTo);
|
||||||
|
// Persist references so conversation grouping can reconstruct threads.
|
||||||
|
if (!parsed.references.isEmpty()) {
|
||||||
|
QStringList refs;
|
||||||
|
QRegularExpression re("<([^>]+)>");
|
||||||
|
auto it = re.globalMatch(parsed.references);
|
||||||
|
while (it.hasNext()) refs.append(it.next().captured(1));
|
||||||
|
item.setReferences(refs);
|
||||||
|
}
|
||||||
|
// Compute and store the thread id at ingest time.
|
||||||
|
if (!item.threadId().isEmpty()) {
|
||||||
|
// already set
|
||||||
|
} else if (!item.inReplyTo().isEmpty() || !item.references().isEmpty()
|
||||||
|
|| (!parsed.references.isEmpty())) {
|
||||||
|
item.setThreadId(ThreadUtils::extractThreadId(
|
||||||
|
item.messageId(), item.inReplyTo(), item.references()));
|
||||||
|
}
|
||||||
item.setSubject(parsed.subject.isEmpty() ? QStringLiteral("(No Subject)") : parsed.subject);
|
item.setSubject(parsed.subject.isEmpty() ? QStringLiteral("(No Subject)") : parsed.subject);
|
||||||
item.setSender(parsed.from);
|
item.setSender(parsed.from);
|
||||||
item.setRecipient(parsed.to);
|
item.setRecipient(parsed.to);
|
||||||
|
|||||||
@@ -85,6 +85,14 @@ bool MimeStorageService::storeMessage(const QString &accountId, const QString &f
|
|||||||
if (!parseMessage(rawMime, parsed)) return false;
|
if (!parseMessage(rawMime, parsed)) return false;
|
||||||
|
|
||||||
if (!parsed.messageId.isEmpty()) mail.setMessageId(parsed.messageId);
|
if (!parsed.messageId.isEmpty()) mail.setMessageId(parsed.messageId);
|
||||||
|
if (!parsed.inReplyTo.isEmpty()) mail.setInReplyTo(parsed.inReplyTo);
|
||||||
|
if (!parsed.references.isEmpty()) {
|
||||||
|
QStringList refs;
|
||||||
|
QRegularExpression re("<([^>]+)>");
|
||||||
|
auto it = re.globalMatch(parsed.references);
|
||||||
|
while (it.hasNext()) refs.append(it.next().captured(1));
|
||||||
|
mail.setReferences(refs);
|
||||||
|
}
|
||||||
if (!parsed.subject.isEmpty()) mail.setSubject(parsed.subject);
|
if (!parsed.subject.isEmpty()) mail.setSubject(parsed.subject);
|
||||||
if (!parsed.from.isEmpty()) mail.setSender(parsed.from);
|
if (!parsed.from.isEmpty()) mail.setSender(parsed.from);
|
||||||
if (!parsed.to.isEmpty()) { mail.setTo(parsed.to); mail.setRecipient(parsed.to); }
|
if (!parsed.to.isEmpty()) { mail.setTo(parsed.to); mail.setRecipient(parsed.to); }
|
||||||
@@ -339,6 +347,8 @@ bool MimeStorageService::parseMessage(const QByteArray &rawMime, ParsedMimeMessa
|
|||||||
};
|
};
|
||||||
const QMap<QString, QString> headers = parseHeaders(rawMime);
|
const QMap<QString, QString> headers = parseHeaders(rawMime);
|
||||||
message.messageId = decodeHeaderValue(headers.value(QStringLiteral("message-id"))).remove('<').remove('>');
|
message.messageId = decodeHeaderValue(headers.value(QStringLiteral("message-id"))).remove('<').remove('>');
|
||||||
|
message.inReplyTo = decodeHeaderValue(headers.value(QStringLiteral("in-reply-to"))).remove('<').remove('>');
|
||||||
|
message.references = decodeHeaderValue(headers.value(QStringLiteral("references")));
|
||||||
message.subject = decodeHeaderValue(headers.value(QStringLiteral("subject")));
|
message.subject = decodeHeaderValue(headers.value(QStringLiteral("subject")));
|
||||||
message.from = decodeHeaderValue(headers.value(QStringLiteral("from")));
|
message.from = decodeHeaderValue(headers.value(QStringLiteral("from")));
|
||||||
message.to = decodeHeaderValue(headers.value(QStringLiteral("to")));
|
message.to = decodeHeaderValue(headers.value(QStringLiteral("to")));
|
||||||
|
|||||||
@@ -27,6 +27,8 @@ struct ParsedMimeMessage
|
|||||||
QString bcc;
|
QString bcc;
|
||||||
QDateTime date;
|
QDateTime date;
|
||||||
QString bodyHtml;
|
QString bodyHtml;
|
||||||
|
QString inReplyTo;
|
||||||
|
QString references;
|
||||||
QVector<ParsedMimeAttachment> attachments;
|
QVector<ParsedMimeAttachment> attachments;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user