From 8ef1faf8edfbe3d3524a34b06009d32e82c48154 Mon Sep 17 00:00:00 2001 From: Javier Date: Fri, 4 Sep 2026 01:35:13 +0200 Subject: [PATCH] feat: real conversation threading at ingest + migration script The IMAP parser only extracted Message-ID, so In-Reply-To / References were never persisted and threadId stayed empty -> conversations never grouped with real data (verified: 162/162 empty in user's DB). - ParsedMimeMessage: add inReplyTo + references fields - MimeStorage::parseMessage: extract In-Reply-To and References headers - ImapSynchronizer parseFetchResponseBytes: propagate inReplyTo/references, compute + persist threadId at ingest time (ThreadUtils::extractThreadId) - MimeStorage::storeMessage: propagate the same headers on MailItem - scripts/rethread.py: migrate existing DBs by re-reading .eml files (recovered In-Reply-To/References from raw MIME and recomputed threadId: 149/162 grouped, incl. Nacho x6, Bitar x4, Mariel x3, etc.) Verified against user's real DB copy. --- scripts/rethread.py | 96 ++++++++++++++++++++++++++ src/services/imap/imapsynchronizer.cpp | 18 +++++ src/services/mimestorage.cpp | 10 +++ src/services/mimestorage.h | 2 + 4 files changed, 126 insertions(+) create mode 100644 scripts/rethread.py diff --git a/scripts/rethread.py b/scripts/rethread.py new file mode 100644 index 0000000..bbd48c2 --- /dev/null +++ b/scripts/rethread.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Recupera In-Reply-To/References del .eml y recalcula threadId en Wino Mail. +Uso: python3 rethread2.py +""" +import sys, re, sqlite3, os, email + +bd = sys.argv[1] +mails_dir = sys.argv[2] +conn = sqlite3.connect(bd) +cur = conn.cursor() + +cur.execute("SELECT id, messageId FROM MailCopy") +rows = cur.fetchall() + +# indexar emls por message-id en nombre de archivo +eml_by_mid = {} +for root, _, files in os.walk(mails_dir): + for f in files: + if not f.endswith(".eml"): + continue + # nombre base contiene el message-id antes de .eml + base = f[:-4] + eml_by_mid[base] = os.path.join(root, f) + +def parse_headers(p): + try: + msg = email.message_from_bytes(open(p, "rb").read()) + except Exception: + return None + mid = msg.get("Message-ID", "") + irt = msg.get("In-Reply-To", "") + refs = msg.get("References", "") + return mid, irt, refs + +def thread_id(mid, irt, refs): + if refs: + m = re.search(r"<([^>]+)>", refs) + return m.group(1) if m else refs.strip("<>") + if irt: + m = re.search(r"<([^>]+)>", irt) + return m.group(1) if m else irt.strip("<>") + if mid: + m = re.search(r"<([^>]+)>", mid) + return m.group(1) if m else mid.strip("<>") + return "" + +upd_thread = 0 +upd_irt = 0 +upd_refs = 0 +not_found = 0 +no_headers = 0 + +for rid, mid in rows: + mid = (mid or "").strip() + # buscar eml: por nombre base == mid, o mid contenido + p = None + if mid in eml_by_mid: + p = eml_by_mid[mid] + if p is None: + # intentar coincidencia parcial (dominio distinto) + mid_nodot = mid.split("@")[0][:20] + for base, path in eml_by_mid.items(): + if mid_nodot and mid_nodot in base: + p = path + break + if p is None: + not_found += 1 + continue + + hdrs = parse_headers(p) + if hdrs is None: + not_found += 1 + continue + emid, eirt, erefs = hdrs + + tid = thread_id(emid or mid, eirt, erefs) + # guardar headers recuperados + cur.execute("UPDATE MailCopy SET inReplyTo=?, \"references\"=?, threadId=? WHERE id=?", + (eirt.strip() if eirt else "", erefs.strip() if erefs else "", tid or "", rid)) + if tid: upd_thread += 1 + if eirt.strip(): upd_irt += 1 + if erefs.strip(): upd_refs += 1 + +conn.commit() +print(f"correos: {len(rows)}") +print(f"eml no encontrados: {not_found}") +print(f"inReplyTo recuperado: {upd_irt}") +print(f"references recuperado: {upd_refs}") +print(f"threadId asignado: {upd_thread}") + +# agrupar en hilos +print("\n=== hilos detectados (threadId con >1 correo) ===") +cur.execute("SELECT threadId, COUNT(*), MAX(sender) FROM MailCopy WHERE threadId!='' GROUP BY threadId HAVING COUNT(*)>1 ORDER BY COUNT(*) DESC LIMIT 15") +for row in cur.fetchall(): + print(row) +conn.close() \ No newline at end of file diff --git a/src/services/imap/imapsynchronizer.cpp b/src/services/imap/imapsynchronizer.cpp index c58aa1a..3fc9423 100644 --- a/src/services/imap/imapsynchronizer.cpp +++ b/src/services/imap/imapsynchronizer.cpp @@ -9,6 +9,7 @@ #include "../../db/dao/folderdao.h" #include "../../db/dao/mailitemdao.h" #include "../../services/mimestorage.h" +#include "../../utils/threadutils.h" #include ImapSynchronizer::ImapSynchronizer(QObject* parent) @@ -526,6 +527,23 @@ QVector ImapSynchronizer::parseFetchResponseBytes(const QByteArray& re MailItem item; item.setUid(uidMatch.captured(1).toLongLong()); item.setMessageId(parsed.messageId); + item.setInReplyTo(parsed.inReplyTo); + // Persist references so conversation grouping can reconstruct threads. + if (!parsed.references.isEmpty()) { + QStringList refs; + QRegularExpression re("<([^>]+)>"); + auto it = re.globalMatch(parsed.references); + while (it.hasNext()) refs.append(it.next().captured(1)); + item.setReferences(refs); + } + // Compute and store the thread id at ingest time. + if (!item.threadId().isEmpty()) { + // already set + } else if (!item.inReplyTo().isEmpty() || !item.references().isEmpty() + || (!parsed.references.isEmpty())) { + item.setThreadId(ThreadUtils::extractThreadId( + item.messageId(), item.inReplyTo(), item.references())); + } item.setSubject(parsed.subject.isEmpty() ? QStringLiteral("(No Subject)") : parsed.subject); item.setSender(parsed.from); item.setRecipient(parsed.to); diff --git a/src/services/mimestorage.cpp b/src/services/mimestorage.cpp index 25df890..4bd74c2 100644 --- a/src/services/mimestorage.cpp +++ b/src/services/mimestorage.cpp @@ -85,6 +85,14 @@ bool MimeStorageService::storeMessage(const QString &accountId, const QString &f if (!parseMessage(rawMime, parsed)) return false; if (!parsed.messageId.isEmpty()) mail.setMessageId(parsed.messageId); + if (!parsed.inReplyTo.isEmpty()) mail.setInReplyTo(parsed.inReplyTo); + if (!parsed.references.isEmpty()) { + QStringList refs; + QRegularExpression re("<([^>]+)>"); + auto it = re.globalMatch(parsed.references); + while (it.hasNext()) refs.append(it.next().captured(1)); + mail.setReferences(refs); + } if (!parsed.subject.isEmpty()) mail.setSubject(parsed.subject); if (!parsed.from.isEmpty()) mail.setSender(parsed.from); if (!parsed.to.isEmpty()) { mail.setTo(parsed.to); mail.setRecipient(parsed.to); } @@ -339,6 +347,8 @@ bool MimeStorageService::parseMessage(const QByteArray &rawMime, ParsedMimeMessa }; const QMap headers = parseHeaders(rawMime); message.messageId = decodeHeaderValue(headers.value(QStringLiteral("message-id"))).remove('<').remove('>'); + message.inReplyTo = decodeHeaderValue(headers.value(QStringLiteral("in-reply-to"))).remove('<').remove('>'); + message.references = decodeHeaderValue(headers.value(QStringLiteral("references"))); message.subject = decodeHeaderValue(headers.value(QStringLiteral("subject"))); message.from = decodeHeaderValue(headers.value(QStringLiteral("from"))); message.to = decodeHeaderValue(headers.value(QStringLiteral("to"))); diff --git a/src/services/mimestorage.h b/src/services/mimestorage.h index 4fe3c4a..587c5e2 100644 --- a/src/services/mimestorage.h +++ b/src/services/mimestorage.h @@ -27,6 +27,8 @@ struct ParsedMimeMessage QString bcc; QDateTime date; QString bodyHtml; + QString inReplyTo; + QString references; QVector attachments; };