From fb56001a527d68e078bbff697e97a136c591fd33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Kaan=20K=C3=B6se?= Date: Mon, 20 Oct 2025 18:27:02 +0200 Subject: [PATCH] Minimum download logic. --- Wino.Core/Synchronizers/GmailSynchronizer.cs | 206 +++++++++++++++++- Wino.Core/Synchronizers/ImapSynchronizer.cs | 1 + .../Synchronizers/OutlookSynchronizer.cs | 78 +++++-- Wino.Core/Synchronizers/WinoSynchronizer.cs | 7 + 4 files changed, 269 insertions(+), 23 deletions(-) diff --git a/Wino.Core/Synchronizers/GmailSynchronizer.cs b/Wino.Core/Synchronizers/GmailSynchronizer.cs index c0d0e851..aea0dd91 100644 --- a/Wino.Core/Synchronizers/GmailSynchronizer.cs +++ b/Wino.Core/Synchronizers/GmailSynchronizer.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Net.Http; using System.Threading; using System.Threading.Tasks; +using System.Web; using CommunityToolkit.Mvvm.Messaging; using Google; using Google.Apis.Calendar.v3.Data; @@ -51,7 +52,7 @@ public class GmailSynchronizer : WinoSynchronizer _folderDownloadCounts = new(); public GmailSynchronizer(MailAccount account, IGmailAuthenticator authenticator, @@ -185,6 +189,9 @@ public class GmailSynchronizer : WinoSynchronizer a.Messages != null).SelectMany(a => a.Messages).Select(a => a.Id)); + // For delta sync or messages beyond the MIME download limit, add to missing messages list + if (!isInitialSync) + { + // For delta sync, add all message IDs - they will be downloaded with MIME + missingMessageIds.AddRange(listChanges.Where(a => a.Messages != null).SelectMany(a => a.Messages).Select(a => a.Id)); + } + else + { + // For initial sync, we've already downloaded the first 50 messages per folder with MIME + // Any remaining messages from the listChanges that weren't downloaded need metadata-only download + var allInitialSyncMessages = listChanges.Where(a => a.Messages != null).SelectMany(a => a.Messages).Select(a => a.Id).ToList(); + var totalDownloadedCount = _folderDownloadCounts.Values.Sum(); + + // Skip the messages that were already downloaded with MIME and add the rest for metadata-only download + if (allInitialSyncMessages.Count > totalDownloadedCount) + { + var remainingMessages = allInitialSyncMessages.Skip(totalDownloadedCount).ToList(); + missingMessageIds.AddRange(remainingMessages); + + _logger.Information("Added {Count} remaining messages for metadata-only download", remainingMessages.Count); + } + } // Add missing message ids from delta changes. foreach (var historyResponse in deltaChanges) @@ -287,10 +333,13 @@ public class GmailSynchronizer : WinoSynchronizer + /// Handles after each single message download with minimal metadata only (no MIME). + /// This involves adding the Gmail message into Wino database with minimal properties. + /// + /// Gmail message + /// Request error + /// Message ID being downloaded + /// Cancellation token + private async Task HandleSingleItemDownloadedCallbackMinimalAsync(Message message, + RequestError error, + string downloadingMessageId, + CancellationToken cancellationToken = default) + { + try + { + await ProcessGmailRequestErrorAsync(error, null); + } + catch (OutOfMemoryException) + { + _logger.Warning("Gmail SDK got OutOfMemoryException due to bug in the SDK"); + } + catch (SynchronizerEntityNotFoundException) + { + _logger.Warning("Resource not found for {DownloadingMessageId}", downloadingMessageId); + } + catch (SynchronizerException synchronizerException) + { + _logger.Error("Gmail SDK returned error for {DownloadingMessageId}\n{SynchronizerException}", downloadingMessageId, synchronizerException); + } + + if (message == null) + { + _logger.Warning("Skipped GMail message download for {DownloadingMessageId}", downloadingMessageId); + + return null; + } + + // Use minimal package creation for metadata-only download + var mailPackage = await CreateNewMailPackagesMinimalAsync(message, cancellationToken); + + // If CreateNewMailPackagesMinimalAsync returns null it means local draft mapping is done. + // We don't need to insert anything else. + if (mailPackage == null) return message; + + foreach (var package in mailPackage) + { + await _gmailChangeProcessor.CreateMailAsync(Account.Id, package).ConfigureAwait(false); + } + + return message; + } + /// /// Handles after each single message download. /// This involves adding the Gmail message into Wino database. @@ -1355,6 +1469,84 @@ public class GmailSynchronizer : WinoSynchronizer + /// Creates a minimal MailCopy from Gmail message without downloading MIME content. + /// This includes only the information available from the Gmail message metadata. + /// + /// Gmail message + /// MailCopy with minimal properties + private static MailCopy CreateMinimalMailCopy(Message gmailMessage) + { + bool isUnread = gmailMessage.GetIsUnread(); + bool isFocused = gmailMessage.GetIsFocused(); + bool isFlagged = gmailMessage.GetIsFlagged(); + bool isDraft = gmailMessage.GetIsDraft(); + + return new MailCopy() + { + CreationDate = DateTime.UtcNow, // We don't have the exact date without MIME, use current time + Subject = HttpUtility.HtmlDecode(gmailMessage.Payload?.Headers?.FirstOrDefault(h => h.Name.Equals("Subject", StringComparison.OrdinalIgnoreCase))?.Value ?? ""), + FromName = HttpUtility.HtmlDecode(gmailMessage.Payload?.Headers?.FirstOrDefault(h => h.Name.Equals("From", StringComparison.OrdinalIgnoreCase))?.Value ?? ""), + FromAddress = ExtractEmailFromHeader(gmailMessage.Payload?.Headers?.FirstOrDefault(h => h.Name.Equals("From", StringComparison.OrdinalIgnoreCase))?.Value ?? ""), + PreviewText = HttpUtility.HtmlDecode(gmailMessage.Snippet), + ThreadId = gmailMessage.ThreadId, + Importance = MailImportance.Normal, // Default importance without MIME + Id = gmailMessage.Id, + IsDraft = isDraft, + HasAttachments = gmailMessage.Payload?.Parts?.Any(p => !string.IsNullOrEmpty(p.Filename)) ?? false, + IsRead = !isUnread, + IsFlagged = isFlagged, + IsFocused = isFocused, + InReplyTo = gmailMessage.Payload?.Headers?.FirstOrDefault(h => h.Name.Equals("In-Reply-To", StringComparison.OrdinalIgnoreCase))?.Value, + MessageId = gmailMessage.Payload?.Headers?.FirstOrDefault(h => h.Name.Equals("Message-Id", StringComparison.OrdinalIgnoreCase))?.Value, + References = gmailMessage.Payload?.Headers?.FirstOrDefault(h => h.Name.Equals("References", StringComparison.OrdinalIgnoreCase))?.Value, + FileId = Guid.NewGuid() + }; + } + + /// + /// Extracts email address from a header value like "Name " or "email@domain.com" + /// + private static string ExtractEmailFromHeader(string headerValue) + { + if (string.IsNullOrEmpty(headerValue)) return ""; + + var match = System.Text.RegularExpressions.Regex.Match(headerValue, @"<(.+?)>"); + if (match.Success) + return match.Groups[1].Value; + + // If no angle brackets, assume the whole value is the email + return headerValue.Trim(); + } + + /// + /// Creates new mail packages for the given message with minimal metadata only (no MIME download). + /// This is used for messages beyond the initial MIME download limit during synchronization. + /// + /// Gmail message to create package for. + /// Cancellation token + /// New mail package with minimal metadata only. + private async Task> CreateNewMailPackagesMinimalAsync(Message message, CancellationToken cancellationToken = default) + { + var packageList = new List(); + + // Create mail copy with minimal properties - no MIME download + var mailCopy = CreateMinimalMailCopy(message); + + // Since this is metadata-only download, we can't check for draft mapping via MIME headers + // Draft mapping will be handled during delta synchronization or when MIME is downloaded on-demand + + if (message.LabelIds is not null) + { + foreach (var labelId in message.LabelIds) + { + packageList.Add(new NewMailItemPackage(mailCopy, null, labelId)); + } + } + + return packageList; + } + /// /// Creates new mail packages for the given message. /// AssignedFolder is null since the LabelId is parsed out of the Message. diff --git a/Wino.Core/Synchronizers/ImapSynchronizer.cs b/Wino.Core/Synchronizers/ImapSynchronizer.cs index 66c57652..05e3682c 100644 --- a/Wino.Core/Synchronizers/ImapSynchronizer.cs +++ b/Wino.Core/Synchronizers/ImapSynchronizer.cs @@ -37,6 +37,7 @@ public class ImapSynchronizer : WinoSynchronizer 1000; public override uint InitialMessageDownloadCountPerFolder => 500; + public override int InitialSyncMimeDownloadCount => 50; #region Idle Implementation diff --git a/Wino.Core/Synchronizers/OutlookSynchronizer.cs b/Wino.Core/Synchronizers/OutlookSynchronizer.cs index bab5680e..17f7abbc 100644 --- a/Wino.Core/Synchronizers/OutlookSynchronizer.cs +++ b/Wino.Core/Synchronizers/OutlookSynchronizer.cs @@ -325,7 +325,9 @@ public class OutlookSynchronizer : WinoSynchronizer /// Downloads mails concurrently with semaphore control to limit concurrent downloads to 10. + /// This overload is used for initial sync where MIME messages are downloaded for the first 50 messages. /// private async Task DownloadMailsConcurrentlyAsync(List mailIds, MailItemFolder folder, List downloadedMessageIds, CancellationToken cancellationToken) { - var downloadTasks = mailIds.Select(async mailId => + await DownloadMailsConcurrentlyAsync(mailIds, folder, downloadedMessageIds, true, cancellationToken).ConfigureAwait(false); + } + + /// + /// Downloads mails concurrently with semaphore control to limit concurrent downloads to 10. + /// + private async Task DownloadMailsConcurrentlyAsync(List mailIds, MailItemFolder folder, List downloadedMessageIds, bool isInitialSync, CancellationToken cancellationToken) + { + var downloadTasks = mailIds.Select(async (mailId, index) => { await _concurrentDownloadSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false); try { - var downloaded = await DownloadSingleMailAsync(mailId, folder, cancellationToken).ConfigureAwait(false); + // Download MIME for the first 50 messages during initial sync only + bool shouldDownloadMime = isInitialSync && index < InitialSyncMimeDownloadCount; + var downloaded = await DownloadSingleMailAsync(mailId, folder, shouldDownloadMime, cancellationToken).ConfigureAwait(false); if (downloaded != null) { lock (downloadedMessageIds) @@ -417,7 +430,7 @@ public class OutlookSynchronizer : WinoSynchronizer /// Downloads a single mail by ID and creates it in the database. /// - private async Task DownloadSingleMailAsync(string mailId, MailItemFolder folder, CancellationToken cancellationToken) + private async Task DownloadSingleMailAsync(string mailId, MailItemFolder folder, bool downloadMime, CancellationToken cancellationToken) { try { @@ -435,27 +448,60 @@ public class OutlookSynchronizer : WinoSynchronizer public abstract uint InitialMessageDownloadCountPerFolder { get; } + /// + /// Number of MIME messages to download during initial synchronization per folder. + /// For the first messages in each folder during initial sync, both metadata and MIME content will be downloaded. + /// Subsequent messages will only have metadata downloaded, with MIME content fetched on-demand. + /// + public virtual int InitialSyncMimeDownloadCount => 50; + /// /// Creates a new Wino Mail Item package out of native message type with full Mime. ///