using System; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using MailKit; using MailKit.Net.Imap; using MailKit.Search; using MoreLinq; using Serilog; using Wino.Core.Domain.Entities.Mail; using Wino.Core.Domain.Enums; using Wino.Core.Domain.Interfaces; using Wino.Core.Domain.Models.MailItem; using Wino.Core.Domain.Models.Synchronization; using Wino.Core.Integration; using Wino.Services.Extensions; using IMailService = Wino.Core.Domain.Interfaces.IMailService; namespace Wino.Core.Synchronizers.ImapSync; /// /// Unified IMAP synchronization strategy that automatically selects the best available method: /// 1. QRESYNC (RFC 5162) - Best: supports quick resync with vanished messages /// 2. CONDSTORE (RFC 4551) - Good: supports mod-seq based change tracking /// 3. UID-based - Fallback: basic UID comparison /// /// This consolidates the previous QResyncSynchronizer, CondstoreSynchronizer, and UidBasedSynchronizer /// into a single, enterprise-grade implementation with proper error handling and partial failure support. /// public class UnifiedImapSynchronizer { private readonly ILogger _logger = Log.ForContext(); private readonly IFolderService _folderService; private readonly IMailService _mailService; private readonly IImapSynchronizerErrorHandlerFactory _errorHandlerFactory; // Minimum summary items to Fetch for mail synchronization from IMAP. private readonly MessageSummaryItems MailSynchronizationFlags = MessageSummaryItems.Flags | MessageSummaryItems.UniqueId | MessageSummaryItems.ThreadId | MessageSummaryItems.EmailId | MessageSummaryItems.Headers | MessageSummaryItems.PreviewText | MessageSummaryItems.GMailThreadId | MessageSummaryItems.References | MessageSummaryItems.ModSeq; public UnifiedImapSynchronizer( IFolderService folderService, IMailService mailService, IImapSynchronizerErrorHandlerFactory errorHandlerFactory) { _folderService = folderService; _mailService = mailService; _errorHandlerFactory = errorHandlerFactory; } /// /// Determines the best synchronization strategy based on server capabilities. /// public ImapSyncStrategy DetermineSyncStrategy(IImapClient client) { if (client is WinoImapClient winoClient && client.Capabilities.HasFlag(ImapCapabilities.QuickResync) && winoClient.IsQResyncEnabled) { return ImapSyncStrategy.QResync; } if (client.Capabilities.HasFlag(ImapCapabilities.CondStore)) { return ImapSyncStrategy.Condstore; } return ImapSyncStrategy.UidBased; } /// /// Main synchronization entry point. Automatically selects the best strategy. /// public async Task SynchronizeFolderAsync( IImapClient client, MailItemFolder folder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default) { var strategy = DetermineSyncStrategy(client); _logger.Debug("Using {Strategy} sync strategy for folder {FolderName}", strategy, folder.FolderName); try { var downloadedIds = strategy switch { ImapSyncStrategy.QResync => await SynchronizeWithQResyncAsync(client, folder, synchronizer, cancellationToken), ImapSyncStrategy.Condstore => await SynchronizeWithCondstoreAsync(client, folder, synchronizer, cancellationToken), _ => await SynchronizeWithUidBasedAsync(client, folder, synchronizer, cancellationToken) }; return FolderSyncResult.Successful(folder.Id, folder.FolderName, downloadedIds.Count); } catch (FolderNotFoundException) { _logger.Warning("Folder {FolderName} not found on server, deleting locally", folder.FolderName); await _folderService.DeleteFolderAsync(folder.MailAccountId, folder.RemoteFolderId).ConfigureAwait(false); return FolderSyncResult.Skipped(folder.Id, folder.FolderName, "Folder not found on server"); } catch (OperationCanceledException) { throw; } catch (Exception ex) { var errorContext = new SynchronizerErrorContext { ErrorMessage = ex.Message, Exception = ex, FolderId = folder.Id, FolderName = folder.FolderName, OperationType = "ImapFolderSync" }; var handled = await _errorHandlerFactory.HandleErrorAsync(errorContext).ConfigureAwait(false); if (errorContext.CanContinueSync) { _logger.Warning(ex, "Folder {FolderName} sync failed with recoverable error", folder.FolderName); return FolderSyncResult.Failed(folder.Id, folder.FolderName, errorContext); } _logger.Error(ex, "Folder {FolderName} sync failed with fatal error", folder.FolderName); throw; } } #region QRESYNC Strategy private async Task> SynchronizeWithQResyncAsync( IImapClient client, MailItemFolder folder, IImapSynchronizer synchronizer, CancellationToken cancellationToken) { var downloadedMessageIds = new List(); if (client is not WinoImapClient winoClient) throw new InvalidOperationException("QRESYNC requires WinoImapClient"); IMailFolder remoteFolder = null; try { remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false); var localHighestModSeq = (ulong)Math.Max(folder.HighestModeSeq, 1); var allUids = await _folderService.GetKnownUidsForFolderAsync(folder.Id); var allUniqueIds = allUids.Select(a => new UniqueId(a)).ToList(); // Subscribe to events before opening remoteFolder.MessagesVanished += (s, e) => HandleMessagesVanished(folder, e.UniqueIds); remoteFolder.MessageFlagsChanged += (s, e) => HandleMessageFlagsChanged(folder, e.UniqueId, e.Flags); // Open with QRESYNC parameters await remoteFolder.OpenAsync(FolderAccess.ReadOnly, folder.UidValidity, localHighestModSeq, allUniqueIds, cancellationToken).ConfigureAwait(false); // Get changed UIDs var changedUids = await remoteFolder.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq), cancellationToken).ConfigureAwait(false); downloadedMessageIds = await ProcessChangedUidsAsync(synchronizer, remoteFolder, folder, changedUids, cancellationToken).ConfigureAwait(false); // Update folder tracking folder.HighestModeSeq = unchecked((long)remoteFolder.HighestModSeq); folder.UidValidity = remoteFolder.UidValidity; // Handle deletions await HandleDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false); await _folderService.UpdateFolderAsync(folder).ConfigureAwait(false); } finally { if (remoteFolder?.IsOpen == true && !cancellationToken.IsCancellationRequested) { await remoteFolder.CloseAsync().ConfigureAwait(false); } } return downloadedMessageIds; } #endregion #region CONDSTORE Strategy private async Task> SynchronizeWithCondstoreAsync( IImapClient client, MailItemFolder folder, IImapSynchronizer synchronizer, CancellationToken cancellationToken) { var downloadedMessageIds = new List(); IMailFolder remoteFolder = null; try { remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false); await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false); var localHighestModSeq = (ulong)folder.HighestModeSeq; bool isInitialSync = localHighestModSeq == 0; if (remoteFolder.HighestModSeq > localHighestModSeq || isInitialSync) { IList changedUids; // Use SORT if available for better ordering if (client.Capabilities.HasFlag(ImapCapabilities.Sort)) { changedUids = await remoteFolder.SortAsync( SearchQuery.ChangedSince(Math.Max(localHighestModSeq, 1)), [OrderBy.ReverseDate], cancellationToken).ConfigureAwait(false); } else { changedUids = await remoteFolder.SearchAsync( SearchQuery.ChangedSince(Math.Max(localHighestModSeq, 1)), cancellationToken).ConfigureAwait(false); } // For initial sync, limit the number of messages if (isInitialSync) { changedUids = changedUids .OrderByDescending(a => a.Id) .Take((int)synchronizer.InitialMessageDownloadCountPerFolder) .ToList(); } downloadedMessageIds = await ProcessChangedUidsAsync(synchronizer, remoteFolder, folder, changedUids, cancellationToken).ConfigureAwait(false); folder.HighestModeSeq = unchecked((long)remoteFolder.HighestModSeq); await _folderService.UpdateFolderAsync(folder).ConfigureAwait(false); } await HandleDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false); } finally { if (remoteFolder?.IsOpen == true && !cancellationToken.IsCancellationRequested) { await remoteFolder.CloseAsync(cancellationToken: cancellationToken).ConfigureAwait(false); } } return downloadedMessageIds; } #endregion #region UID-Based Strategy (Fallback) private async Task> SynchronizeWithUidBasedAsync( IImapClient client, MailItemFolder folder, IImapSynchronizer synchronizer, CancellationToken cancellationToken) { var downloadedMessageIds = new List(); IMailFolder remoteFolder = null; try { remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false); await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false); // Get all remote UIDs and take the most recent ones var remoteUids = await remoteFolder.SearchAsync(SearchQuery.All, cancellationToken).ConfigureAwait(false); var limitedUids = remoteUids .OrderByDescending(a => a.Id) .Take((int)synchronizer.InitialMessageDownloadCountPerFolder) .ToList(); downloadedMessageIds = await ProcessChangedUidsAsync(synchronizer, remoteFolder, folder, limitedUids, cancellationToken).ConfigureAwait(false); await HandleDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false); } finally { if (remoteFolder?.IsOpen == true && !cancellationToken.IsCancellationRequested) { await remoteFolder.CloseAsync(cancellationToken: cancellationToken).ConfigureAwait(false); } } return downloadedMessageIds; } #endregion #region Shared Processing Methods private async Task> ProcessChangedUidsAsync( IImapSynchronizer synchronizer, IMailFolder remoteFolder, MailItemFolder localFolder, IList changedUids, CancellationToken cancellationToken) { var downloadedMessageIds = new List(); if (changedUids == null || changedUids.Count == 0) return downloadedMessageIds; // Get existing mails to determine what's new vs. updated var existingMails = await _mailService.GetExistingMailsAsync(localFolder.Id, changedUids).ConfigureAwait(false); var existingMailUids = existingMails.Select(m => MailkitClientExtensions.ResolveUidStruct(m.Id)).ToArray(); var newMessageUids = changedUids.Except(existingMailUids).ToList(); // Update flags for existing mails if (existingMailUids.Any()) { var existingFlagData = await remoteFolder.FetchAsync(existingMailUids, MessageSummaryItems.Flags | MessageSummaryItems.UniqueId, cancellationToken).ConfigureAwait(false); foreach (var update in existingFlagData) { if (update.UniqueId == UniqueId.Invalid || update.Flags == null) continue; var existingMail = existingMails.FirstOrDefault(m => MailkitClientExtensions.ResolveUidStruct(m.Id).Id == update.UniqueId.Id); if (existingMail != null) { await UpdateMailFlagsAsync(existingMail, update.Flags.Value).ConfigureAwait(false); } } } // Download new messages in batches var batches = newMessageUids.Batch(50); foreach (var batch in batches) { cancellationToken.ThrowIfCancellationRequested(); var batchList = batch.ToList(); downloadedMessageIds.AddRange(batchList.Select(uid => MailkitClientExtensions.CreateUid(localFolder.Id, uid.Id))); await DownloadMessagesAsync(synchronizer, remoteFolder, localFolder, new UniqueIdSet(batchList, SortOrder.Ascending), cancellationToken).ConfigureAwait(false); } return downloadedMessageIds; } private async Task DownloadMessagesAsync( IImapSynchronizer synchronizer, IMailFolder folder, MailItemFolder localFolder, UniqueIdSet uniqueIdSet, CancellationToken cancellationToken) { var summaries = await folder.FetchAsync(uniqueIdSet, MailSynchronizationFlags, cancellationToken).ConfigureAwait(false); foreach (var summary in summaries) { try { var mimeMessage = await folder.GetMessageAsync(summary.UniqueId, cancellationToken).ConfigureAwait(false); var creationPackage = new ImapMessageCreationPackage(summary, mimeMessage); var mailPackages = await synchronizer.CreateNewMailPackagesAsync(creationPackage, localFolder, cancellationToken).ConfigureAwait(false); if (mailPackages != null) { foreach (var package in mailPackages) { if (package != null) { await _mailService.CreateMailAsync(localFolder.MailAccountId, package).ConfigureAwait(false); } } } } catch (Exception ex) { _logger.Warning(ex, "Failed to download message {UniqueId} in folder {FolderName}", summary.UniqueId, localFolder.FolderName); // Continue with other messages } } } private async Task HandleDeletedMessagesAsync(MailItemFolder localFolder, IMailFolder remoteFolder, CancellationToken cancellationToken) { var allLocalUids = (await _folderService.GetKnownUidsForFolderAsync(localFolder.Id)).Select(a => new UniqueId(a)).ToList(); if (allLocalUids.Count == 0) return; var remoteAllUids = await remoteFolder.SearchAsync(SearchQuery.All, cancellationToken).ConfigureAwait(false); var deletedUids = allLocalUids.Except(remoteAllUids).ToList(); foreach (var deletedUid in deletedUids) { var localMailCopyId = MailkitClientExtensions.CreateUid(localFolder.Id, deletedUid.Id); await _mailService.DeleteMailAsync(localFolder.MailAccountId, localMailCopyId).ConfigureAwait(false); } } private async Task UpdateMailFlagsAsync(MailCopy mailCopy, MessageFlags flags) { var isFlagged = MailkitClientExtensions.GetIsFlagged(flags); var isRead = MailkitClientExtensions.GetIsRead(flags); if (isFlagged != mailCopy.IsFlagged) { await _mailService.ChangeFlagStatusAsync(mailCopy.Id, isFlagged).ConfigureAwait(false); } if (isRead != mailCopy.IsRead) { await _mailService.ChangeReadStatusAsync(mailCopy.Id, isRead).ConfigureAwait(false); } } private void HandleMessagesVanished(MailItemFolder folder, IList uniqueIds) { // Fire and forget - these are event handlers _ = Task.Run(async () => { foreach (var uniqueId in uniqueIds) { var localMailCopyId = MailkitClientExtensions.CreateUid(folder.Id, uniqueId.Id); await _mailService.DeleteMailAsync(folder.MailAccountId, localMailCopyId).ConfigureAwait(false); } }); } private void HandleMessageFlagsChanged(MailItemFolder folder, UniqueId? uniqueId, MessageFlags flags) { if (uniqueId == null) return; _ = Task.Run(async () => { var localMailCopyId = MailkitClientExtensions.CreateUid(folder.Id, uniqueId.Value.Id); var isFlagged = MailkitClientExtensions.GetIsFlagged(flags); var isRead = MailkitClientExtensions.GetIsRead(flags); await _mailService.ChangeReadStatusAsync(localMailCopyId, isRead).ConfigureAwait(false); await _mailService.ChangeFlagStatusAsync(localMailCopyId, isFlagged).ConfigureAwait(false); }); } #endregion } /// /// IMAP synchronization strategy enumeration. /// public enum ImapSyncStrategy { /// /// RFC 5162 Quick Resync - supports vanished messages and efficient delta sync. /// QResync, /// /// RFC 4551 Conditional Store - supports mod-seq based change tracking. /// Condstore, /// /// Basic UID-based synchronization - fallback for servers without advanced features. /// UidBased }