Refactored all synchronizers to deal with some of the chronic issues.
This commit is contained in:
Burak Kaan Köse
@@ -0,0 +1,473 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MailKit;
|
||||
using MailKit.Net.Imap;
|
||||
using MailKit.Search;
|
||||
using MoreLinq;
|
||||
using Serilog;
|
||||
using Wino.Core.Domain.Entities.Mail;
|
||||
using Wino.Core.Domain.Enums;
|
||||
using Wino.Core.Domain.Interfaces;
|
||||
using Wino.Core.Domain.Models.MailItem;
|
||||
using Wino.Core.Domain.Models.Synchronization;
|
||||
using Wino.Core.Integration;
|
||||
using Wino.Services.Extensions;
|
||||
using IMailService = Wino.Core.Domain.Interfaces.IMailService;
|
||||
|
||||
namespace Wino.Core.Synchronizers.ImapSync;
|
||||
|
||||
/// <summary>
|
||||
/// Unified IMAP synchronization strategy that automatically selects the best available method:
|
||||
/// 1. QRESYNC (RFC 5162) - Best: supports quick resync with vanished messages
|
||||
/// 2. CONDSTORE (RFC 4551) - Good: supports mod-seq based change tracking
|
||||
/// 3. UID-based - Fallback: basic UID comparison
|
||||
///
|
||||
/// This consolidates the previous QResyncSynchronizer, CondstoreSynchronizer, and UidBasedSynchronizer
|
||||
/// into a single, enterprise-grade implementation with proper error handling and partial failure support.
|
||||
/// </summary>
|
||||
public class UnifiedImapSynchronizer
|
||||
{
|
||||
private readonly ILogger _logger = Log.ForContext<UnifiedImapSynchronizer>();
|
||||
private readonly IFolderService _folderService;
|
||||
private readonly IMailService _mailService;
|
||||
private readonly IImapSynchronizerErrorHandlerFactory _errorHandlerFactory;
|
||||
|
||||
// Minimum summary items to Fetch for mail synchronization from IMAP.
|
||||
private readonly MessageSummaryItems MailSynchronizationFlags =
|
||||
MessageSummaryItems.Flags |
|
||||
MessageSummaryItems.UniqueId |
|
||||
MessageSummaryItems.ThreadId |
|
||||
MessageSummaryItems.EmailId |
|
||||
MessageSummaryItems.Headers |
|
||||
MessageSummaryItems.PreviewText |
|
||||
MessageSummaryItems.GMailThreadId |
|
||||
MessageSummaryItems.References |
|
||||
MessageSummaryItems.ModSeq;
|
||||
|
||||
public UnifiedImapSynchronizer(
|
||||
IFolderService folderService,
|
||||
IMailService mailService,
|
||||
IImapSynchronizerErrorHandlerFactory errorHandlerFactory)
|
||||
{
|
||||
_folderService = folderService;
|
||||
_mailService = mailService;
|
||||
_errorHandlerFactory = errorHandlerFactory;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines the best synchronization strategy based on server capabilities.
|
||||
/// </summary>
|
||||
public ImapSyncStrategy DetermineSyncStrategy(IImapClient client)
|
||||
{
|
||||
if (client is WinoImapClient winoClient &&
|
||||
client.Capabilities.HasFlag(ImapCapabilities.QuickResync) &&
|
||||
winoClient.IsQResyncEnabled)
|
||||
{
|
||||
return ImapSyncStrategy.QResync;
|
||||
}
|
||||
|
||||
if (client.Capabilities.HasFlag(ImapCapabilities.CondStore))
|
||||
{
|
||||
return ImapSyncStrategy.Condstore;
|
||||
}
|
||||
|
||||
return ImapSyncStrategy.UidBased;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Main synchronization entry point. Automatically selects the best strategy.
|
||||
/// </summary>
|
||||
public async Task<FolderSyncResult> SynchronizeFolderAsync(
|
||||
IImapClient client,
|
||||
MailItemFolder folder,
|
||||
IImapSynchronizer synchronizer,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var strategy = DetermineSyncStrategy(client);
|
||||
_logger.Debug("Using {Strategy} sync strategy for folder {FolderName}", strategy, folder.FolderName);
|
||||
|
||||
try
|
||||
{
|
||||
var downloadedIds = strategy switch
|
||||
{
|
||||
ImapSyncStrategy.QResync => await SynchronizeWithQResyncAsync(client, folder, synchronizer, cancellationToken),
|
||||
ImapSyncStrategy.Condstore => await SynchronizeWithCondstoreAsync(client, folder, synchronizer, cancellationToken),
|
||||
_ => await SynchronizeWithUidBasedAsync(client, folder, synchronizer, cancellationToken)
|
||||
};
|
||||
|
||||
return FolderSyncResult.Successful(folder.Id, folder.FolderName, downloadedIds.Count);
|
||||
}
|
||||
catch (FolderNotFoundException)
|
||||
{
|
||||
_logger.Warning("Folder {FolderName} not found on server, deleting locally", folder.FolderName);
|
||||
await _folderService.DeleteFolderAsync(folder.MailAccountId, folder.RemoteFolderId).ConfigureAwait(false);
|
||||
|
||||
return FolderSyncResult.Skipped(folder.Id, folder.FolderName, "Folder not found on server");
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
var errorContext = new SynchronizerErrorContext
|
||||
{
|
||||
ErrorMessage = ex.Message,
|
||||
Exception = ex,
|
||||
FolderId = folder.Id,
|
||||
FolderName = folder.FolderName,
|
||||
OperationType = "ImapFolderSync"
|
||||
};
|
||||
|
||||
var handled = await _errorHandlerFactory.HandleErrorAsync(errorContext).ConfigureAwait(false);
|
||||
|
||||
if (errorContext.CanContinueSync)
|
||||
{
|
||||
_logger.Warning(ex, "Folder {FolderName} sync failed with recoverable error", folder.FolderName);
|
||||
return FolderSyncResult.Failed(folder.Id, folder.FolderName, errorContext);
|
||||
}
|
||||
|
||||
_logger.Error(ex, "Folder {FolderName} sync failed with fatal error", folder.FolderName);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
#region QRESYNC Strategy
|
||||
|
||||
private async Task<List<string>> SynchronizeWithQResyncAsync(
|
||||
IImapClient client,
|
||||
MailItemFolder folder,
|
||||
IImapSynchronizer synchronizer,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var downloadedMessageIds = new List<string>();
|
||||
|
||||
if (client is not WinoImapClient winoClient)
|
||||
throw new InvalidOperationException("QRESYNC requires WinoImapClient");
|
||||
|
||||
IMailFolder remoteFolder = null;
|
||||
|
||||
try
|
||||
{
|
||||
remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var localHighestModSeq = (ulong)Math.Max(folder.HighestModeSeq, 1);
|
||||
var allUids = await _folderService.GetKnownUidsForFolderAsync(folder.Id);
|
||||
var allUniqueIds = allUids.Select(a => new UniqueId(a)).ToList();
|
||||
|
||||
// Subscribe to events before opening
|
||||
remoteFolder.MessagesVanished += (s, e) => HandleMessagesVanished(folder, e.UniqueIds);
|
||||
remoteFolder.MessageFlagsChanged += (s, e) => HandleMessageFlagsChanged(folder, e.UniqueId, e.Flags);
|
||||
|
||||
// Open with QRESYNC parameters
|
||||
await remoteFolder.OpenAsync(FolderAccess.ReadOnly, folder.UidValidity, localHighestModSeq, allUniqueIds, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Get changed UIDs
|
||||
var changedUids = await remoteFolder.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq), cancellationToken).ConfigureAwait(false);
|
||||
|
||||
downloadedMessageIds = await ProcessChangedUidsAsync(synchronizer, remoteFolder, folder, changedUids, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Update folder tracking
|
||||
folder.HighestModeSeq = unchecked((long)remoteFolder.HighestModSeq);
|
||||
folder.UidValidity = remoteFolder.UidValidity;
|
||||
|
||||
// Handle deletions
|
||||
await HandleDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await _folderService.UpdateFolderAsync(folder).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (remoteFolder?.IsOpen == true && !cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
await remoteFolder.CloseAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
return downloadedMessageIds;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region CONDSTORE Strategy
|
||||
|
||||
private async Task<List<string>> SynchronizeWithCondstoreAsync(
|
||||
IImapClient client,
|
||||
MailItemFolder folder,
|
||||
IImapSynchronizer synchronizer,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var downloadedMessageIds = new List<string>();
|
||||
IMailFolder remoteFolder = null;
|
||||
|
||||
try
|
||||
{
|
||||
remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false);
|
||||
await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var localHighestModSeq = (ulong)folder.HighestModeSeq;
|
||||
bool isInitialSync = localHighestModSeq == 0;
|
||||
|
||||
if (remoteFolder.HighestModSeq > localHighestModSeq || isInitialSync)
|
||||
{
|
||||
IList<UniqueId> changedUids;
|
||||
|
||||
// Use SORT if available for better ordering
|
||||
if (client.Capabilities.HasFlag(ImapCapabilities.Sort))
|
||||
{
|
||||
changedUids = await remoteFolder.SortAsync(
|
||||
SearchQuery.ChangedSince(Math.Max(localHighestModSeq, 1)),
|
||||
[OrderBy.ReverseDate],
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
changedUids = await remoteFolder.SearchAsync(
|
||||
SearchQuery.ChangedSince(Math.Max(localHighestModSeq, 1)),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// For initial sync, limit the number of messages
|
||||
if (isInitialSync)
|
||||
{
|
||||
changedUids = changedUids
|
||||
.OrderByDescending(a => a.Id)
|
||||
.Take((int)synchronizer.InitialMessageDownloadCountPerFolder)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
downloadedMessageIds = await ProcessChangedUidsAsync(synchronizer, remoteFolder, folder, changedUids, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
folder.HighestModeSeq = unchecked((long)remoteFolder.HighestModSeq);
|
||||
await _folderService.UpdateFolderAsync(folder).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await HandleDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (remoteFolder?.IsOpen == true && !cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
await remoteFolder.CloseAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
return downloadedMessageIds;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region UID-Based Strategy (Fallback)
|
||||
|
||||
private async Task<List<string>> SynchronizeWithUidBasedAsync(
|
||||
IImapClient client,
|
||||
MailItemFolder folder,
|
||||
IImapSynchronizer synchronizer,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var downloadedMessageIds = new List<string>();
|
||||
IMailFolder remoteFolder = null;
|
||||
|
||||
try
|
||||
{
|
||||
remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false);
|
||||
await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Get all remote UIDs and take the most recent ones
|
||||
var remoteUids = await remoteFolder.SearchAsync(SearchQuery.All, cancellationToken).ConfigureAwait(false);
|
||||
var limitedUids = remoteUids
|
||||
.OrderByDescending(a => a.Id)
|
||||
.Take((int)synchronizer.InitialMessageDownloadCountPerFolder)
|
||||
.ToList();
|
||||
|
||||
downloadedMessageIds = await ProcessChangedUidsAsync(synchronizer, remoteFolder, folder, limitedUids, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await HandleDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (remoteFolder?.IsOpen == true && !cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
await remoteFolder.CloseAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
return downloadedMessageIds;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Shared Processing Methods
|
||||
|
||||
private async Task<List<string>> ProcessChangedUidsAsync(
|
||||
IImapSynchronizer synchronizer,
|
||||
IMailFolder remoteFolder,
|
||||
MailItemFolder localFolder,
|
||||
IList<UniqueId> changedUids,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var downloadedMessageIds = new List<string>();
|
||||
|
||||
if (changedUids == null || changedUids.Count == 0)
|
||||
return downloadedMessageIds;
|
||||
|
||||
// Get existing mails to determine what's new vs. updated
|
||||
var existingMails = await _mailService.GetExistingMailsAsync(localFolder.Id, changedUids).ConfigureAwait(false);
|
||||
var existingMailUids = existingMails.Select(m => MailkitClientExtensions.ResolveUidStruct(m.Id)).ToArray();
|
||||
|
||||
var newMessageUids = changedUids.Except(existingMailUids).ToList();
|
||||
|
||||
// Update flags for existing mails
|
||||
if (existingMailUids.Any())
|
||||
{
|
||||
var existingFlagData = await remoteFolder.FetchAsync(existingMailUids, MessageSummaryItems.Flags | MessageSummaryItems.UniqueId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
foreach (var update in existingFlagData)
|
||||
{
|
||||
if (update.UniqueId == UniqueId.Invalid || update.Flags == null) continue;
|
||||
|
||||
var existingMail = existingMails.FirstOrDefault(m => MailkitClientExtensions.ResolveUidStruct(m.Id).Id == update.UniqueId.Id);
|
||||
if (existingMail != null)
|
||||
{
|
||||
await UpdateMailFlagsAsync(existingMail, update.Flags.Value).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Download new messages in batches
|
||||
var batches = newMessageUids.Batch(50);
|
||||
foreach (var batch in batches)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var batchList = batch.ToList();
|
||||
downloadedMessageIds.AddRange(batchList.Select(uid => MailkitClientExtensions.CreateUid(localFolder.Id, uid.Id)));
|
||||
|
||||
await DownloadMessagesAsync(synchronizer, remoteFolder, localFolder, new UniqueIdSet(batchList, SortOrder.Ascending), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return downloadedMessageIds;
|
||||
}
|
||||
|
||||
private async Task DownloadMessagesAsync(
|
||||
IImapSynchronizer synchronizer,
|
||||
IMailFolder folder,
|
||||
MailItemFolder localFolder,
|
||||
UniqueIdSet uniqueIdSet,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var summaries = await folder.FetchAsync(uniqueIdSet, MailSynchronizationFlags, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
foreach (var summary in summaries)
|
||||
{
|
||||
try
|
||||
{
|
||||
var mimeMessage = await folder.GetMessageAsync(summary.UniqueId, cancellationToken).ConfigureAwait(false);
|
||||
var creationPackage = new ImapMessageCreationPackage(summary, mimeMessage);
|
||||
var mailPackages = await synchronizer.CreateNewMailPackagesAsync(creationPackage, localFolder, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (mailPackages != null)
|
||||
{
|
||||
foreach (var package in mailPackages)
|
||||
{
|
||||
if (package != null)
|
||||
{
|
||||
await _mailService.CreateMailAsync(localFolder.MailAccountId, package).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Warning(ex, "Failed to download message {UniqueId} in folder {FolderName}", summary.UniqueId, localFolder.FolderName);
|
||||
// Continue with other messages
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HandleDeletedMessagesAsync(MailItemFolder localFolder, IMailFolder remoteFolder, CancellationToken cancellationToken)
|
||||
{
|
||||
var allLocalUids = (await _folderService.GetKnownUidsForFolderAsync(localFolder.Id)).Select(a => new UniqueId(a)).ToList();
|
||||
|
||||
if (allLocalUids.Count == 0) return;
|
||||
|
||||
var remoteAllUids = await remoteFolder.SearchAsync(SearchQuery.All, cancellationToken).ConfigureAwait(false);
|
||||
var deletedUids = allLocalUids.Except(remoteAllUids).ToList();
|
||||
|
||||
foreach (var deletedUid in deletedUids)
|
||||
{
|
||||
var localMailCopyId = MailkitClientExtensions.CreateUid(localFolder.Id, deletedUid.Id);
|
||||
await _mailService.DeleteMailAsync(localFolder.MailAccountId, localMailCopyId).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task UpdateMailFlagsAsync(MailCopy mailCopy, MessageFlags flags)
|
||||
{
|
||||
var isFlagged = MailkitClientExtensions.GetIsFlagged(flags);
|
||||
var isRead = MailkitClientExtensions.GetIsRead(flags);
|
||||
|
||||
if (isFlagged != mailCopy.IsFlagged)
|
||||
{
|
||||
await _mailService.ChangeFlagStatusAsync(mailCopy.Id, isFlagged).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (isRead != mailCopy.IsRead)
|
||||
{
|
||||
await _mailService.ChangeReadStatusAsync(mailCopy.Id, isRead).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleMessagesVanished(MailItemFolder folder, IList<UniqueId> uniqueIds)
|
||||
{
|
||||
// Fire and forget - these are event handlers
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
foreach (var uniqueId in uniqueIds)
|
||||
{
|
||||
var localMailCopyId = MailkitClientExtensions.CreateUid(folder.Id, uniqueId.Id);
|
||||
await _mailService.DeleteMailAsync(folder.MailAccountId, localMailCopyId).ConfigureAwait(false);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private void HandleMessageFlagsChanged(MailItemFolder folder, UniqueId? uniqueId, MessageFlags flags)
|
||||
{
|
||||
if (uniqueId == null) return;
|
||||
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
var localMailCopyId = MailkitClientExtensions.CreateUid(folder.Id, uniqueId.Value.Id);
|
||||
var isFlagged = MailkitClientExtensions.GetIsFlagged(flags);
|
||||
var isRead = MailkitClientExtensions.GetIsRead(flags);
|
||||
|
||||
await _mailService.ChangeReadStatusAsync(localMailCopyId, isRead).ConfigureAwait(false);
|
||||
await _mailService.ChangeFlagStatusAsync(localMailCopyId, isFlagged).ConfigureAwait(false);
|
||||
});
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// IMAP synchronization strategy enumeration.
|
||||
/// </summary>
|
||||
public enum ImapSyncStrategy
|
||||
{
|
||||
/// <summary>
|
||||
/// RFC 5162 Quick Resync - supports vanished messages and efficient delta sync.
|
||||
/// </summary>
|
||||
QResync,
|
||||
|
||||
/// <summary>
|
||||
/// RFC 4551 Conditional Store - supports mod-seq based change tracking.
|
||||
/// </summary>
|
||||
Condstore,
|
||||
|
||||
/// <summary>
|
||||
/// Basic UID-based synchronization - fallback for servers without advanced features.
|
||||
/// </summary>
|
||||
UidBased
|
||||
}
|
||||
Reference in New Issue
Block a user