Refactored impa synchronization.
This commit is contained in:
Burak Kaan Köse
@@ -1,132 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MailKit;
|
||||
using MailKit.Net.Imap;
|
||||
using MailKit.Search;
|
||||
using Wino.Core.Domain.Entities.Mail;
|
||||
using Wino.Core.Domain.Exceptions;
|
||||
using Wino.Core.Domain.Interfaces;
|
||||
using Wino.Core.Integration;
|
||||
using IMailService = Wino.Core.Domain.Interfaces.IMailService;
|
||||
|
||||
namespace Wino.Core.Synchronizers.ImapSync;
|
||||
|
||||
/// <summary>
|
||||
/// RFC 4551 CONDSTORE IMAP Synchronization strategy.
|
||||
/// </summary>
|
||||
internal class CondstoreSynchronizer : ImapSynchronizationStrategyBase
|
||||
{
|
||||
public CondstoreSynchronizer(IFolderService folderService, IMailService mailService) : base(folderService, mailService)
|
||||
{
|
||||
}
|
||||
|
||||
public async override Task<List<string>> HandleSynchronizationAsync(IImapClient client,
|
||||
MailItemFolder folder,
|
||||
IImapSynchronizer synchronizer,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (client is not WinoImapClient winoClient)
|
||||
throw new ArgumentException("Client must be of type WinoImapClient.", nameof(client));
|
||||
|
||||
if (!client.Capabilities.HasFlag(ImapCapabilities.CondStore))
|
||||
throw new ImapSynchronizerStrategyException("Server does not support CONDSTORE.");
|
||||
|
||||
IMailFolder remoteFolder = null;
|
||||
|
||||
var downloadedMessageIds = new List<string>();
|
||||
|
||||
Folder = folder;
|
||||
|
||||
try
|
||||
{
|
||||
remoteFolder = await winoClient.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var localHighestModSeq = (ulong)folder.HighestModeSeq;
|
||||
|
||||
bool isInitialSynchronization = localHighestModSeq == 0;
|
||||
|
||||
// There are some changes on new messages or flag changes.
|
||||
// Deletions are tracked separately because some servers do not increase
|
||||
// the MODSEQ value for deleted messages.
|
||||
if (remoteFolder.HighestModSeq > localHighestModSeq)
|
||||
{
|
||||
var changedUids = await GetChangedUidsAsync(client, remoteFolder, synchronizer, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Get locally exists mails for the returned UIDs.
|
||||
downloadedMessageIds = await HandleChangedUIdsAsync(synchronizer, remoteFolder, changedUids, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
folder.HighestModeSeq = unchecked((long)remoteFolder.HighestModSeq);
|
||||
|
||||
await FolderService.UpdateFolderAsync(folder).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await ManageUUIdBasedDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return downloadedMessageIds;
|
||||
}
|
||||
catch (FolderNotFoundException)
|
||||
{
|
||||
await FolderService.DeleteFolderAsync(folder.MailAccountId, folder.RemoteFolderId).ConfigureAwait(false);
|
||||
|
||||
return default;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
if (remoteFolder != null)
|
||||
{
|
||||
if (remoteFolder.IsOpen)
|
||||
{
|
||||
await remoteFolder.CloseAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal override async Task<IList<UniqueId>> GetChangedUidsAsync(IImapClient winoClient, IMailFolder remoteFolder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var localHighestModSeq = (ulong)Folder.HighestModeSeq;
|
||||
var remoteHighestModSeq = remoteFolder.HighestModSeq;
|
||||
|
||||
// Search for emails with a MODSEQ greater than the last known value.
|
||||
// Use SORT extension if server supports.
|
||||
|
||||
IList<UniqueId> changedUids = null;
|
||||
|
||||
if (winoClient.Capabilities.HasFlag(ImapCapabilities.Sort))
|
||||
{
|
||||
// Highest mod seq must be greater than 0 for SORT.
|
||||
changedUids = await remoteFolder.SortAsync(SearchQuery.ChangedSince(Math.Max(localHighestModSeq, 1)), [OrderBy.ReverseDate], cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
changedUids = await remoteFolder.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
changedUids = await remoteFolder.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq), cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// For initial synchronizations, take the first allowed number of items.
|
||||
// For consequtive synchronizations, take all the items. We don't want to miss any changes.
|
||||
// Smaller uid means newer message. For initial sync, we need start taking items from the top.
|
||||
|
||||
bool isInitialSynchronization = localHighestModSeq == 0;
|
||||
|
||||
if (isInitialSynchronization)
|
||||
{
|
||||
changedUids = changedUids.OrderByDescending(a => a.Id).Take((int)synchronizer.InitialMessageDownloadCountPerFolder).ToList();
|
||||
}
|
||||
|
||||
return changedUids;
|
||||
}
|
||||
}
|
||||
@@ -1,193 +0,0 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MailKit;
|
||||
using MailKit.Net.Imap;
|
||||
using MailKit.Search;
|
||||
using MoreLinq;
|
||||
using Serilog;
|
||||
using Wino.Core.Domain.Entities.Mail;
|
||||
using Wino.Core.Domain.Interfaces;
|
||||
using Wino.Core.Domain.Models.MailItem;
|
||||
using Wino.Services.Extensions;
|
||||
using IMailService = Wino.Core.Domain.Interfaces.IMailService;
|
||||
|
||||
namespace Wino.Core.Synchronizers.ImapSync;
|
||||
|
||||
public abstract class ImapSynchronizationStrategyBase : IImapSynchronizerStrategy
|
||||
{
|
||||
// Minimum summary items to Fetch for mail synchronization from IMAP.
|
||||
protected readonly MessageSummaryItems MailSynchronizationFlags =
|
||||
MessageSummaryItems.Flags |
|
||||
MessageSummaryItems.UniqueId |
|
||||
MessageSummaryItems.ThreadId |
|
||||
MessageSummaryItems.EmailId |
|
||||
MessageSummaryItems.Headers |
|
||||
MessageSummaryItems.PreviewText |
|
||||
MessageSummaryItems.GMailThreadId |
|
||||
MessageSummaryItems.References |
|
||||
MessageSummaryItems.ModSeq;
|
||||
|
||||
protected IFolderService FolderService { get; }
|
||||
protected IMailService MailService { get; }
|
||||
protected MailItemFolder Folder { get; set; }
|
||||
|
||||
protected ImapSynchronizationStrategyBase(IFolderService folderService, IMailService mailService)
|
||||
{
|
||||
FolderService = folderService;
|
||||
MailService = mailService;
|
||||
}
|
||||
|
||||
public abstract Task<List<string>> HandleSynchronizationAsync(IImapClient client, MailItemFolder folder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default);
|
||||
internal abstract Task<IList<UniqueId>> GetChangedUidsAsync(IImapClient client, IMailFolder remoteFolder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default);
|
||||
|
||||
protected async Task<List<string>> HandleChangedUIdsAsync(IImapSynchronizer synchronizer,
|
||||
IMailFolder remoteFolder,
|
||||
IList<UniqueId> changedUids,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
List<string> downloadedMessageIds = new();
|
||||
|
||||
var existingMails = await MailService.GetExistingMailsAsync(Folder.Id, changedUids).ConfigureAwait(false);
|
||||
var existingMailUids = existingMails.Select(m => MailkitClientExtensions.ResolveUidStruct(m.Id)).ToArray();
|
||||
|
||||
// These are the non-existing mails. They will be downloaded + processed.
|
||||
var newMessageIds = changedUids.Except(existingMailUids).ToList();
|
||||
var deletedMessageIds = existingMailUids.Except(changedUids).ToList();
|
||||
|
||||
// Fetch minimum data for the existing mails in one query.
|
||||
var existingFlagData = await remoteFolder.FetchAsync(existingMailUids, MessageSummaryItems.Flags | MessageSummaryItems.UniqueId).ConfigureAwait(false);
|
||||
|
||||
foreach (var update in existingFlagData)
|
||||
{
|
||||
if (update.UniqueId == UniqueId.Invalid)
|
||||
{
|
||||
Log.Warning($"Couldn't fetch UniqueId for the mail. FetchAsync failed.");
|
||||
continue;
|
||||
}
|
||||
|
||||
if (update.Flags == null)
|
||||
{
|
||||
Log.Warning($"Couldn't fetch flags for the mail with UID {update.UniqueId.Id}. FetchAsync failed.");
|
||||
continue;
|
||||
}
|
||||
|
||||
var existingMail = existingMails.FirstOrDefault(m => MailkitClientExtensions.ResolveUidStruct(m.Id).Id == update.UniqueId.Id);
|
||||
|
||||
if (existingMail == null)
|
||||
{
|
||||
Log.Warning($"Couldn't find the mail with UID {update.UniqueId.Id} in the local database. Flag update is ignored.");
|
||||
continue;
|
||||
}
|
||||
|
||||
await HandleMessageFlagsChangeAsync(existingMail, update.Flags.Value).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// Fetch the new mails in batch.
|
||||
|
||||
var batchedMessageIds = newMessageIds.Batch(50).ToList();
|
||||
// Create tasks for each batch.
|
||||
foreach (var group in batchedMessageIds)
|
||||
{
|
||||
downloadedMessageIds.AddRange(group.Select(a => MailkitClientExtensions.CreateUid(Folder.Id, a.Id)));
|
||||
|
||||
await DownloadMessagesAsync(synchronizer, remoteFolder, Folder, new UniqueIdSet(group, SortOrder.Ascending), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return downloadedMessageIds;
|
||||
}
|
||||
|
||||
protected async Task HandleMessageFlagsChangeAsync(UniqueId? uniqueId, MessageFlags flags)
|
||||
{
|
||||
if (Folder == null) return;
|
||||
if (uniqueId == null) return;
|
||||
|
||||
var localMailCopyId = MailkitClientExtensions.CreateUid(Folder.Id, uniqueId.Value.Id);
|
||||
|
||||
var isFlagged = MailkitClientExtensions.GetIsFlagged(flags);
|
||||
var isRead = MailkitClientExtensions.GetIsRead(flags);
|
||||
|
||||
await MailService.ChangeReadStatusAsync(localMailCopyId, isRead).ConfigureAwait(false);
|
||||
await MailService.ChangeFlagStatusAsync(localMailCopyId, isFlagged).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
protected async Task HandleMessageFlagsChangeAsync(MailCopy mailCopy, MessageFlags flags)
|
||||
{
|
||||
if (mailCopy == null) return;
|
||||
|
||||
var isFlagged = MailkitClientExtensions.GetIsFlagged(flags);
|
||||
var isRead = MailkitClientExtensions.GetIsRead(flags);
|
||||
|
||||
if (isFlagged != mailCopy.IsFlagged)
|
||||
{
|
||||
await MailService.ChangeFlagStatusAsync(mailCopy.Id, isFlagged).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (isRead != mailCopy.IsRead)
|
||||
{
|
||||
await MailService.ChangeReadStatusAsync(mailCopy.Id, isRead).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
protected async Task HandleMessageDeletedAsync(IList<UniqueId> uniqueIds)
|
||||
{
|
||||
if (Folder == null) return;
|
||||
if (uniqueIds == null || uniqueIds.Count == 0) return;
|
||||
|
||||
foreach (var uniqueId in uniqueIds)
|
||||
{
|
||||
var localMailCopyId = MailkitClientExtensions.CreateUid(Folder.Id, uniqueId.Id);
|
||||
|
||||
await MailService.DeleteMailAsync(Folder.MailAccountId, localMailCopyId).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
protected void OnMessagesVanished(object sender, MessagesVanishedEventArgs args)
|
||||
=> HandleMessageDeletedAsync(args.UniqueIds).ConfigureAwait(false);
|
||||
|
||||
protected void OnMessageFlagsChanged(object sender, MessageFlagsChangedEventArgs args)
|
||||
=> HandleMessageFlagsChangeAsync(args.UniqueId, args.Flags).ConfigureAwait(false);
|
||||
|
||||
protected async Task ManageUUIdBasedDeletedMessagesAsync(MailItemFolder localFolder, IMailFolder remoteFolder, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var allUids = (await FolderService.GetKnownUidsForFolderAsync(localFolder.Id)).Select(a => new UniqueId(a)).ToList();
|
||||
|
||||
if (allUids.Count > 0)
|
||||
{
|
||||
var remoteAllUids = await remoteFolder.SearchAsync(SearchQuery.All, cancellationToken);
|
||||
var deletedUids = allUids.Except(remoteAllUids).ToList();
|
||||
|
||||
await HandleMessageDeletedAsync(deletedUids).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DownloadMessagesAsync(IImapSynchronizer synchronizer,
|
||||
IMailFolder folder,
|
||||
MailItemFolder localFolder,
|
||||
UniqueIdSet uniqueIdSet,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var summaries = await folder.FetchAsync(uniqueIdSet, MailSynchronizationFlags, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
foreach (var summary in summaries)
|
||||
{
|
||||
var mimeMessage = await folder.GetMessageAsync(summary.UniqueId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var creationPackage = new ImapMessageCreationPackage(summary, mimeMessage);
|
||||
|
||||
var mailPackages = await synchronizer.CreateNewMailPackagesAsync(creationPackage, localFolder, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (mailPackages != null)
|
||||
{
|
||||
foreach (var package in mailPackages)
|
||||
{
|
||||
// Local draft is mapped. We don't need to create a new mail copy.
|
||||
if (package == null) continue;
|
||||
|
||||
await MailService.CreateMailAsync(localFolder.MailAccountId, package).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
using MailKit.Net.Imap;
|
||||
using Wino.Core.Domain.Interfaces;
|
||||
using Wino.Core.Integration;
|
||||
|
||||
namespace Wino.Core.Synchronizers.ImapSync;
|
||||
|
||||
internal class ImapSynchronizationStrategyProvider : IImapSynchronizationStrategyProvider
|
||||
{
|
||||
private readonly QResyncSynchronizer _qResyncSynchronizer;
|
||||
private readonly CondstoreSynchronizer _condstoreSynchronizer;
|
||||
private readonly UidBasedSynchronizer _uidBasedSynchronizer;
|
||||
|
||||
public ImapSynchronizationStrategyProvider(QResyncSynchronizer qResyncSynchronizer, CondstoreSynchronizer condstoreSynchronizer, UidBasedSynchronizer uidBasedSynchronizer)
|
||||
{
|
||||
_qResyncSynchronizer = qResyncSynchronizer;
|
||||
_condstoreSynchronizer = condstoreSynchronizer;
|
||||
_uidBasedSynchronizer = uidBasedSynchronizer;
|
||||
}
|
||||
|
||||
public IImapSynchronizerStrategy GetSynchronizationStrategy(IImapClient client)
|
||||
{
|
||||
if (client is not WinoImapClient winoImapClient)
|
||||
throw new System.ArgumentException("Client must be of type WinoImapClient.", nameof(client));
|
||||
|
||||
if (client.Capabilities.HasFlag(ImapCapabilities.QuickResync) && winoImapClient.IsQResyncEnabled) return _qResyncSynchronizer;
|
||||
if (client.Capabilities.HasFlag(ImapCapabilities.CondStore)) return _condstoreSynchronizer;
|
||||
|
||||
return _uidBasedSynchronizer;
|
||||
}
|
||||
}
|
||||
@@ -1,124 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MailKit;
|
||||
using MailKit.Net.Imap;
|
||||
using MailKit.Search;
|
||||
using Wino.Core.Domain.Entities.Mail;
|
||||
using Wino.Core.Domain.Exceptions;
|
||||
using Wino.Core.Domain.Interfaces;
|
||||
using Wino.Core.Integration;
|
||||
using IMailService = Wino.Core.Domain.Interfaces.IMailService;
|
||||
|
||||
namespace Wino.Core.Synchronizers.ImapSync;
|
||||
|
||||
/// <summary>
|
||||
/// RFC 5162 QRESYNC IMAP Synchronization strategy.
|
||||
/// </summary>
|
||||
internal class QResyncSynchronizer : ImapSynchronizationStrategyBase
|
||||
{
|
||||
public QResyncSynchronizer(IFolderService folderService, IMailService mailService) : base(folderService, mailService)
|
||||
{
|
||||
}
|
||||
|
||||
public override async Task<List<string>> HandleSynchronizationAsync(IImapClient client,
|
||||
MailItemFolder folder,
|
||||
IImapSynchronizer synchronizer,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var downloadedMessageIds = new List<string>();
|
||||
|
||||
if (client is not WinoImapClient winoClient)
|
||||
throw new ImapSynchronizerStrategyException("Client must be of type WinoImapClient.");
|
||||
|
||||
if (!client.Capabilities.HasFlag(ImapCapabilities.QuickResync))
|
||||
throw new ImapSynchronizerStrategyException("Server does not support QRESYNC.");
|
||||
|
||||
if (!winoClient.IsQResyncEnabled)
|
||||
throw new ImapSynchronizerStrategyException("QRESYNC is not enabled for WinoImapClient.");
|
||||
|
||||
// Ready to implement QRESYNC synchronization.
|
||||
|
||||
IMailFolder remoteFolder = null;
|
||||
|
||||
Folder = folder;
|
||||
|
||||
try
|
||||
{
|
||||
remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Check the Uid validity first.
|
||||
// If they don't match, clear all the local data and perform full-resync.
|
||||
|
||||
bool isCacheValid = remoteFolder.UidValidity == folder.UidValidity;
|
||||
|
||||
if (!isCacheValid)
|
||||
{
|
||||
// TODO: Remove all local data.
|
||||
}
|
||||
|
||||
// Perform QRESYNC synchronization.
|
||||
var localHighestModSeq = (ulong)folder.HighestModeSeq;
|
||||
// HIGHESTMODSEQ must be a positive integer, 0 is illegal.
|
||||
// It's harmless to set it to 1, as RFC-compliant server without mod-seq would ignore this parameter.
|
||||
if (localHighestModSeq == 0) localHighestModSeq = 1;
|
||||
|
||||
remoteFolder.MessagesVanished += OnMessagesVanished;
|
||||
remoteFolder.MessageFlagsChanged += OnMessageFlagsChanged;
|
||||
|
||||
var allUids = await FolderService.GetKnownUidsForFolderAsync(folder.Id);
|
||||
var allUniqueIds = allUids.Select(a => new UniqueId(a)).ToList();
|
||||
|
||||
await remoteFolder.OpenAsync(FolderAccess.ReadOnly, folder.UidValidity, localHighestModSeq, allUniqueIds).ConfigureAwait(false);
|
||||
|
||||
var changedUids = await GetChangedUidsAsync(client, remoteFolder, synchronizer, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
downloadedMessageIds = await HandleChangedUIdsAsync(synchronizer, remoteFolder, changedUids, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Update the local folder with the new highest mod-seq and validity.
|
||||
folder.HighestModeSeq = unchecked((long)remoteFolder.HighestModSeq);
|
||||
folder.UidValidity = remoteFolder.UidValidity;
|
||||
|
||||
await ManageUUIdBasedDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await FolderService.UpdateFolderAsync(folder).ConfigureAwait(false);
|
||||
}
|
||||
catch (FolderNotFoundException)
|
||||
{
|
||||
await FolderService.DeleteFolderAsync(folder.MailAccountId, folder.RemoteFolderId).ConfigureAwait(false);
|
||||
|
||||
return default;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
if (remoteFolder != null)
|
||||
{
|
||||
remoteFolder.MessagesVanished -= OnMessagesVanished;
|
||||
remoteFolder.MessageFlagsChanged -= OnMessageFlagsChanged;
|
||||
|
||||
if (remoteFolder.IsOpen)
|
||||
{
|
||||
await remoteFolder.CloseAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return downloadedMessageIds;
|
||||
}
|
||||
|
||||
internal override async Task<IList<UniqueId>> GetChangedUidsAsync(IImapClient client, IMailFolder remoteFolder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var localHighestModSeq = (ulong)Folder.HighestModeSeq;
|
||||
if (localHighestModSeq == 0) localHighestModSeq = 1;
|
||||
return await remoteFolder.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -1,80 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MailKit;
|
||||
using MailKit.Net.Imap;
|
||||
using MailKit.Search;
|
||||
using Wino.Core.Domain.Entities.Mail;
|
||||
using Wino.Core.Domain.Interfaces;
|
||||
using Wino.Core.Integration;
|
||||
|
||||
namespace Wino.Core.Synchronizers.ImapSync;
|
||||
|
||||
/// <summary>
|
||||
/// Uid based IMAP Synchronization strategy.
|
||||
/// </summary>
|
||||
internal class UidBasedSynchronizer : ImapSynchronizationStrategyBase
|
||||
{
|
||||
public UidBasedSynchronizer(IFolderService folderService, Domain.Interfaces.IMailService mailService) : base(folderService, mailService)
|
||||
{
|
||||
}
|
||||
|
||||
public override async Task<List<string>> HandleSynchronizationAsync(IImapClient client, MailItemFolder folder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (client is not WinoImapClient winoClient)
|
||||
throw new ArgumentException("Client must be of type WinoImapClient.", nameof(client));
|
||||
|
||||
Folder = folder;
|
||||
|
||||
var downloadedMessageIds = new List<string>();
|
||||
IMailFolder remoteFolder = null;
|
||||
|
||||
try
|
||||
{
|
||||
remoteFolder = await winoClient.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Fetch UIDs from the remote folder
|
||||
var remoteUids = await remoteFolder.SearchAsync(SearchQuery.All, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
remoteUids = remoteUids.OrderByDescending(a => a.Id).Take((int)synchronizer.InitialMessageDownloadCountPerFolder).ToList();
|
||||
|
||||
await HandleChangedUIdsAsync(synchronizer, remoteFolder, remoteUids, cancellationToken).ConfigureAwait(false);
|
||||
await ManageUUIdBasedDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (FolderNotFoundException)
|
||||
{
|
||||
await FolderService.DeleteFolderAsync(folder.MailAccountId, folder.RemoteFolderId).ConfigureAwait(false);
|
||||
|
||||
return default;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
if (remoteFolder != null)
|
||||
{
|
||||
if (remoteFolder.IsOpen)
|
||||
{
|
||||
await remoteFolder.CloseAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return downloadedMessageIds;
|
||||
}
|
||||
|
||||
internal override Task<IList<UniqueId>> GetChangedUidsAsync(IImapClient client, IMailFolder remoteFolder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
}
|
||||
@@ -23,29 +23,29 @@ namespace Wino.Core.Synchronizers.ImapSync;
|
||||
/// Unified IMAP synchronization strategy that automatically selects the best available method:
|
||||
/// 1. QRESYNC (RFC 5162) - Best: supports quick resync with vanished messages
|
||||
/// 2. CONDSTORE (RFC 4551) - Good: supports mod-seq based change tracking
|
||||
/// 3. UID-based - Fallback: basic UID comparison
|
||||
///
|
||||
/// This consolidates the previous QResyncSynchronizer, CondstoreSynchronizer, and UidBasedSynchronizer
|
||||
/// into a single, enterprise-grade implementation with proper error handling and partial failure support.
|
||||
/// 3. UID-based delta - Fallback: tracks UIDNEXT/high-water UID without sequence-number persistence
|
||||
/// </summary>
|
||||
public class UnifiedImapSynchronizer
|
||||
{
|
||||
private static readonly TimeSpan UidReconcileInterval = TimeSpan.FromHours(12);
|
||||
|
||||
private readonly ILogger _logger = Log.ForContext<UnifiedImapSynchronizer>();
|
||||
private readonly IFolderService _folderService;
|
||||
private readonly IMailService _mailService;
|
||||
private readonly IImapSynchronizerErrorHandlerFactory _errorHandlerFactory;
|
||||
|
||||
// Minimum summary items to Fetch for mail synchronization from IMAP.
|
||||
private readonly MessageSummaryItems MailSynchronizationFlags =
|
||||
// Metadata-first synchronization flags: no full MIME body download.
|
||||
private readonly MessageSummaryItems _mailSynchronizationFlags =
|
||||
MessageSummaryItems.Flags |
|
||||
MessageSummaryItems.UniqueId |
|
||||
MessageSummaryItems.ThreadId |
|
||||
MessageSummaryItems.EmailId |
|
||||
MessageSummaryItems.InternalDate |
|
||||
MessageSummaryItems.Envelope |
|
||||
MessageSummaryItems.Headers |
|
||||
MessageSummaryItems.PreviewText |
|
||||
MessageSummaryItems.GMailThreadId |
|
||||
MessageSummaryItems.References |
|
||||
MessageSummaryItems.ModSeq;
|
||||
MessageSummaryItems.ModSeq |
|
||||
MessageSummaryItems.BodyStructure;
|
||||
|
||||
public UnifiedImapSynchronizer(
|
||||
IFolderService folderService,
|
||||
@@ -58,21 +58,25 @@ public class UnifiedImapSynchronizer
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines the best synchronization strategy based on server capabilities.
|
||||
/// Determines the best synchronization strategy based on server capabilities and known quirks.
|
||||
/// </summary>
|
||||
public ImapSyncStrategy DetermineSyncStrategy(IImapClient client)
|
||||
public ImapSyncStrategy DetermineSyncStrategy(IImapClient client, string serverHost)
|
||||
{
|
||||
if (client is WinoImapClient winoClient &&
|
||||
client.Capabilities.HasFlag(ImapCapabilities.QuickResync) &&
|
||||
winoClient.IsQResyncEnabled)
|
||||
{
|
||||
return ImapSyncStrategy.QResync;
|
||||
}
|
||||
var capabilities = client.Capabilities;
|
||||
var isQResyncEnabled = client is WinoImapClient winoClient && winoClient.IsQResyncEnabled;
|
||||
|
||||
if (client.Capabilities.HasFlag(ImapCapabilities.CondStore))
|
||||
{
|
||||
return DetermineSyncStrategy(capabilities, isQResyncEnabled, serverHost);
|
||||
}
|
||||
|
||||
public ImapSyncStrategy DetermineSyncStrategy(ImapCapabilities capabilities, bool isQResyncEnabled, string serverHost = null)
|
||||
{
|
||||
var quirks = ImapServerQuirks.Resolve(serverHost);
|
||||
|
||||
if (!quirks.DisableQResync && capabilities.HasFlag(ImapCapabilities.QuickResync) && isQResyncEnabled)
|
||||
return ImapSyncStrategy.QResync;
|
||||
|
||||
if (!quirks.DisableCondstore && capabilities.HasFlag(ImapCapabilities.CondStore))
|
||||
return ImapSyncStrategy.Condstore;
|
||||
}
|
||||
|
||||
return ImapSyncStrategy.UidBased;
|
||||
}
|
||||
@@ -84,20 +88,48 @@ public class UnifiedImapSynchronizer
|
||||
IImapClient client,
|
||||
MailItemFolder folder,
|
||||
IImapSynchronizer synchronizer,
|
||||
string serverHost,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var strategy = DetermineSyncStrategy(client);
|
||||
var strategy = DetermineSyncStrategy(client, serverHost);
|
||||
_logger.Debug("Using {Strategy} sync strategy for folder {FolderName}", strategy, folder.FolderName);
|
||||
|
||||
var originalHighestModeSeq = folder.HighestModeSeq;
|
||||
var originalUidValidity = folder.UidValidity;
|
||||
var originalHighestKnownUid = folder.HighestKnownUid;
|
||||
var originalLastUidReconcileUtc = folder.LastUidReconcileUtc;
|
||||
|
||||
try
|
||||
{
|
||||
var downloadedIds = strategy switch
|
||||
{
|
||||
ImapSyncStrategy.QResync => await SynchronizeWithQResyncAsync(client, folder, synchronizer, cancellationToken),
|
||||
ImapSyncStrategy.Condstore => await SynchronizeWithCondstoreAsync(client, folder, synchronizer, cancellationToken),
|
||||
_ => await SynchronizeWithUidBasedAsync(client, folder, synchronizer, cancellationToken)
|
||||
ImapSyncStrategy.QResync => await SynchronizeWithQResyncAsync(client, folder, synchronizer, cancellationToken).ConfigureAwait(false),
|
||||
ImapSyncStrategy.Condstore => await SynchronizeWithCondstoreAsync(client, folder, synchronizer, cancellationToken).ConfigureAwait(false),
|
||||
_ => await SynchronizeWithUidDeltaAsync(client, folder, synchronizer, cancellationToken).ConfigureAwait(false)
|
||||
};
|
||||
|
||||
if (strategy == ImapSyncStrategy.QResync)
|
||||
{
|
||||
if (folder.HighestModeSeq != originalHighestModeSeq)
|
||||
{
|
||||
await _folderService.UpdateFolderHighestModeSeqAsync(folder.Id, folder.HighestModeSeq).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
bool requiresFullFolderUpdate =
|
||||
folder.UidValidity != originalUidValidity
|
||||
|| folder.HighestKnownUid != originalHighestKnownUid
|
||||
|| folder.LastUidReconcileUtc != originalLastUidReconcileUtc;
|
||||
|
||||
if (requiresFullFolderUpdate)
|
||||
{
|
||||
await _folderService.UpdateFolderAsync(folder).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
await _folderService.UpdateFolderAsync(folder).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return FolderSyncResult.Successful(folder.Id, folder.FolderName, downloadedIds.Count);
|
||||
}
|
||||
catch (FolderNotFoundException)
|
||||
@@ -122,7 +154,7 @@ public class UnifiedImapSynchronizer
|
||||
OperationType = "ImapFolderSync"
|
||||
};
|
||||
|
||||
var handled = await _errorHandlerFactory.HandleErrorAsync(errorContext).ConfigureAwait(false);
|
||||
_ = await _errorHandlerFactory.HandleErrorAsync(errorContext).ConfigureAwait(false);
|
||||
|
||||
if (errorContext.CanContinueSync)
|
||||
{
|
||||
@@ -135,7 +167,41 @@ public class UnifiedImapSynchronizer
|
||||
}
|
||||
}
|
||||
|
||||
#region QRESYNC Strategy
|
||||
/// <summary>
|
||||
/// Metadata-only message download helper used by IMAP online search.
|
||||
/// </summary>
|
||||
public async Task<List<string>> DownloadMessagesByUidsAsync(
|
||||
IImapClient client,
|
||||
IMailFolder remoteFolder,
|
||||
MailItemFolder localFolder,
|
||||
IList<UniqueId> uids,
|
||||
IImapSynchronizer synchronizer,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (uids == null || uids.Count == 0)
|
||||
return [];
|
||||
|
||||
if (!remoteFolder.IsOpen)
|
||||
await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var downloadedMessageIds = new List<string>();
|
||||
|
||||
foreach (var batch in uids.Distinct().OrderBy(a => a.Id).Batch(50))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var summaryBatch = await remoteFolder
|
||||
.FetchAsync(new UniqueIdSet(batch.ToList(), SortOrder.Ascending), _mailSynchronizationFlags, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
downloadedMessageIds.AddRange(await ProcessSummariesAsync(synchronizer, localFolder, summaryBatch, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
|
||||
UpdateHighestKnownUid(localFolder, remoteFolder, uids.Select(a => a.Id));
|
||||
return downloadedMessageIds;
|
||||
}
|
||||
|
||||
#region Strategy Implementations
|
||||
|
||||
private async Task<List<string>> SynchronizeWithQResyncAsync(
|
||||
IImapClient client,
|
||||
@@ -143,57 +209,87 @@ public class UnifiedImapSynchronizer
|
||||
IImapSynchronizer synchronizer,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (client is not WinoImapClient)
|
||||
throw new InvalidOperationException("QRESYNC requires WinoImapClient.");
|
||||
|
||||
var downloadedMessageIds = new List<string>();
|
||||
|
||||
if (client is not WinoImapClient winoClient)
|
||||
throw new InvalidOperationException("QRESYNC requires WinoImapClient");
|
||||
|
||||
IMailFolder remoteFolder = null;
|
||||
|
||||
var vanishedUids = new List<UniqueId>();
|
||||
var changedFlags = new Dictionary<uint, MessageFlags>();
|
||||
|
||||
void OnMessagesVanished(object sender, MessagesVanishedEventArgs args)
|
||||
{
|
||||
lock (vanishedUids)
|
||||
{
|
||||
vanishedUids.AddRange(args.UniqueIds);
|
||||
}
|
||||
}
|
||||
|
||||
void OnMessageFlagsChanged(object sender, MessageFlagsChangedEventArgs args)
|
||||
{
|
||||
if (args.UniqueId is not UniqueId uniqueId)
|
||||
return;
|
||||
|
||||
lock (changedFlags)
|
||||
{
|
||||
changedFlags[uniqueId.Id] = args.Flags;
|
||||
}
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Open once to validate UIDVALIDITY and reset local state if needed.
|
||||
await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false);
|
||||
await EnsureUidValidityStateAsync(folder, remoteFolder).ConfigureAwait(false);
|
||||
await remoteFolder.CloseAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var knownUids = await _folderService.GetKnownUidsForFolderAsync(folder.Id).ConfigureAwait(false);
|
||||
var knownUidStructs = knownUids.Select(a => new UniqueId(a)).ToList();
|
||||
var localHighestModSeq = (ulong)Math.Max(folder.HighestModeSeq, 1);
|
||||
var allUids = await _folderService.GetKnownUidsForFolderAsync(folder.Id);
|
||||
var allUniqueIds = allUids.Select(a => new UniqueId(a)).ToList();
|
||||
|
||||
// Subscribe to events before opening
|
||||
remoteFolder.MessagesVanished += (s, e) => HandleMessagesVanished(folder, e.UniqueIds);
|
||||
remoteFolder.MessageFlagsChanged += (s, e) => HandleMessageFlagsChanged(folder, e.UniqueId, e.Flags);
|
||||
remoteFolder.MessagesVanished += OnMessagesVanished;
|
||||
remoteFolder.MessageFlagsChanged += OnMessageFlagsChanged;
|
||||
|
||||
// Open with QRESYNC parameters
|
||||
await remoteFolder.OpenAsync(FolderAccess.ReadOnly, folder.UidValidity, localHighestModSeq, allUniqueIds, cancellationToken).ConfigureAwait(false);
|
||||
await remoteFolder
|
||||
.OpenAsync(FolderAccess.ReadOnly, folder.UidValidity, localHighestModSeq, knownUidStructs, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Get changed UIDs
|
||||
var changedUids = await remoteFolder.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq), cancellationToken).ConfigureAwait(false);
|
||||
var changedUids = await remoteFolder
|
||||
.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
downloadedMessageIds = await ProcessChangedUidsAsync(synchronizer, remoteFolder, folder, changedUids, cancellationToken).ConfigureAwait(false);
|
||||
downloadedMessageIds = await DownloadMessagesByUidsAsync(client, remoteFolder, folder, changedUids, synchronizer, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Update folder tracking
|
||||
folder.HighestModeSeq = unchecked((long)remoteFolder.HighestModSeq);
|
||||
folder.UidValidity = remoteFolder.UidValidity;
|
||||
|
||||
// Handle deletions
|
||||
await HandleDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false);
|
||||
await ApplyFlagChangesAsync(folder, changedFlags).ConfigureAwait(false);
|
||||
await ApplyDeletedUidsAsync(folder, vanishedUids).ConfigureAwait(false);
|
||||
|
||||
await _folderService.UpdateFolderAsync(folder).ConfigureAwait(false);
|
||||
if (ShouldRunUidReconcile(folder))
|
||||
{
|
||||
await ReconcileDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (remoteFolder?.IsOpen == true && !cancellationToken.IsCancellationRequested)
|
||||
if (remoteFolder != null)
|
||||
{
|
||||
await remoteFolder.CloseAsync().ConfigureAwait(false);
|
||||
remoteFolder.MessagesVanished -= OnMessagesVanished;
|
||||
remoteFolder.MessageFlagsChanged -= OnMessageFlagsChanged;
|
||||
|
||||
if (remoteFolder.IsOpen && !cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
await remoteFolder.CloseAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return downloadedMessageIds;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region CONDSTORE Strategy
|
||||
|
||||
private async Task<List<string>> SynchronizeWithCondstoreAsync(
|
||||
IImapClient client,
|
||||
MailItemFolder folder,
|
||||
@@ -208,29 +304,28 @@ public class UnifiedImapSynchronizer
|
||||
remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false);
|
||||
await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var localHighestModSeq = (ulong)folder.HighestModeSeq;
|
||||
bool isInitialSync = localHighestModSeq == 0;
|
||||
await EnsureUidValidityStateAsync(folder, remoteFolder).ConfigureAwait(false);
|
||||
|
||||
var localHighestModSeq = (ulong)Math.Max(folder.HighestModeSeq, 1);
|
||||
bool isInitialSync = folder.HighestModeSeq == 0;
|
||||
|
||||
if (remoteFolder.HighestModSeq > localHighestModSeq || isInitialSync)
|
||||
{
|
||||
IList<UniqueId> changedUids;
|
||||
|
||||
// Use SORT if available for better ordering
|
||||
if (client.Capabilities.HasFlag(ImapCapabilities.Sort))
|
||||
{
|
||||
changedUids = await remoteFolder.SortAsync(
|
||||
SearchQuery.ChangedSince(Math.Max(localHighestModSeq, 1)),
|
||||
[OrderBy.ReverseDate],
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
changedUids = await remoteFolder
|
||||
.SortAsync(SearchQuery.ChangedSince(localHighestModSeq), [OrderBy.ReverseDate], cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
changedUids = await remoteFolder.SearchAsync(
|
||||
SearchQuery.ChangedSince(Math.Max(localHighestModSeq, 1)),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
changedUids = await remoteFolder
|
||||
.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// For initial sync, limit the number of messages
|
||||
if (isInitialSync)
|
||||
{
|
||||
changedUids = changedUids
|
||||
@@ -239,13 +334,14 @@ public class UnifiedImapSynchronizer
|
||||
.ToList();
|
||||
}
|
||||
|
||||
downloadedMessageIds = await ProcessChangedUidsAsync(synchronizer, remoteFolder, folder, changedUids, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
downloadedMessageIds = await DownloadMessagesByUidsAsync(client, remoteFolder, folder, changedUids, synchronizer, cancellationToken).ConfigureAwait(false);
|
||||
folder.HighestModeSeq = unchecked((long)remoteFolder.HighestModSeq);
|
||||
await _folderService.UpdateFolderAsync(folder).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await HandleDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false);
|
||||
if (ShouldRunUidReconcile(folder))
|
||||
{
|
||||
await ReconcileDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -258,11 +354,7 @@ public class UnifiedImapSynchronizer
|
||||
return downloadedMessageIds;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region UID-Based Strategy (Fallback)
|
||||
|
||||
private async Task<List<string>> SynchronizeWithUidBasedAsync(
|
||||
private async Task<List<string>> SynchronizeWithUidDeltaAsync(
|
||||
IImapClient client,
|
||||
MailItemFolder folder,
|
||||
IImapSynchronizer synchronizer,
|
||||
@@ -276,16 +368,35 @@ public class UnifiedImapSynchronizer
|
||||
remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false);
|
||||
await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Get all remote UIDs and take the most recent ones
|
||||
var remoteUids = await remoteFolder.SearchAsync(SearchQuery.All, cancellationToken).ConfigureAwait(false);
|
||||
var limitedUids = remoteUids
|
||||
.OrderByDescending(a => a.Id)
|
||||
.Take((int)synchronizer.InitialMessageDownloadCountPerFolder)
|
||||
.ToList();
|
||||
await EnsureUidValidityStateAsync(folder, remoteFolder).ConfigureAwait(false);
|
||||
|
||||
downloadedMessageIds = await ProcessChangedUidsAsync(synchronizer, remoteFolder, folder, limitedUids, cancellationToken).ConfigureAwait(false);
|
||||
if (folder.HighestKnownUid == 0)
|
||||
{
|
||||
var remoteUids = await remoteFolder.SearchAsync(SearchQuery.All, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await HandleDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false);
|
||||
var initialUids = remoteUids
|
||||
.OrderByDescending(a => a.Id)
|
||||
.Take((int)synchronizer.InitialMessageDownloadCountPerFolder)
|
||||
.ToList();
|
||||
|
||||
downloadedMessageIds = await DownloadMessagesByUidsAsync(client, remoteFolder, folder, initialUids, synchronizer, cancellationToken).ConfigureAwait(false);
|
||||
UpdateHighestKnownUid(folder, remoteFolder, remoteUids.Select(a => a.Id));
|
||||
}
|
||||
else
|
||||
{
|
||||
var minUid = new UniqueId(folder.HighestKnownUid + 1);
|
||||
var deltaUids = await remoteFolder
|
||||
.SearchAsync(SearchQuery.Uids(new UniqueIdRange(minUid, UniqueId.MaxValue)), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
downloadedMessageIds = await DownloadMessagesByUidsAsync(client, remoteFolder, folder, deltaUids, synchronizer, cancellationToken).ConfigureAwait(false);
|
||||
UpdateHighestKnownUid(folder, remoteFolder, deltaUids.Select(a => a.Id));
|
||||
}
|
||||
|
||||
if (ShouldRunUidReconcile(folder))
|
||||
{
|
||||
await ReconcileDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -300,108 +411,89 @@ public class UnifiedImapSynchronizer
|
||||
|
||||
#endregion
|
||||
|
||||
#region Shared Processing Methods
|
||||
#region Shared Helpers
|
||||
|
||||
private async Task<List<string>> ProcessChangedUidsAsync(
|
||||
private async Task EnsureUidValidityStateAsync(MailItemFolder folder, IMailFolder remoteFolder)
|
||||
{
|
||||
if (folder.UidValidity != 0 && remoteFolder.UidValidity != folder.UidValidity)
|
||||
{
|
||||
_logger.Warning("UIDVALIDITY changed for folder {FolderName}. Resetting local folder state.", folder.FolderName);
|
||||
|
||||
var existingMails = await _mailService.GetMailsByFolderIdAsync(folder.Id).ConfigureAwait(false);
|
||||
foreach (var mail in existingMails)
|
||||
{
|
||||
await _mailService.DeleteMailAsync(folder.MailAccountId, mail.Id).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
folder.HighestKnownUid = 0;
|
||||
folder.HighestModeSeq = 0;
|
||||
folder.LastUidReconcileUtc = null;
|
||||
}
|
||||
|
||||
folder.UidValidity = remoteFolder.UidValidity;
|
||||
}
|
||||
|
||||
private async Task<List<string>> ProcessSummariesAsync(
|
||||
IImapSynchronizer synchronizer,
|
||||
IMailFolder remoteFolder,
|
||||
MailItemFolder localFolder,
|
||||
IList<UniqueId> changedUids,
|
||||
IList<IMessageSummary> summaries,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var downloadedMessageIds = new List<string>();
|
||||
|
||||
if (changedUids == null || changedUids.Count == 0)
|
||||
if (summaries == null || summaries.Count == 0)
|
||||
return downloadedMessageIds;
|
||||
|
||||
// Get existing mails to determine what's new vs. updated
|
||||
var existingMails = await _mailService.GetExistingMailsAsync(localFolder.Id, changedUids).ConfigureAwait(false);
|
||||
var existingMailUids = existingMails.Select(m => MailkitClientExtensions.ResolveUidStruct(m.Id)).ToArray();
|
||||
var uniqueIds = summaries
|
||||
.Where(s => s.UniqueId != UniqueId.Invalid)
|
||||
.Select(s => s.UniqueId)
|
||||
.ToList();
|
||||
|
||||
var newMessageUids = changedUids.Except(existingMailUids).ToList();
|
||||
if (uniqueIds.Count == 0)
|
||||
return downloadedMessageIds;
|
||||
|
||||
// Update flags for existing mails
|
||||
if (existingMailUids.Any())
|
||||
{
|
||||
var existingFlagData = await remoteFolder.FetchAsync(existingMailUids, MessageSummaryItems.Flags | MessageSummaryItems.UniqueId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
foreach (var update in existingFlagData)
|
||||
{
|
||||
if (update.UniqueId == UniqueId.Invalid || update.Flags == null) continue;
|
||||
|
||||
var existingMail = existingMails.FirstOrDefault(m => MailkitClientExtensions.ResolveUidStruct(m.Id).Id == update.UniqueId.Id);
|
||||
if (existingMail != null)
|
||||
{
|
||||
await UpdateMailFlagsAsync(existingMail, update.Flags.Value).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Download new messages in batches
|
||||
var batches = newMessageUids.Batch(50);
|
||||
foreach (var batch in batches)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var batchList = batch.ToList();
|
||||
downloadedMessageIds.AddRange(batchList.Select(uid => MailkitClientExtensions.CreateUid(localFolder.Id, uid.Id)));
|
||||
|
||||
await DownloadMessagesAsync(synchronizer, remoteFolder, localFolder, new UniqueIdSet(batchList, SortOrder.Ascending), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return downloadedMessageIds;
|
||||
}
|
||||
|
||||
private async Task DownloadMessagesAsync(
|
||||
IImapSynchronizer synchronizer,
|
||||
IMailFolder folder,
|
||||
MailItemFolder localFolder,
|
||||
UniqueIdSet uniqueIdSet,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var summaries = await folder.FetchAsync(uniqueIdSet, MailSynchronizationFlags, cancellationToken).ConfigureAwait(false);
|
||||
var existingMails = await _mailService.GetExistingMailsAsync(localFolder.Id, uniqueIds).ConfigureAwait(false);
|
||||
var existingByUid = existingMails
|
||||
.Select(m => (Uid: MailkitClientExtensions.ResolveUidStruct(m.Id), Mail: m))
|
||||
.ToDictionary(a => a.Uid.Id, a => a.Mail);
|
||||
|
||||
foreach (var summary in summaries)
|
||||
{
|
||||
try
|
||||
{
|
||||
var mimeMessage = await folder.GetMessageAsync(summary.UniqueId, cancellationToken).ConfigureAwait(false);
|
||||
var creationPackage = new ImapMessageCreationPackage(summary, mimeMessage);
|
||||
var mailPackages = await synchronizer.CreateNewMailPackagesAsync(creationPackage, localFolder, cancellationToken).ConfigureAwait(false);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (mailPackages != null)
|
||||
if (summary.UniqueId == UniqueId.Invalid)
|
||||
continue;
|
||||
|
||||
if (existingByUid.TryGetValue(summary.UniqueId.Id, out var existingMail))
|
||||
{
|
||||
if (summary.Flags != null)
|
||||
{
|
||||
foreach (var package in mailPackages)
|
||||
{
|
||||
if (package != null)
|
||||
{
|
||||
await _mailService.CreateMailAsync(localFolder.MailAccountId, package).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
await UpdateMailFlagsAsync(existingMail, summary.Flags.Value).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
var creationPackage = new ImapMessageCreationPackage(summary, mimeMessage: null);
|
||||
var mailPackages = await synchronizer.CreateNewMailPackagesAsync(creationPackage, localFolder, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (mailPackages == null)
|
||||
continue;
|
||||
|
||||
foreach (var package in mailPackages)
|
||||
{
|
||||
if (package == null)
|
||||
continue;
|
||||
|
||||
var inserted = await _mailService.CreateMailAsync(localFolder.MailAccountId, package).ConfigureAwait(false);
|
||||
if (inserted)
|
||||
{
|
||||
downloadedMessageIds.Add(package.Copy.Id);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Warning(ex, "Failed to download message {UniqueId} in folder {FolderName}", summary.UniqueId, localFolder.FolderName);
|
||||
// Continue with other messages
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HandleDeletedMessagesAsync(MailItemFolder localFolder, IMailFolder remoteFolder, CancellationToken cancellationToken)
|
||||
{
|
||||
var allLocalUids = (await _folderService.GetKnownUidsForFolderAsync(localFolder.Id)).Select(a => new UniqueId(a)).ToList();
|
||||
|
||||
if (allLocalUids.Count == 0) return;
|
||||
|
||||
var remoteAllUids = await remoteFolder.SearchAsync(SearchQuery.All, cancellationToken).ConfigureAwait(false);
|
||||
var deletedUids = allLocalUids.Except(remoteAllUids).ToList();
|
||||
|
||||
foreach (var deletedUid in deletedUids)
|
||||
{
|
||||
var localMailCopyId = MailkitClientExtensions.CreateUid(localFolder.Id, deletedUid.Id);
|
||||
await _mailService.DeleteMailAsync(localFolder.MailAccountId, localMailCopyId).ConfigureAwait(false);
|
||||
}
|
||||
return downloadedMessageIds;
|
||||
}
|
||||
|
||||
private async Task UpdateMailFlagsAsync(MailCopy mailCopy, MessageFlags flags)
|
||||
@@ -420,32 +512,95 @@ public class UnifiedImapSynchronizer
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleMessagesVanished(MailItemFolder folder, IList<UniqueId> uniqueIds)
|
||||
private async Task ApplyDeletedUidsAsync(MailItemFolder folder, IList<UniqueId> uniqueIds)
|
||||
{
|
||||
// Fire and forget - these are event handlers
|
||||
_ = Task.Run(async () =>
|
||||
if (uniqueIds == null || uniqueIds.Count == 0)
|
||||
return;
|
||||
|
||||
foreach (var uniqueId in uniqueIds.Distinct())
|
||||
{
|
||||
foreach (var uniqueId in uniqueIds)
|
||||
{
|
||||
var localMailCopyId = MailkitClientExtensions.CreateUid(folder.Id, uniqueId.Id);
|
||||
await _mailService.DeleteMailAsync(folder.MailAccountId, localMailCopyId).ConfigureAwait(false);
|
||||
}
|
||||
});
|
||||
var localMailCopyId = MailkitClientExtensions.CreateUid(folder.Id, uniqueId.Id);
|
||||
await _mailService.DeleteMailAsync(folder.MailAccountId, localMailCopyId).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleMessageFlagsChanged(MailItemFolder folder, UniqueId? uniqueId, MessageFlags flags)
|
||||
private async Task ApplyFlagChangesAsync(MailItemFolder folder, IDictionary<uint, MessageFlags> changedFlags)
|
||||
{
|
||||
if (uniqueId == null) return;
|
||||
if (changedFlags == null || changedFlags.Count == 0)
|
||||
return;
|
||||
|
||||
_ = Task.Run(async () =>
|
||||
foreach (var changed in changedFlags)
|
||||
{
|
||||
var localMailCopyId = MailkitClientExtensions.CreateUid(folder.Id, uniqueId.Value.Id);
|
||||
var isFlagged = MailkitClientExtensions.GetIsFlagged(flags);
|
||||
var isRead = MailkitClientExtensions.GetIsRead(flags);
|
||||
var localMailCopyId = MailkitClientExtensions.CreateUid(folder.Id, changed.Key);
|
||||
var isFlagged = MailkitClientExtensions.GetIsFlagged(changed.Value);
|
||||
var isRead = MailkitClientExtensions.GetIsRead(changed.Value);
|
||||
|
||||
await _mailService.ChangeReadStatusAsync(localMailCopyId, isRead).ConfigureAwait(false);
|
||||
await _mailService.ChangeFlagStatusAsync(localMailCopyId, isFlagged).ConfigureAwait(false);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private bool ShouldRunUidReconcile(MailItemFolder folder)
|
||||
{
|
||||
return ShouldRunUidReconcile(folder.LastUidReconcileUtc, DateTime.UtcNow, UidReconcileInterval);
|
||||
}
|
||||
|
||||
private async Task ReconcileDeletedMessagesAsync(MailItemFolder localFolder, IMailFolder remoteFolder, CancellationToken cancellationToken)
|
||||
{
|
||||
var allLocalUids = (await _folderService.GetKnownUidsForFolderAsync(localFolder.Id).ConfigureAwait(false))
|
||||
.Select(a => new UniqueId(a))
|
||||
.ToList();
|
||||
|
||||
if (allLocalUids.Count == 0)
|
||||
{
|
||||
localFolder.LastUidReconcileUtc = DateTime.UtcNow;
|
||||
return;
|
||||
}
|
||||
|
||||
var remoteAllUids = await remoteFolder.SearchAsync(SearchQuery.All, cancellationToken).ConfigureAwait(false);
|
||||
var deletedUids = allLocalUids.Except(remoteAllUids).ToList();
|
||||
|
||||
await ApplyDeletedUidsAsync(localFolder, deletedUids).ConfigureAwait(false);
|
||||
localFolder.LastUidReconcileUtc = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
private static void UpdateHighestKnownUid(MailItemFolder folder, IMailFolder remoteFolder, IEnumerable<uint> observedUids)
|
||||
{
|
||||
folder.HighestKnownUid = CalculateHighestKnownUid(folder.HighestKnownUid, remoteFolder?.UidNext, observedUids);
|
||||
}
|
||||
|
||||
public static bool ShouldRunUidReconcile(DateTime? lastUidReconcileUtc, DateTime utcNow, TimeSpan reconcileInterval)
|
||||
{
|
||||
if (!lastUidReconcileUtc.HasValue)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return utcNow - lastUidReconcileUtc.Value >= reconcileInterval;
|
||||
}
|
||||
|
||||
public static uint CalculateHighestKnownUid(uint currentHighestKnownUid, UniqueId? uidNext, IEnumerable<uint> observedUids)
|
||||
{
|
||||
uint observedMax = 0;
|
||||
|
||||
if (observedUids != null)
|
||||
{
|
||||
foreach (var uid in observedUids)
|
||||
{
|
||||
if (uid > observedMax)
|
||||
{
|
||||
observedMax = uid;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
uint uidNextBased = 0;
|
||||
if (uidNext.HasValue)
|
||||
{
|
||||
uidNextBased = uidNext.Value.Id > 0 ? uidNext.Value.Id - 1 : 0;
|
||||
}
|
||||
|
||||
return Math.Max(currentHighestKnownUid, Math.Max(observedMax, uidNextBased));
|
||||
}
|
||||
|
||||
#endregion
|
||||
@@ -467,7 +622,7 @@ public enum ImapSyncStrategy
|
||||
Condstore,
|
||||
|
||||
/// <summary>
|
||||
/// Basic UID-based synchronization - fallback for servers without advanced features.
|
||||
/// UID-based delta synchronization fallback.
|
||||
/// </summary>
|
||||
UidBased
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user