IMAP Improvements (#558)

* Fixing an issue where scrollviewer overrides a part of template in mail list. Adjusted zoomed out header grid's corner radius.

* IDLE implementation, imap synchronization strategies basics and condstore synchronization.

* Adding iCloud and Yahoo as special IMAP handling scenario.

* iCloud special imap handling.

* Support for killing synchronizers.

* Update privacy policy url.

* Batching condstore downloads into 50, using SORT extension for searches if supported.

* Bumping some nugets. More on the imap synchronizers.

* Delegating idle synchronizations to server to post-sync operations.

* Update mailkit to resolve qresync bug with iCloud.

* Fixing remote highest mode seq checks for qresync and condstore synchronizers.

* Yahoo custom settings.

* Bump google sdk package.

* Fixing the build issue....

* NRE on canceled token accounts during setup.

* Server crash handlers.

* Remove ARM32. Upgrade server to .NET 9.

* Fix icons for yahoo and apple.

* Fixed an issue where disabled folders causing an exception on forced sync.

* Remove smtp encoding constraint.

* Remove commented code.

* Fixing merge conflict

* Addressing double registrations for mailkit remote folder events in synchronizers.

* Making sure idle canceled result is not reported.

* Fixing custom imap server dialog opening.

* Fixing the issue with account creation making the previously selected account as selected as well.

* Fixing app close behavior and logging app close.
This commit is contained in:
Burak Kaan Köse
2025-02-15 12:53:32 +01:00
committed by GitHub
parent 30f1257983
commit ee9e41c5a7
108 changed files with 2092 additions and 1166 deletions

View File

@@ -0,0 +1,131 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MailKit;
using MailKit.Net.Imap;
using MailKit.Search;
using Wino.Core.Domain.Entities.Mail;
using Wino.Core.Domain.Exceptions;
using Wino.Core.Domain.Interfaces;
using Wino.Core.Integration;
using IMailService = Wino.Core.Domain.Interfaces.IMailService;
namespace Wino.Core.Synchronizers.ImapSync
{
/// <summary>
/// RFC 4551 CONDSTORE IMAP Synchronization strategy.
/// </summary>
internal class CondstoreSynchronizer : ImapSynchronizationStrategyBase
{
public CondstoreSynchronizer(IFolderService folderService, IMailService mailService) : base(folderService, mailService)
{
}
public async override Task<List<string>> HandleSynchronizationAsync(IImapClient client,
MailItemFolder folder,
IImapSynchronizer synchronizer,
CancellationToken cancellationToken = default)
{
if (client is not WinoImapClient winoClient)
throw new ArgumentException("Client must be of type WinoImapClient.", nameof(client));
if (!client.Capabilities.HasFlag(ImapCapabilities.CondStore))
throw new ImapSynchronizerStrategyException("Server does not support CONDSTORE.");
IMailFolder remoteFolder = null;
var downloadedMessageIds = new List<string>();
try
{
remoteFolder = await winoClient.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false);
await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false);
var localHighestModSeq = (ulong)folder.HighestModeSeq;
bool isInitialSynchronization = localHighestModSeq == 0;
// There are some changes on new messages or flag changes.
// Deletions are tracked separately because some servers do not increase
// the MODSEQ value for deleted messages.
if (remoteFolder.HighestModSeq > localHighestModSeq)
{
var changedUids = await GetChangedUidsAsync(client, remoteFolder, synchronizer, cancellationToken).ConfigureAwait(false);
// Get locally exists mails for the returned UIDs.
downloadedMessageIds = await HandleChangedUIdsAsync(synchronizer, remoteFolder, changedUids, cancellationToken).ConfigureAwait(false);
folder.HighestModeSeq = unchecked((long)remoteFolder.HighestModSeq);
await FolderService.UpdateFolderAsync(folder).ConfigureAwait(false);
}
await ManageUUIdBasedDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false);
return downloadedMessageIds;
}
catch (FolderNotFoundException)
{
await FolderService.DeleteFolderAsync(folder.MailAccountId, folder.RemoteFolderId).ConfigureAwait(false);
return default;
}
catch (Exception)
{
throw;
}
finally
{
if (!cancellationToken.IsCancellationRequested)
{
if (remoteFolder != null)
{
if (remoteFolder.IsOpen)
{
await remoteFolder.CloseAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
}
}
}
}
}
internal override async Task<IList<UniqueId>> GetChangedUidsAsync(IImapClient winoClient, IMailFolder remoteFolder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default)
{
var localHighestModSeq = (ulong)Folder.HighestModeSeq;
var remoteHighestModSeq = remoteFolder.HighestModSeq;
// Search for emails with a MODSEQ greater than the last known value.
// Use SORT extension if server supports.
IList<UniqueId> changedUids = null;
if (winoClient.Capabilities.HasFlag(ImapCapabilities.Sort))
{
// Highest mod seq must be greater than 0 for SORT.
changedUids = await remoteFolder.SortAsync(SearchQuery.ChangedSince(Math.Max(localHighestModSeq, 1)), [OrderBy.ReverseDate], cancellationToken).ConfigureAwait(false);
}
else
{
changedUids = await remoteFolder.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq), cancellationToken).ConfigureAwait(false);
}
changedUids = await remoteFolder.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq), cancellationToken).ConfigureAwait(false);
// For initial synchronizations, take the first allowed number of items.
// For consequtive synchronizations, take all the items. We don't want to miss any changes.
// Smaller uid means newer message. For initial sync, we need start taking items from the top.
bool isInitialSynchronization = localHighestModSeq == 0;
if (isInitialSynchronization)
{
changedUids = changedUids.OrderByDescending(a => a.Id).Take((int)synchronizer.InitialMessageDownloadCountPerFolder).ToList();
}
return changedUids;
}
}
}

View File

@@ -0,0 +1,185 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MailKit;
using MailKit.Net.Imap;
using MailKit.Search;
using MoreLinq;
using Serilog;
using Wino.Core.Domain.Entities.Mail;
using Wino.Core.Domain.Interfaces;
using Wino.Core.Domain.Models.MailItem;
using Wino.Services.Extensions;
using IMailService = Wino.Core.Domain.Interfaces.IMailService;
namespace Wino.Core.Synchronizers.ImapSync
{
public abstract class ImapSynchronizationStrategyBase : IImapSynchronizerStrategy
{
// Minimum summary items to Fetch for mail synchronization from IMAP.
protected readonly MessageSummaryItems MailSynchronizationFlags =
MessageSummaryItems.Flags |
MessageSummaryItems.UniqueId |
MessageSummaryItems.ThreadId |
MessageSummaryItems.EmailId |
MessageSummaryItems.Headers |
MessageSummaryItems.PreviewText |
MessageSummaryItems.GMailThreadId |
MessageSummaryItems.References |
MessageSummaryItems.ModSeq;
protected IFolderService FolderService { get; }
protected IMailService MailService { get; }
protected MailItemFolder Folder { get; set; }
protected ImapSynchronizationStrategyBase(IFolderService folderService, IMailService mailService)
{
FolderService = folderService;
MailService = mailService;
}
public abstract Task<List<string>> HandleSynchronizationAsync(IImapClient client, MailItemFolder folder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default);
internal abstract Task<IList<UniqueId>> GetChangedUidsAsync(IImapClient client, IMailFolder remoteFolder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default);
protected async Task<List<string>> HandleChangedUIdsAsync(IImapSynchronizer synchronizer, IMailFolder remoteFolder, IList<UniqueId> changedUids, CancellationToken cancellationToken)
{
List<string> downloadedMessageIds = new();
var existingMails = await MailService.GetExistingMailsAsync(Folder.Id, changedUids).ConfigureAwait(false);
var existingMailUids = existingMails.Select(m => MailkitClientExtensions.ResolveUidStruct(m.Id)).ToArray();
// These are the non-existing mails. They will be downloaded + processed.
var newMessageIds = changedUids.Except(existingMailUids).ToList();
var deletedMessageIds = existingMailUids.Except(changedUids).ToList();
// Fetch minimum data for the existing mails in one query.
var existingFlagData = await remoteFolder.FetchAsync(existingMailUids, MessageSummaryItems.Flags | MessageSummaryItems.UniqueId).ConfigureAwait(false);
foreach (var update in existingFlagData)
{
if (update.UniqueId == null)
{
Log.Warning($"Couldn't fetch UniqueId for the mail. FetchAsync failed.");
continue;
}
if (update.Flags == null)
{
Log.Warning($"Couldn't fetch flags for the mail with UID {update.UniqueId.Id}. FetchAsync failed.");
continue;
}
var existingMail = existingMails.FirstOrDefault(m => MailkitClientExtensions.ResolveUidStruct(m.Id).Id == update.UniqueId.Id);
if (existingMail == null)
{
Log.Warning($"Couldn't find the mail with UID {update.UniqueId.Id} in the local database. Flag update is ignored.");
continue;
}
await HandleMessageFlagsChangeAsync(existingMail, update.Flags.Value).ConfigureAwait(false);
}
// Fetch the new mails in batch.
var batchedMessageIds = newMessageIds.Batch(50);
foreach (var group in batchedMessageIds)
{
var summaries = await remoteFolder.FetchAsync(group, MailSynchronizationFlags, cancellationToken).ConfigureAwait(false);
foreach (var summary in summaries)
{
var mimeMessage = await remoteFolder.GetMessageAsync(summary.UniqueId, cancellationToken).ConfigureAwait(false);
var creationPackage = new ImapMessageCreationPackage(summary, mimeMessage);
var mailPackages = await synchronizer.CreateNewMailPackagesAsync(creationPackage, Folder, cancellationToken).ConfigureAwait(false);
if (mailPackages != null)
{
foreach (var package in mailPackages)
{
// Local draft is mapped. We don't need to create a new mail copy.
if (package == null) continue;
bool isCreatedNew = await MailService.CreateMailAsync(Folder.MailAccountId, package).ConfigureAwait(false);
// This is upsert. We are not interested in updated mails.
if (isCreatedNew) downloadedMessageIds.Add(package.Copy.Id);
}
}
}
}
return downloadedMessageIds;
}
protected async Task HandleMessageFlagsChangeAsync(UniqueId? uniqueId, MessageFlags flags)
{
if (Folder == null) return;
if (uniqueId == null) return;
var localMailCopyId = MailkitClientExtensions.CreateUid(Folder.Id, uniqueId.Value.Id);
var isFlagged = MailkitClientExtensions.GetIsFlagged(flags);
var isRead = MailkitClientExtensions.GetIsRead(flags);
await MailService.ChangeReadStatusAsync(localMailCopyId, isRead).ConfigureAwait(false);
await MailService.ChangeFlagStatusAsync(localMailCopyId, isFlagged).ConfigureAwait(false);
}
protected async Task HandleMessageFlagsChangeAsync(MailCopy mailCopy, MessageFlags flags)
{
if (mailCopy == null) return;
var isFlagged = MailkitClientExtensions.GetIsFlagged(flags);
var isRead = MailkitClientExtensions.GetIsRead(flags);
if (isFlagged != mailCopy.IsFlagged)
{
await MailService.ChangeFlagStatusAsync(mailCopy.Id, isFlagged).ConfigureAwait(false);
}
if (isRead != mailCopy.IsRead)
{
await MailService.ChangeReadStatusAsync(mailCopy.Id, isRead).ConfigureAwait(false);
}
}
protected async Task HandleMessageDeletedAsync(IList<UniqueId> uniqueIds)
{
if (Folder == null) return;
if (uniqueIds == null || uniqueIds.Count == 0) return;
foreach (var uniqueId in uniqueIds)
{
if (uniqueId == null) continue;
var localMailCopyId = MailkitClientExtensions.CreateUid(Folder.Id, uniqueId.Id);
await MailService.DeleteMailAsync(Folder.MailAccountId, localMailCopyId).ConfigureAwait(false);
}
}
protected void OnMessagesVanished(object sender, MessagesVanishedEventArgs args)
=> HandleMessageDeletedAsync(args.UniqueIds).ConfigureAwait(false);
protected void OnMessageFlagsChanged(object sender, MessageFlagsChangedEventArgs args)
=> HandleMessageFlagsChangeAsync(args.UniqueId, args.Flags).ConfigureAwait(false);
protected async Task ManageUUIdBasedDeletedMessagesAsync(MailItemFolder localFolder, IMailFolder remoteFolder, CancellationToken cancellationToken = default)
{
var allUids = (await FolderService.GetKnownUidsForFolderAsync(localFolder.Id)).Select(a => new UniqueId(a)).ToList();
if (allUids.Count > 0)
{
var remoteAllUids = await remoteFolder.SearchAsync(SearchQuery.All, cancellationToken);
var deletedUids = allUids.Except(remoteAllUids).ToList();
await HandleMessageDeletedAsync(deletedUids).ConfigureAwait(false);
}
}
}
}

View File

@@ -0,0 +1,31 @@
using MailKit.Net.Imap;
using Wino.Core.Domain.Interfaces;
using Wino.Core.Integration;
namespace Wino.Core.Synchronizers.ImapSync
{
internal class ImapSynchronizationStrategyProvider : IImapSynchronizationStrategyProvider
{
private readonly QResyncSynchronizer _qResyncSynchronizer;
private readonly CondstoreSynchronizer _condstoreSynchronizer;
private readonly UidBasedSynchronizer _uidBasedSynchronizer;
public ImapSynchronizationStrategyProvider(QResyncSynchronizer qResyncSynchronizer, CondstoreSynchronizer condstoreSynchronizer, UidBasedSynchronizer uidBasedSynchronizer)
{
_qResyncSynchronizer = qResyncSynchronizer;
_condstoreSynchronizer = condstoreSynchronizer;
_uidBasedSynchronizer = uidBasedSynchronizer;
}
public IImapSynchronizerStrategy GetSynchronizationStrategy(IImapClient client)
{
if (client is not WinoImapClient winoImapClient)
throw new System.ArgumentException("Client must be of type WinoImapClient.", nameof(client));
if (client.Capabilities.HasFlag(ImapCapabilities.QuickResync) && winoImapClient.IsQResyncEnabled) return _qResyncSynchronizer;
if (client.Capabilities.HasFlag(ImapCapabilities.CondStore)) return _condstoreSynchronizer;
return _uidBasedSynchronizer;
}
}
}

View File

@@ -0,0 +1,121 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MailKit;
using MailKit.Net.Imap;
using MailKit.Search;
using Wino.Core.Domain.Entities.Mail;
using Wino.Core.Domain.Exceptions;
using Wino.Core.Domain.Interfaces;
using Wino.Core.Integration;
using IMailService = Wino.Core.Domain.Interfaces.IMailService;
namespace Wino.Core.Synchronizers.ImapSync
{
/// <summary>
/// RFC 5162 QRESYNC IMAP Synchronization strategy.
/// </summary>
internal class QResyncSynchronizer : ImapSynchronizationStrategyBase
{
public QResyncSynchronizer(IFolderService folderService, IMailService mailService) : base(folderService, mailService)
{
}
public override async Task<List<string>> HandleSynchronizationAsync(IImapClient client,
MailItemFolder folder,
IImapSynchronizer synchronizer,
CancellationToken cancellationToken = default)
{
var downloadedMessageIds = new List<string>();
if (client is not WinoImapClient winoClient)
throw new ImapSynchronizerStrategyException("Client must be of type WinoImapClient.");
if (!client.Capabilities.HasFlag(ImapCapabilities.QuickResync))
throw new ImapSynchronizerStrategyException("Server does not support QRESYNC.");
if (!winoClient.IsQResyncEnabled)
throw new ImapSynchronizerStrategyException("QRESYNC is not enabled for WinoImapClient.");
// Ready to implement QRESYNC synchronization.
IMailFolder remoteFolder = null;
Folder = folder;
try
{
remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false);
// Check the Uid validity first.
// If they don't match, clear all the local data and perform full-resync.
bool isCacheValid = remoteFolder.UidValidity == folder.UidValidity;
if (!isCacheValid)
{
// TODO: Remove all local data.
}
// Perform QRESYNC synchronization.
var localHighestModSeq = (ulong)folder.HighestModeSeq;
remoteFolder.MessagesVanished += OnMessagesVanished;
remoteFolder.MessageFlagsChanged += OnMessageFlagsChanged;
var allUids = await FolderService.GetKnownUidsForFolderAsync(folder.Id);
var allUniqueIds = allUids.Select(a => new UniqueId(a)).ToList();
await remoteFolder.OpenAsync(FolderAccess.ReadOnly, folder.UidValidity, localHighestModSeq, allUniqueIds).ConfigureAwait(false);
var changedUids = await GetChangedUidsAsync(client, remoteFolder, synchronizer, cancellationToken).ConfigureAwait(false);
downloadedMessageIds = await HandleChangedUIdsAsync(synchronizer, remoteFolder, changedUids, cancellationToken).ConfigureAwait(false);
// Update the local folder with the new highest mod-seq and validity.
folder.HighestModeSeq = unchecked((long)remoteFolder.HighestModSeq);
folder.UidValidity = remoteFolder.UidValidity;
await ManageUUIdBasedDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false);
await FolderService.UpdateFolderAsync(folder).ConfigureAwait(false);
}
catch (FolderNotFoundException)
{
await FolderService.DeleteFolderAsync(folder.MailAccountId, folder.RemoteFolderId).ConfigureAwait(false);
return default;
}
catch (Exception)
{
throw;
}
finally
{
if (!cancellationToken.IsCancellationRequested)
{
if (remoteFolder != null)
{
remoteFolder.MessagesVanished -= OnMessagesVanished;
remoteFolder.MessageFlagsChanged -= OnMessageFlagsChanged;
if (remoteFolder.IsOpen)
{
await remoteFolder.CloseAsync();
}
}
}
}
return downloadedMessageIds;
}
internal override async Task<IList<UniqueId>> GetChangedUidsAsync(IImapClient client, IMailFolder remoteFolder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default)
{
var localHighestModSeq = (ulong)Folder.HighestModeSeq;
return await remoteFolder.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq), cancellationToken).ConfigureAwait(false);
}
}
}

View File

@@ -0,0 +1,81 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MailKit;
using MailKit.Net.Imap;
using MailKit.Search;
using Wino.Core.Domain.Entities.Mail;
using Wino.Core.Domain.Interfaces;
using Wino.Core.Integration;
namespace Wino.Core.Synchronizers.ImapSync
{
/// <summary>
/// Uid based IMAP Synchronization strategy.
/// </summary>
internal class UidBasedSynchronizer : ImapSynchronizationStrategyBase
{
public UidBasedSynchronizer(IFolderService folderService, Domain.Interfaces.IMailService mailService) : base(folderService, mailService)
{
}
public override async Task<List<string>> HandleSynchronizationAsync(IImapClient client, MailItemFolder folder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default)
{
if (client is not WinoImapClient winoClient)
throw new ArgumentException("Client must be of type WinoImapClient.", nameof(client));
Folder = folder;
var downloadedMessageIds = new List<string>();
IMailFolder remoteFolder = null;
try
{
remoteFolder = await winoClient.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false);
await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false);
// Fetch UIDs from the remote folder
var remoteUids = await remoteFolder.SearchAsync(SearchQuery.All, cancellationToken).ConfigureAwait(false);
remoteUids = remoteUids.OrderByDescending(a => a.Id).Take((int)synchronizer.InitialMessageDownloadCountPerFolder).ToList();
await HandleChangedUIdsAsync(synchronizer, remoteFolder, remoteUids, cancellationToken).ConfigureAwait(false);
await ManageUUIdBasedDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false);
}
catch (FolderNotFoundException)
{
await FolderService.DeleteFolderAsync(folder.MailAccountId, folder.RemoteFolderId).ConfigureAwait(false);
return default;
}
catch (Exception)
{
throw;
}
finally
{
if (!cancellationToken.IsCancellationRequested)
{
if (remoteFolder != null)
{
if (remoteFolder.IsOpen)
{
await remoteFolder.CloseAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
}
}
}
}
return downloadedMessageIds;
}
internal override Task<IList<UniqueId>> GetChangedUidsAsync(IImapClient client, IMailFolder remoteFolder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default)
{
throw new NotImplementedException();
}
}
}