From 1c96c0ccbfd253875c0ccd38b4643858bd6dfb9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Kaan=20K=C3=B6se?= Date: Fri, 21 Jun 2024 04:27:17 +0200 Subject: [PATCH] Reworked IMAP folder synchronization logic. Gained 4x and fixed bunch of bugs around it. --- Wino.Core.Domain/Interfaces/IFolderService.cs | 6 + .../Processors/DefaultChangeProcessor.cs | 13 + .../Processors/ImapChangeProcessor.cs | 7 + Wino.Core/Synchronizers/ImapSynchronizer.cs | 299 ++++++++++++------ 4 files changed, 228 insertions(+), 97 deletions(-) diff --git a/Wino.Core.Domain/Interfaces/IFolderService.cs b/Wino.Core.Domain/Interfaces/IFolderService.cs index fab3fadc..3a1c597e 100644 --- a/Wino.Core.Domain/Interfaces/IFolderService.cs +++ b/Wino.Core.Domain/Interfaces/IFolderService.cs @@ -85,5 +85,11 @@ namespace Wino.Core.Domain.Interfaces Task UpdateFolderLastSyncDateAsync(Guid folderId); Task TestAsync(); + + /// + /// Updates the given folder. + /// + /// Folder to update. + Task UpdateFolderAsync(MailItemFolder folder); } } diff --git a/Wino.Core/Integration/Processors/DefaultChangeProcessor.cs b/Wino.Core/Integration/Processors/DefaultChangeProcessor.cs index bc5e0595..69709bd0 100644 --- a/Wino.Core/Integration/Processors/DefaultChangeProcessor.cs +++ b/Wino.Core/Integration/Processors/DefaultChangeProcessor.cs @@ -68,6 +68,19 @@ namespace Wino.Core.Integration.Processors /// /// Folder id to retrieve uIds for. Task> GetKnownUidsForFolderAsync(Guid folderId); + + /// + /// Returns the list of folders that are available for account. + /// + /// Account id to get folders for. + /// All folders. + Task> GetLocalIMAPFoldersAsync(Guid accountId); + + /// + /// Updates folder. + /// + /// Folder to update. + Task UpdateFolderAsync(MailItemFolder folder); } public class DefaultChangeProcessor(IDatabaseService databaseService, diff --git a/Wino.Core/Integration/Processors/ImapChangeProcessor.cs b/Wino.Core/Integration/Processors/ImapChangeProcessor.cs index f4bbdee2..d1c0d98c 100644 --- a/Wino.Core/Integration/Processors/ImapChangeProcessor.cs +++ b/Wino.Core/Integration/Processors/ImapChangeProcessor.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using Wino.Core.Domain.Entities; using Wino.Core.Domain.Interfaces; using Wino.Core.Services; @@ -18,5 +19,11 @@ namespace Wino.Core.Integration.Processors public Task> GetKnownUidsForFolderAsync(Guid folderId) => FolderService.GetKnownUidsForFolderAsync(folderId); + + public Task> GetLocalIMAPFoldersAsync(Guid accountId) + => FolderService.GetFoldersAsync(accountId); + + public Task UpdateFolderAsync(MailItemFolder folder) + => FolderService.UpdateFolderAsync(folder); } } diff --git a/Wino.Core/Synchronizers/ImapSynchronizer.cs b/Wino.Core/Synchronizers/ImapSynchronizer.cs index 4e8fc5eb..ea59cc16 100644 --- a/Wino.Core/Synchronizers/ImapSynchronizer.cs +++ b/Wino.Core/Synchronizers/ImapSynchronizer.cs @@ -499,122 +499,226 @@ namespace Wino.Core.Synchronizers } } - // TODO: This can be optimized by starting checking the local folders UidValidtyNext first. - // TODO: We need to determine deleted folders here. Also for that we need to start by local folders. - // TODO: UpdateFolderStructureAsync - private async Task SynchronizeFoldersAsync(CancellationToken cancellationToken = default) + /// + /// Assigns special folder type for the given local folder. + /// If server doesn't support special folders, we can't determine the type. MailKit will throw for GetFolder. + /// Default type is Other. + /// + /// ImapClient from the pool + /// Assigning remote folder. + /// Assigning local folder. + private void AssignSpecialFolderType(ImapClient executorClient, IMailFolder remoteFolder, MailItemFolder localFolder) { - // https://www.rfc-editor.org/rfc/rfc4549#section-1.1 + bool isSpecialFoldersSupported = executorClient.Capabilities.HasFlag(ImapCapabilities.SpecialUse) || executorClient.Capabilities.HasFlag(ImapCapabilities.XList); - var _synchronizationClient = await _clientPool.GetClientAsync(); - var nameSpace = _synchronizationClient.PersonalNamespaces[0]; - - var folders = (await _synchronizationClient.GetFoldersAsync(nameSpace, cancellationToken: cancellationToken)).ToList(); - - // Special folders - - bool isSpecialFoldersSupported = _synchronizationClient.Capabilities.HasFlag(ImapCapabilities.SpecialUse) || _synchronizationClient.Capabilities.HasFlag(ImapCapabilities.XList); - - // Inbox is always available. - IMailFolder inbox = _synchronizationClient.Inbox, drafts = null, junk = null, trash = null, sent = null, archive = null, important = null, starred = null; - - if (isSpecialFoldersSupported) + if (!isSpecialFoldersSupported) { - drafts = _synchronizationClient.GetFolder(SpecialFolder.Drafts); - junk = _synchronizationClient.GetFolder(SpecialFolder.Junk); - trash = _synchronizationClient.GetFolder(SpecialFolder.Trash); - sent = _synchronizationClient.GetFolder(SpecialFolder.Sent); - archive = _synchronizationClient.GetFolder(SpecialFolder.Archive); - important = _synchronizationClient.GetFolder(SpecialFolder.Important); - starred = _synchronizationClient.GetFolder(SpecialFolder.Flagged); + localFolder.SpecialFolderType = SpecialFolderType.Other; + return; } - var mailItemFolders = new List(); + if (remoteFolder == executorClient.Inbox) + localFolder.SpecialFolderType = SpecialFolderType.Inbox; + else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Drafts)) + localFolder.SpecialFolderType = SpecialFolderType.Draft; + else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Junk)) + localFolder.SpecialFolderType = SpecialFolderType.Junk; + else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Trash)) + localFolder.SpecialFolderType = SpecialFolderType.Deleted; + else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Sent)) + localFolder.SpecialFolderType = SpecialFolderType.Sent; + else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Archive)) + localFolder.SpecialFolderType = SpecialFolderType.Archive; + else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Important)) + localFolder.SpecialFolderType = SpecialFolderType.Important; + else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Flagged)) + localFolder.SpecialFolderType = SpecialFolderType.Starred; + } - // Sometimes Inbox is the root namespace. We need to check for that. - if (inbox != null && !folders.Contains(inbox)) - folders.Add(inbox); + /// + /// Whether the local folder should be updated with the remote folder. + /// + /// Remote folder + /// Local folder. + private bool ShouldUpdateFolder(IMailFolder remoteFolder, MailItemFolder localFolder) + { + return remoteFolder.Name != localFolder.FolderName; + } - foreach (var item in folders) + private async Task SynchronizeFoldersAsync(CancellationToken cancellationToken = default) + { + // https://www.rfc-editor.org/rfc/rfc4549#section-1.1 + + var localFolders = await _imapChangeProcessor.GetLocalIMAPFoldersAsync(Account.Id).ConfigureAwait(false); + + ImapClient executorClient = null; + + try { - // Namespaces are not needed as folders. - // Non-existed folders don't need to be synchronized. + executorClient = await _clientPool.GetClientAsync().ConfigureAwait(false); - if ((item.IsNamespace && !item.Attributes.HasFlag(FolderAttributes.Inbox)) || !item.Exists) - continue; + var remoteFolders = (await executorClient.GetFoldersAsync(executorClient.PersonalNamespaces[0], cancellationToken: cancellationToken)).ToList(); - // Try to synchronize all folders. + // 1. First check deleted folders. - try + // 1.a If local folder doesn't exists remotely, delete it. + // 1.b If local folder exists remotely, check if it is still a valid folder. If UidValidity is changed, delete it. + + List deletedFolders = new(); + + foreach (var localFolder in localFolders) { - var localFolder = item.GetLocalFolder(); + if (!localFolder.IsSynchronizationEnabled) continue; - if (item == inbox) - localFolder.SpecialFolderType = SpecialFolderType.Inbox; + IMailFolder remoteFolder = null; - if (isSpecialFoldersSupported) + try { - if (item == drafts) - localFolder.SpecialFolderType = SpecialFolderType.Draft; - else if (item == junk) - localFolder.SpecialFolderType = SpecialFolderType.Junk; - else if (item == trash) - localFolder.SpecialFolderType = SpecialFolderType.Deleted; - else if (item == sent) - localFolder.SpecialFolderType = SpecialFolderType.Sent; - else if (item == archive) - localFolder.SpecialFolderType = SpecialFolderType.Archive; - else if (item == important) - localFolder.SpecialFolderType = SpecialFolderType.Important; - else if (item == starred) - localFolder.SpecialFolderType = SpecialFolderType.Starred; + remoteFolder = remoteFolders.FirstOrDefault(a => a.FullName == localFolder.RemoteFolderId); + + bool shouldDeleteLocalFolder = false; + + // Check UidValidity of the remote folder if exists. + + if (remoteFolder != null) + { + // UidValidity won't be available until it's opened. + await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false); + + shouldDeleteLocalFolder = remoteFolder.UidValidity != localFolder.UidValidity; + } + else + { + // Remote folder doesn't exist. Delete it. + shouldDeleteLocalFolder = true; + } + + if (shouldDeleteLocalFolder) + { + await _imapChangeProcessor.DeleteFolderAsync(Account.Id, localFolder.RemoteFolderId).ConfigureAwait(false); + + deletedFolders.Add(localFolder); + } } - - if (localFolder.SpecialFolderType != SpecialFolderType.Other) + catch (Exception) { - localFolder.IsSystemFolder = true; - localFolder.IsSticky = true; - localFolder.IsSynchronizationEnabled = true; + throw; + } + finally + { + if (remoteFolder != null) + { + await remoteFolder.CloseAsync().ConfigureAwait(false); + } + } + } + + deletedFolders.ForEach(a => localFolders.Remove(a)); + + // 2. Get all remote folders and insert/update each of them. + + var nameSpace = executorClient.PersonalNamespaces[0]; + + // var remoteFolders = (await executorClient.GetFoldersAsync(nameSpace, cancellationToken: cancellationToken)).ToList(); + + IMailFolder inbox = executorClient.Inbox; + + // Sometimes Inbox is the root namespace. We need to check for that. + if (inbox != null && !remoteFolders.Contains(inbox)) + remoteFolders.Add(inbox); + + foreach (var remoteFolder in remoteFolders) + { + // Namespaces are not needed as folders. + // Non-existed folders don't need to be synchronized. + + if ((remoteFolder.IsNamespace && !remoteFolder.Attributes.HasFlag(FolderAttributes.Inbox)) || !remoteFolder.Exists) + continue; + + var existingLocalFolder = localFolders.FirstOrDefault(a => a.RemoteFolderId == remoteFolder.FullName); + + if (existingLocalFolder == null) + { + // Folder doesn't exist locally. Insert it. + + var localFolder = remoteFolder.GetLocalFolder(); + + // Check whether this is a special folder. + AssignSpecialFolderType(executorClient, remoteFolder, localFolder); + + bool isSystemFolder = localFolder.SpecialFolderType != SpecialFolderType.Other; + + localFolder.IsSynchronizationEnabled = isSystemFolder; + localFolder.IsSticky = isSystemFolder; + + // By default, all special folders update unread count in the UI except Trash. + localFolder.ShowUnreadCount = localFolder.SpecialFolderType != SpecialFolderType.Deleted || localFolder.SpecialFolderType != SpecialFolderType.Other; + + localFolder.MailAccountId = Account.Id; + + // Sometimes sub folders are parented under Inbox. + // Even though this makes sense in server level, in the client it sucks. + // That will make sub folders to be parented under Inbox in the client. + // Instead, we will mark them as non-parented folders. + // This is better. Model allows personalized folder structure anyways + // even though we don't have the page/control to adjust it. + + if (remoteFolder.ParentFolder == executorClient.Inbox) + localFolder.ParentRemoteFolderId = string.Empty; + + // Set UidValidity for cache expiration. + // Folder must be opened for this. + + await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken); + + localFolder.UidValidity = remoteFolder.UidValidity; + + await remoteFolder.CloseAsync(cancellationToken: cancellationToken); + + localFolders.Add(localFolder); } else { - localFolder.IsSynchronizationEnabled = false; + // Update existing folder. Right now we only update the name. + + // TODO: Moving servers around different parents. This is not supported right now. + // We will need more comphrensive folder update mechanism to support this. + + if (ShouldUpdateFolder(remoteFolder, existingLocalFolder)) + { + existingLocalFolder.FolderName = remoteFolder.Name; + } + else + { + // Remove it from the local folder list to skip additional folder updates. + localFolders.Remove(existingLocalFolder); + } } - - // By default, all special folders update unread count in the UI except Trash. - localFolder.ShowUnreadCount = localFolder.SpecialFolderType != SpecialFolderType.Deleted || localFolder.SpecialFolderType != SpecialFolderType.Other; - - localFolder.MailAccountId = Account.Id; - - // Sometimes sub folders are parented under Inbox. - // Even though this makes sense in server level, in the client it sucks. - // That will make sub folders to be parented under Inbox in the client. - // Instead, we will mark them as non-parented folders. - // This is better. Model allows personalized folder structure anyways - // even though we don't have the page/control to adjust it. - - if (item.ParentFolder == _synchronizationClient.Inbox) - localFolder.ParentRemoteFolderId = string.Empty; - - // Set UidValidity for cache expiration. - // Folder must be opened for this. - - await item.OpenAsync(FolderAccess.ReadOnly); - - localFolder.UidValidity = item.UidValidity; - - await item.CloseAsync(); - - mailItemFolders.Add(localFolder); } - catch (Exception ex) + + if (localFolders.Any()) { - Log.Error(ex, $"Folder with name '{item.Name}' failed to be synchronized."); + await _imapChangeProcessor.UpdateFolderStructureAsync(Account.Id, localFolders); + } + else + { + _logger.Information("No update is needed for imap folders."); } } + catch (Exception ex) + { + _logger.Error(ex, "Synchronizing IMAP folders failed."); - await _imapChangeProcessor.UpdateFolderStructureAsync(Account.Id, mailItemFolders); + throw; + } + finally + { + if (executorClient != null) + { + _clientPool.Release(executorClient); + } + } } private async Task> SynchronizeFolderInternalAsync(MailItemFolder folder, CancellationToken cancellationToken = default) @@ -798,17 +902,17 @@ namespace Wino.Core.Synchronizers // Fetch completely missing new items in the end. // Limit check. - //if (missingMailIds.Count > TAKE_COUNT) - //{ - // missingMailIds = new UniqueIdSet(missingMailIds.TakeLast(TAKE_COUNT)); - //} + if (missingMailIds.Count > InitialMessageDownloadCountPerFolder) + { + missingMailIds = new UniqueIdSet(missingMailIds.TakeLast((int)InitialMessageDownloadCountPerFolder)); + } // In case of the high input, we'll batch them by 50 to reflect changes quickly. var batchedMissingMailIds = missingMailIds.Batch(50).Select(a => new UniqueIdSet(a, SortOrder.Descending)); foreach (var batchMissingMailIds in batchedMissingMailIds) { - var summaries = await imapFolder.FetchAsync(batchMissingMailIds, mailSynchronizationFlags, cancellationToken); + var summaries = await imapFolder.FetchAsync(batchMissingMailIds, mailSynchronizationFlags, cancellationToken).ConfigureAwait(false); foreach (var summary in summaries) { @@ -823,7 +927,7 @@ namespace Wino.Core.Synchronizers foreach (var mailPackage in createdMailPackages) { - await _imapChangeProcessor.CreateMailAsync(Account.Id, mailPackage); + await _imapChangeProcessor.CreateMailAsync(Account.Id, mailPackage).ConfigureAwait(false); } } } @@ -831,7 +935,8 @@ namespace Wino.Core.Synchronizers if (folder.HighestModeSeq != (long)imapFolder.HighestModSeq) { folder.HighestModeSeq = (long)imapFolder.HighestModSeq; - await _imapChangeProcessor.InsertFolderAsync(folder); + + await _imapChangeProcessor.UpdateFolderAsync(folder).ConfigureAwait(false); } // Update last synchronization date for the folder.. @@ -842,7 +947,7 @@ namespace Wino.Core.Synchronizers } catch (FolderNotFoundException) { - await _imapChangeProcessor.DeleteFolderAsync(Account.Id, folder.RemoteFolderId); + await _imapChangeProcessor.DeleteFolderAsync(Account.Id, folder.RemoteFolderId).ConfigureAwait(false); return default; }