javier/Wino-Mail
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reworked IMAP folder synchronization logic. Gained 4x and fixed bunch of bugs around it.

Browse Source
This commit is contained in:
Burak Kaan Köse
2024-06-21 04:27:17 +02:00
parent cf8ad3d697
commit 1c96c0ccbf
4 changed files with 228 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Wino.Core.Domain/Interfaces/IFolderService.cs
View File

@@ -85,5 +85,11 @@ namespace Wino.Core.Domain.Interfaces
Task UpdateFolderLastSyncDateAsync(Guid folderId); Task UpdateFolderLastSyncDateAsync(Guid folderId);
Task TestAsync(); Task TestAsync();
/// <summary>
/// Updates the given folder.
/// </summary>
/// <param name="folder">Folder to update.</param>
Task UpdateFolderAsync(MailItemFolder folder);
} }
} }

13
Wino.Core/Integration/Processors/DefaultChangeProcessor.cs
View File

@@ -68,6 +68,19 @@ namespace Wino.Core.Integration.Processors
/// </summary> /// </summary>
/// <param name="folderId">Folder id to retrieve uIds for.</param> /// <param name="folderId">Folder id to retrieve uIds for.</param>
Task<IList<uint>> GetKnownUidsForFolderAsync(Guid folderId); Task<IList<uint>> GetKnownUidsForFolderAsync(Guid folderId);
/// <summary>
/// Returns the list of folders that are available for account.
/// </summary>
/// <param name="accountId">Account id to get folders for.</param>
/// <returns>All folders.</returns>
Task<List<MailItemFolder>> GetLocalIMAPFoldersAsync(Guid accountId);
/// <summary>
/// Updates folder.
/// </summary>
/// <param name="folder">Folder to update.</param>
Task UpdateFolderAsync(MailItemFolder folder);
} }
public class DefaultChangeProcessor(IDatabaseService databaseService, public class DefaultChangeProcessor(IDatabaseService databaseService,

7
Wino.Core/Integration/Processors/ImapChangeProcessor.cs
View File

@@ -1,6 +1,7 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Threading.Tasks; using System.Threading.Tasks;
using Wino.Core.Domain.Entities;
using Wino.Core.Domain.Interfaces; using Wino.Core.Domain.Interfaces;
using Wino.Core.Services; using Wino.Core.Services;
@@ -18,5 +19,11 @@ namespace Wino.Core.Integration.Processors
public Task<IList<uint>> GetKnownUidsForFolderAsync(Guid folderId) public Task<IList<uint>> GetKnownUidsForFolderAsync(Guid folderId)
=> FolderService.GetKnownUidsForFolderAsync(folderId); => FolderService.GetKnownUidsForFolderAsync(folderId);
public Task<List<MailItemFolder>> GetLocalIMAPFoldersAsync(Guid accountId)
=> FolderService.GetFoldersAsync(accountId);
public Task UpdateFolderAsync(MailItemFolder folder)
=> FolderService.UpdateFolderAsync(folder);
} }
} }

299
Wino.Core/Synchronizers/ImapSynchronizer.cs
View File

@@ -499,122 +499,226 @@ namespace Wino.Core.Synchronizers
} }
} }
// TODO: This can be optimized by starting checking the local folders UidValidtyNext first. /// <summary>
// TODO: We need to determine deleted folders here. Also for that we need to start by local folders. /// Assigns special folder type for the given local folder.
// TODO: UpdateFolderStructureAsync /// If server doesn't support special folders, we can't determine the type. MailKit will throw for GetFolder.
private async Task SynchronizeFoldersAsync(CancellationToken cancellationToken = default) /// Default type is Other.
/// </summary>
/// <param name="executorClient">ImapClient from the pool</param>
/// <param name="remoteFolder">Assigning remote folder.</param>
/// <param name="localFolder">Assigning local folder.</param>
private void AssignSpecialFolderType(ImapClient executorClient, IMailFolder remoteFolder, MailItemFolder localFolder)
{ {
// https://www.rfc-editor.org/rfc/rfc4549#section-1.1 bool isSpecialFoldersSupported = executorClient.Capabilities.HasFlag(ImapCapabilities.SpecialUse) || executorClient.Capabilities.HasFlag(ImapCapabilities.XList);
var _synchronizationClient = await _clientPool.GetClientAsync();
var nameSpace = _synchronizationClient.PersonalNamespaces[0]; if (!isSpecialFoldersSupported)
var folders = (await _synchronizationClient.GetFoldersAsync(nameSpace, cancellationToken: cancellationToken)).ToList();
// Special folders
bool isSpecialFoldersSupported = _synchronizationClient.Capabilities.HasFlag(ImapCapabilities.SpecialUse) || _synchronizationClient.Capabilities.HasFlag(ImapCapabilities.XList);
// Inbox is always available.
IMailFolder inbox = _synchronizationClient.Inbox, drafts = null, junk = null, trash = null, sent = null, archive = null, important = null, starred = null;
if (isSpecialFoldersSupported)
{ {
drafts = _synchronizationClient.GetFolder(SpecialFolder.Drafts); localFolder.SpecialFolderType = SpecialFolderType.Other;
junk = _synchronizationClient.GetFolder(SpecialFolder.Junk); return;
trash = _synchronizationClient.GetFolder(SpecialFolder.Trash);
sent = _synchronizationClient.GetFolder(SpecialFolder.Sent);
archive = _synchronizationClient.GetFolder(SpecialFolder.Archive);
important = _synchronizationClient.GetFolder(SpecialFolder.Important);
starred = _synchronizationClient.GetFolder(SpecialFolder.Flagged);
} }
var mailItemFolders = new List<MailItemFolder>(); if (remoteFolder == executorClient.Inbox)
localFolder.SpecialFolderType = SpecialFolderType.Inbox;
else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Drafts))
localFolder.SpecialFolderType = SpecialFolderType.Draft;
else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Junk))
localFolder.SpecialFolderType = SpecialFolderType.Junk;
else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Trash))
localFolder.SpecialFolderType = SpecialFolderType.Deleted;
else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Sent))
localFolder.SpecialFolderType = SpecialFolderType.Sent;
else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Archive))
localFolder.SpecialFolderType = SpecialFolderType.Archive;
else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Important))
localFolder.SpecialFolderType = SpecialFolderType.Important;
else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Flagged))
localFolder.SpecialFolderType = SpecialFolderType.Starred;
}
// Sometimes Inbox is the root namespace. We need to check for that. /// <summary>
if (inbox != null && !folders.Contains(inbox)) /// Whether the local folder should be updated with the remote folder.
folders.Add(inbox); /// </summary>
/// <param name="remoteFolder">Remote folder</param>
/// <param name="localFolder">Local folder.</param>
private bool ShouldUpdateFolder(IMailFolder remoteFolder, MailItemFolder localFolder)
{
return remoteFolder.Name != localFolder.FolderName;
}
foreach (var item in folders) private async Task SynchronizeFoldersAsync(CancellationToken cancellationToken = default)
{
// https://www.rfc-editor.org/rfc/rfc4549#section-1.1
var localFolders = await _imapChangeProcessor.GetLocalIMAPFoldersAsync(Account.Id).ConfigureAwait(false);
ImapClient executorClient = null;
try
{ {
// Namespaces are not needed as folders. executorClient = await _clientPool.GetClientAsync().ConfigureAwait(false);
// Non-existed folders don't need to be synchronized.
if ((item.IsNamespace && !item.Attributes.HasFlag(FolderAttributes.Inbox)) || !item.Exists) var remoteFolders = (await executorClient.GetFoldersAsync(executorClient.PersonalNamespaces[0], cancellationToken: cancellationToken)).ToList();
continue;
// Try to synchronize all folders. // 1. First check deleted folders.
try // 1.a If local folder doesn't exists remotely, delete it.
// 1.b If local folder exists remotely, check if it is still a valid folder. If UidValidity is changed, delete it.
List<MailItemFolder> deletedFolders = new();
foreach (var localFolder in localFolders)
{ {
var localFolder = item.GetLocalFolder(); if (!localFolder.IsSynchronizationEnabled) continue;
if (item == inbox) IMailFolder remoteFolder = null;
localFolder.SpecialFolderType = SpecialFolderType.Inbox;
if (isSpecialFoldersSupported) try
{ {
if (item == drafts) remoteFolder = remoteFolders.FirstOrDefault(a => a.FullName == localFolder.RemoteFolderId);
localFolder.SpecialFolderType = SpecialFolderType.Draft;
else if (item == junk) bool shouldDeleteLocalFolder = false;
localFolder.SpecialFolderType = SpecialFolderType.Junk;
else if (item == trash) // Check UidValidity of the remote folder if exists.
localFolder.SpecialFolderType = SpecialFolderType.Deleted;
else if (item == sent) if (remoteFolder != null)
localFolder.SpecialFolderType = SpecialFolderType.Sent; {
else if (item == archive) // UidValidity won't be available until it's opened.
localFolder.SpecialFolderType = SpecialFolderType.Archive; await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false);
else if (item == important)
localFolder.SpecialFolderType = SpecialFolderType.Important; shouldDeleteLocalFolder = remoteFolder.UidValidity != localFolder.UidValidity;
else if (item == starred) }
localFolder.SpecialFolderType = SpecialFolderType.Starred; else
{
// Remote folder doesn't exist. Delete it.
shouldDeleteLocalFolder = true;
}
if (shouldDeleteLocalFolder)
{
await _imapChangeProcessor.DeleteFolderAsync(Account.Id, localFolder.RemoteFolderId).ConfigureAwait(false);
deletedFolders.Add(localFolder);
}
} }
catch (Exception)
if (localFolder.SpecialFolderType != SpecialFolderType.Other)
{ {
localFolder.IsSystemFolder = true; throw;
localFolder.IsSticky = true; }
localFolder.IsSynchronizationEnabled = true; finally
{
if (remoteFolder != null)
{
await remoteFolder.CloseAsync().ConfigureAwait(false);
}
}
}
deletedFolders.ForEach(a => localFolders.Remove(a));
// 2. Get all remote folders and insert/update each of them.
var nameSpace = executorClient.PersonalNamespaces[0];
// var remoteFolders = (await executorClient.GetFoldersAsync(nameSpace, cancellationToken: cancellationToken)).ToList();
IMailFolder inbox = executorClient.Inbox;
// Sometimes Inbox is the root namespace. We need to check for that.
if (inbox != null && !remoteFolders.Contains(inbox))
remoteFolders.Add(inbox);
foreach (var remoteFolder in remoteFolders)
{
// Namespaces are not needed as folders.
// Non-existed folders don't need to be synchronized.
if ((remoteFolder.IsNamespace && !remoteFolder.Attributes.HasFlag(FolderAttributes.Inbox)) || !remoteFolder.Exists)
continue;
var existingLocalFolder = localFolders.FirstOrDefault(a => a.RemoteFolderId == remoteFolder.FullName);
if (existingLocalFolder == null)
{
// Folder doesn't exist locally. Insert it.
var localFolder = remoteFolder.GetLocalFolder();
// Check whether this is a special folder.
AssignSpecialFolderType(executorClient, remoteFolder, localFolder);
bool isSystemFolder = localFolder.SpecialFolderType != SpecialFolderType.Other;
localFolder.IsSynchronizationEnabled = isSystemFolder;
localFolder.IsSticky = isSystemFolder;
// By default, all special folders update unread count in the UI except Trash.
localFolder.ShowUnreadCount = localFolder.SpecialFolderType != SpecialFolderType.Deleted || localFolder.SpecialFolderType != SpecialFolderType.Other;
localFolder.MailAccountId = Account.Id;
// Sometimes sub folders are parented under Inbox.
// Even though this makes sense in server level, in the client it sucks.
// That will make sub folders to be parented under Inbox in the client.
// Instead, we will mark them as non-parented folders.
// This is better. Model allows personalized folder structure anyways
// even though we don't have the page/control to adjust it.
if (remoteFolder.ParentFolder == executorClient.Inbox)
localFolder.ParentRemoteFolderId = string.Empty;
// Set UidValidity for cache expiration.
// Folder must be opened for this.
await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken);
localFolder.UidValidity = remoteFolder.UidValidity;
await remoteFolder.CloseAsync(cancellationToken: cancellationToken);
localFolders.Add(localFolder);
} }
else else
{ {
localFolder.IsSynchronizationEnabled = false; // Update existing folder. Right now we only update the name.
// TODO: Moving servers around different parents. This is not supported right now.
// We will need more comphrensive folder update mechanism to support this.
if (ShouldUpdateFolder(remoteFolder, existingLocalFolder))
{
existingLocalFolder.FolderName = remoteFolder.Name;
}
else
{
// Remove it from the local folder list to skip additional folder updates.
localFolders.Remove(existingLocalFolder);
}
} }
// By default, all special folders update unread count in the UI except Trash.
localFolder.ShowUnreadCount = localFolder.SpecialFolderType != SpecialFolderType.Deleted || localFolder.SpecialFolderType != SpecialFolderType.Other;
localFolder.MailAccountId = Account.Id;
// Sometimes sub folders are parented under Inbox.
// Even though this makes sense in server level, in the client it sucks.
// That will make sub folders to be parented under Inbox in the client.
// Instead, we will mark them as non-parented folders.
// This is better. Model allows personalized folder structure anyways
// even though we don't have the page/control to adjust it.
if (item.ParentFolder == _synchronizationClient.Inbox)
localFolder.ParentRemoteFolderId = string.Empty;
// Set UidValidity for cache expiration.
// Folder must be opened for this.
await item.OpenAsync(FolderAccess.ReadOnly);
localFolder.UidValidity = item.UidValidity;
await item.CloseAsync();
mailItemFolders.Add(localFolder);
} }
catch (Exception ex)
if (localFolders.Any())
{ {
Log.Error(ex, $"Folder with name '{item.Name}' failed to be synchronized."); await _imapChangeProcessor.UpdateFolderStructureAsync(Account.Id, localFolders);
}
else
{
_logger.Information("No update is needed for imap folders.");
} }
} }
catch (Exception ex)
{
_logger.Error(ex, "Synchronizing IMAP folders failed.");
await _imapChangeProcessor.UpdateFolderStructureAsync(Account.Id, mailItemFolders); throw;
}
finally
{
if (executorClient != null)
{
_clientPool.Release(executorClient);
}
}
} }
private async Task<IEnumerable<string>> SynchronizeFolderInternalAsync(MailItemFolder folder, CancellationToken cancellationToken = default) private async Task<IEnumerable<string>> SynchronizeFolderInternalAsync(MailItemFolder folder, CancellationToken cancellationToken = default)
@@ -798,17 +902,17 @@ namespace Wino.Core.Synchronizers
// Fetch completely missing new items in the end. // Fetch completely missing new items in the end.
// Limit check. // Limit check.
//if (missingMailIds.Count > TAKE_COUNT) if (missingMailIds.Count > InitialMessageDownloadCountPerFolder)
//{ {
// missingMailIds = new UniqueIdSet(missingMailIds.TakeLast(TAKE_COUNT)); missingMailIds = new UniqueIdSet(missingMailIds.TakeLast((int)InitialMessageDownloadCountPerFolder));
//} }
// In case of the high input, we'll batch them by 50 to reflect changes quickly. // In case of the high input, we'll batch them by 50 to reflect changes quickly.
var batchedMissingMailIds = missingMailIds.Batch(50).Select(a => new UniqueIdSet(a, SortOrder.Descending)); var batchedMissingMailIds = missingMailIds.Batch(50).Select(a => new UniqueIdSet(a, SortOrder.Descending));
foreach (var batchMissingMailIds in batchedMissingMailIds) foreach (var batchMissingMailIds in batchedMissingMailIds)
{ {
var summaries = await imapFolder.FetchAsync(batchMissingMailIds, mailSynchronizationFlags, cancellationToken); var summaries = await imapFolder.FetchAsync(batchMissingMailIds, mailSynchronizationFlags, cancellationToken).ConfigureAwait(false);
foreach (var summary in summaries) foreach (var summary in summaries)
{ {
@@ -823,7 +927,7 @@ namespace Wino.Core.Synchronizers
foreach (var mailPackage in createdMailPackages) foreach (var mailPackage in createdMailPackages)
{ {
await _imapChangeProcessor.CreateMailAsync(Account.Id, mailPackage); await _imapChangeProcessor.CreateMailAsync(Account.Id, mailPackage).ConfigureAwait(false);
} }
} }
} }
@@ -831,7 +935,8 @@ namespace Wino.Core.Synchronizers
if (folder.HighestModeSeq != (long)imapFolder.HighestModSeq) if (folder.HighestModeSeq != (long)imapFolder.HighestModSeq)
{ {
folder.HighestModeSeq = (long)imapFolder.HighestModSeq; folder.HighestModeSeq = (long)imapFolder.HighestModSeq;
await _imapChangeProcessor.InsertFolderAsync(folder);
await _imapChangeProcessor.UpdateFolderAsync(folder).ConfigureAwait(false);
} }
// Update last synchronization date for the folder.. // Update last synchronization date for the folder..
@@ -842,7 +947,7 @@ namespace Wino.Core.Synchronizers
} }
catch (FolderNotFoundException) catch (FolderNotFoundException)
{ {
await _imapChangeProcessor.DeleteFolderAsync(Account.Id, folder.RemoteFolderId); await _imapChangeProcessor.DeleteFolderAsync(Account.Id, folder.RemoteFolderId).ConfigureAwait(false);
return default; return default;
} }