javier/Wino-Mail
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved online search performance when doing local operations (#584)

Browse Source 
* Improved online search performance when doing local operations

* Retruning an empty list on no item searches.

* Fixed an issue with batch imap downloads.

---------

Co-authored-by: Burak Kaan Köse <bkaankose@outlook.com>
This commit is contained in:
Aleh Khantsevich
2025-02-23 22:17:40 +01:00
committed by GitHub
parent 31b859ba1a
commit 9facfaffa8
8 changed files with 148 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
Wino.Core.Domain/Interfaces/IImapSynchronizerStrategy.cs
View File

@@ -17,16 +17,24 @@ public interface IImapSynchronizerStrategy
/// <param name="synchronizer">Imap synchronizer that downloads messages.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>List of new downloaded message ids that don't exist locally.</returns>
Task<List<string>> HandleSynchronizationAsync(IImapClient client, MailItemFolder folder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default);
Task<List<string>> HandleSynchronizationAsync(IImapClient client,
MailItemFolder folder,
IImapSynchronizer synchronizer,
CancellationToken cancellationToken = default);
/// <summary>
/// Downloads given set of messages from the folder.
/// Folder is expected to be opened and synchronizer is connected.
/// </summary>
/// <param name="synchronizer">Synchronizer that performs the action.</param>
/// <param name="folder">Remote folder to download messages from.</param>
/// <param name="remoteFolder">Remote folder to download messages from.</param>
/// <param name="localFolder">Local folder to assign mails to.</param>
/// <param name="uniqueIdSet">Set of message uniqueids.</param>
/// <param name="cancellationToken">Cancellation token.</param>
Task DownloadMessagesAsync(IImapSynchronizer synchronizer, IMailFolder folder, UniqueIdSet uniqueIdSet, CancellationToken cancellationToken = default);
Task DownloadMessagesAsync(IImapSynchronizer synchronizer,
IMailFolder remoteFolder,
MailItemFolder localFolder,
UniqueIdSet uniqueIdSet,
CancellationToken cancellationToken = default);
}

17
Wino.Core.Domain/Interfaces/IMailService.cs
View File

@@ -18,9 +18,13 @@ public interface IMailService
/// Returns the single mail item with the given mail copy id.
/// Caution: This method is not safe. Use other overrides.
/// </summary>
/// <param name="mailCopyId"></param>
/// <returns></returns>
Task<MailCopy> GetSingleMailItemAsync(string mailCopyId);
/// <summary>
/// Returns the multiple mail item with the given mail copy ids.
/// Caution: This method is not safe. Use other overrides.
/// </summary>
Task<List<MailCopy>> GetMailItemsAsync(IEnumerable<string> mailCopyIds);
Task<List<IMailItem>> FetchMailsAsync(MailListInitializationOptions options, CancellationToken cancellationToken = default);
/// <summary>
@@ -88,6 +92,15 @@ public interface IMailService
/// <param name="mailCopyId">Native mail id of the message.</param>
Task<bool> IsMailExistsAsync(string mailCopyId);
/// <summary>
/// Checks whether the given mail copy ids exists in the database.
/// Safely used for Outlook to prevent downloading the same mail twice.
/// For Gmail, it should be avoided since one mail may belong to multiple folders.
/// </summary>
/// <param name="mailCopyIds">Native mail id of the messages.</param>
/// <returns>List of Mail ids that already exists in the database.</returns>
Task<List<string>> AreMailsExistsAsync(IEnumerable<string> mailCopyIds);
/// <summary>
/// Returns all mails for given folder id.
/// </summary>

8
Wino.Core/Integration/Processors/DefaultChangeProcessor.cs
View File

@@ -61,6 +61,7 @@ public interface IDefaultChangeProcessor
Task UpdateCalendarDeltaSynchronizationToken(Guid calendarId, string deltaToken);
Task<MailCopy> GetMailCopyAsync(string mailCopyId);
Task<List<MailCopy>> GetMailCopiesAsync(IEnumerable<string> mailCopyIds);
Task CreateMailRawAsync(MailAccount account, MailItemFolder mailItemFolder, NewMailItemPackage package);
Task DeleteUserMailCacheAsync(Guid accountId);
@@ -73,6 +74,7 @@ public interface IDefaultChangeProcessor
/// <param name="folderId">Folder's local id.</param>
/// <returns>Whether mail exists in the folder or not.</returns>
Task<bool> IsMailExistsInFolderAsync(string messageId, Guid folderId);
Task<List<string>> AreMailsExistsAsync(IEnumerable<string> mailCopyIds);
}
public interface IGmailChangeProcessor : IDefaultChangeProcessor
@@ -139,9 +141,15 @@ public class DefaultChangeProcessor(IDatabaseService databaseService,
public Task<bool> IsMailExistsAsync(string messageId)
=> MailService.IsMailExistsAsync(messageId);
public Task<List<string>> AreMailsExistsAsync(IEnumerable<string> mailCopyIds)
=> MailService.AreMailsExistsAsync(mailCopyIds);
public Task<MailCopy> GetMailCopyAsync(string mailCopyId)
=> MailService.GetSingleMailItemAsync(mailCopyId);
public Task<List<MailCopy>> GetMailCopiesAsync(IEnumerable<string> mailCopyIds)
=> MailService.GetMailItemsAsync(mailCopyIds);
public Task ChangeMailReadStatusAsync(string mailCopyId, bool isRead)
=> MailService.ChangeReadStatusAsync(mailCopyId, isRead);

38
Wino.Core/Synchronizers/GmailSynchronizer.cs
View File

@@ -1021,7 +1021,7 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
string pageToken = null;
var messagesToDownload = new List<Message>();
List<Message> messagesToDownload = [];
do
{
@@ -1030,7 +1030,7 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
// Ignore the folders if the query starts with these keywords.
// User is trying to list everything.
}
else if (folders?.Any() ?? false)
else if (folders?.Count > 0)
{
request.LabelIds = folders.Select(a => a.RemoteFolderId).ToList();
}
@@ -1044,49 +1044,23 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
if (response.Messages == null) break;
// Handle skipping manually
foreach (var message in response.Messages)
{
messagesToDownload.Add(message);
}
messagesToDownload.AddRange(response.Messages);
pageToken = response.NextPageToken;
} while (!string.IsNullOrEmpty(pageToken));
// Do not download messages that exists, but return them for listing.
var messageIds = messagesToDownload.Select(a => a.Id).ToList();
var messageIds = messagesToDownload.Select(a => a.Id);
List<string> downloadRequireMessageIds = new();
foreach (var messageId in messageIds)
{
var exists = await _gmailChangeProcessor.IsMailExistsAsync(messageId).ConfigureAwait(false);
if (!exists)
{
downloadRequireMessageIds.Add(messageId);
}
}
var downloadRequireMessageIds = messageIds.Except(await _gmailChangeProcessor.AreMailsExistsAsync(messageIds));
// Download missing messages.
await BatchDownloadMessagesAsync(downloadRequireMessageIds, cancellationToken);
// Get results from database and return.
var searchResults = new List<MailCopy>();
foreach (var messageId in messageIds)
{
var copy = await _gmailChangeProcessor.GetMailCopyAsync(messageId).ConfigureAwait(false);
if (copy == null) continue;
searchResults.Add(copy);
}
return searchResults;
// TODO: Return the search result ids.
return await _gmailChangeProcessor.GetMailCopiesAsync(messageIds);
}
public override async Task DownloadMissingMimeMessageAsync(IMailItem mailItem,

13
Wino.Core/Synchronizers/ImapSync/ImapSynchronizationStrategyBase.cs
View File

@@ -84,19 +84,13 @@ public abstract class ImapSynchronizationStrategyBase : IImapSynchronizerStrateg
// Fetch the new mails in batch.
var batchedMessageIds = newMessageIds.Batch(50).ToList();
var downloadTasks = new List<Task>();
// Create tasks for each batch.
foreach (var group in batchedMessageIds)
{
downloadedMessageIds.AddRange(group.Select(a => MailkitClientExtensions.CreateUid(Folder.Id, a.Id)));
var task = DownloadMessagesAsync(synchronizer, remoteFolder, new UniqueIdSet(group), cancellationToken);
downloadTasks.Add(task);
await DownloadMessagesAsync(synchronizer, remoteFolder, Folder, new UniqueIdSet(group), cancellationToken).ConfigureAwait(false);
}
// Wait for all batches to complete.
await Task.WhenAll(downloadTasks).ConfigureAwait(false);
return downloadedMessageIds;
}
@@ -167,6 +161,7 @@ public abstract class ImapSynchronizationStrategyBase : IImapSynchronizerStrateg
public async Task DownloadMessagesAsync(IImapSynchronizer synchronizer,
IMailFolder folder,
MailItemFolder localFolder,
UniqueIdSet uniqueIdSet,
CancellationToken cancellationToken = default)
{
@@ -178,7 +173,7 @@ public abstract class ImapSynchronizationStrategyBase : IImapSynchronizerStrateg
var creationPackage = new ImapMessageCreationPackage(summary, mimeMessage);
var mailPackages = await synchronizer.CreateNewMailPackagesAsync(creationPackage, Folder, cancellationToken).ConfigureAwait(false);
var mailPackages = await synchronizer.CreateNewMailPackagesAsync(creationPackage, localFolder, cancellationToken).ConfigureAwait(false);
if (mailPackages != null)
{
@@ -187,7 +182,7 @@ public abstract class ImapSynchronizationStrategyBase : IImapSynchronizerStrateg
// Local draft is mapped. We don't need to create a new mail copy.
if (package == null) continue;
await MailService.CreateMailAsync(Folder.MailAccountId, package).ConfigureAwait(false);
await MailService.CreateMailAsync(localFolder.MailAccountId, package).ConfigureAwait(false);
}
}
}

46
Wino.Core/Synchronizers/ImapSynchronizer.cs
View File

@@ -639,52 +639,48 @@ public class ImapSynchronizer : WinoSynchronizer<ImapRequest, ImapMessageCreatio
{
client = await _clientPool.GetClientAsync().ConfigureAwait(false);
var searchResults = new List<MailCopy>();
List<string> searchResultFolderMailUids = new();
List<MailCopy> searchResults = [];
List<string> searchResultFolderMailUids = [];
foreach (var folder in folders)
{
var remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId).ConfigureAwait(false);
var remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false);
await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false);
// Look for subject and body.
var query = SearchQuery.BodyContains(queryText).Or(SearchQuery.SubjectContains(queryText));
var searchResultsInFolder = await remoteFolder.SearchAsync(query, cancellationToken).ConfigureAwait(false);
var nonExisttingUniqueIds = new List<UniqueId>();
Dictionary<string, UniqueId> searchResultsIdsInFolder = [];
foreach (var searchResultId in searchResultsInFolder)
{
var folderMailUid = MailkitClientExtensions.CreateUid(folder.Id, searchResultId.Id);
searchResultFolderMailUids.Add(folderMailUid);
bool exists = await _imapChangeProcessor.IsMailExistsAsync(folderMailUid);
if (!exists)
{
nonExisttingUniqueIds.Add(searchResultId);
}
searchResultsIdsInFolder.Add(folderMailUid, searchResultId);
}
if (nonExisttingUniqueIds.Any())
// Populate no foundIds
var foundIds = await _imapChangeProcessor.AreMailsExistsAsync(searchResultsIdsInFolder.Select(a => a.Key));
var notFoundIds = searchResultsIdsInFolder.Keys.Except(foundIds);
List<UniqueId> nonExistingUniqueIds = [];
foreach (var nonExistingId in notFoundIds)
{
nonExistingUniqueIds.Add(searchResultsIdsInFolder[nonExistingId]);
}
if (nonExistingUniqueIds.Count != 0)
{
var syncStrategy = _imapSynchronizationStrategyProvider.GetSynchronizationStrategy(client);
await syncStrategy.DownloadMessagesAsync(this, remoteFolder, new UniqueIdSet(nonExisttingUniqueIds, SortOrder.Ascending), cancellationToken).ConfigureAwait(false);
await syncStrategy.DownloadMessagesAsync(this, remoteFolder, folder as MailItemFolder, new UniqueIdSet(nonExistingUniqueIds, SortOrder.Ascending), cancellationToken).ConfigureAwait(false);
}
await remoteFolder.CloseAsync().ConfigureAwait(false);
await remoteFolder.CloseAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
}
foreach (var messageId in searchResultFolderMailUids)
{
var copy = await _imapChangeProcessor.GetMailCopyAsync(messageId).ConfigureAwait(false);
if (copy == null) continue;
searchResults.Add(copy);
}
return searchResults;
return await _imapChangeProcessor.GetMailCopiesAsync(searchResultFolderMailUids);
}
catch (Exception ex)
{
@@ -700,8 +696,6 @@ public class ImapSynchronizer : WinoSynchronizer<ImapRequest, ImapMessageCreatio
_clientPool.Release(client);
}
return new List<MailCopy>();
}
private async Task<IEnumerable<string>> SynchronizeFolderInternalAsync(MailItemFolder folder, CancellationToken cancellationToken = default)

13
Wino.Core/Synchronizers/OutlookSynchronizer.cs
View File

@@ -1054,18 +1054,7 @@ public class OutlookSynchronizer : WinoSynchronizer<RequestInformation, Message,
}
// Get results from database and return.
var searchResults = new List<MailCopy>();
foreach (var messageId in existingMessageIds)
{
var copy = await _outlookChangeProcessor.GetMailCopyAsync(messageId).ConfigureAwait(false);
if (copy == null) continue;
searchResults.Add(copy);
}
return searchResults;
return await _outlookChangeProcessor.GetMailCopiesAsync(existingMessageIds);
}
private async Task<MimeMessage> DownloadMimeMessageAsync(string messageId, CancellationToken cancellationToken = default)

143
Wino.Services/MailService.cs
View File

@@ -143,7 +143,7 @@ public class MailService : BaseDatabaseService, IMailService
return unreadMails;
}
private string BuildMailFetchQuery(MailListInitializationOptions options)
private static string BuildMailFetchQuery(MailListInitializationOptions options)
{
// If the search query is there, we should ignore some properties and trim it.
//if (!string.IsNullOrEmpty(options.SearchQuery))
@@ -231,7 +231,7 @@ public class MailService : BaseDatabaseService, IMailService
// Avoid DBs calls as possible, storing info in a dictionary.
foreach (var mail in mails)
{
await LoadAssignedPropertiesWithCacheAsync(mail, folderCache, accountCache).ConfigureAwait(false);
await LoadAssignedPropertiesWithCacheAsync(mail, folderCache, accountCache, contactCache).ConfigureAwait(false);
}
// Remove items that has no assigned account or folder.
@@ -244,12 +244,12 @@ public class MailService : BaseDatabaseService, IMailService
// Threading is disabled. Just return everything as it is.
mails.Sort(options.SortingOptionType == SortingOptionType.ReceiveDate ? new DateComparer() : new NameComparer());
return new List<IMailItem>(mails);
return [.. mails];
}
// Populate threaded items.
var threadedItems = new List<IMailItem>();
List<IMailItem> threadedItems = [];
// Each account items must be threaded separately.
foreach (var group in mails.GroupBy(a => a.AssignedAccount.Id))
@@ -270,7 +270,7 @@ public class MailService : BaseDatabaseService, IMailService
foreach (var mail in accountThreadedItems)
{
cancellationToken.ThrowIfCancellationRequested();
await LoadAssignedPropertiesWithCacheAsync(mail, folderCache, accountCache).ConfigureAwait(false);
await LoadAssignedPropertiesWithCacheAsync(mail, folderCache, accountCache, contactCache).ConfigureAwait(false);
}
if (accountThreadedItems != null)
@@ -283,76 +283,65 @@ public class MailService : BaseDatabaseService, IMailService
cancellationToken.ThrowIfCancellationRequested();
return threadedItems;
}
// Recursive function to populate folder and account assignments for each mail item.
async Task LoadAssignedPropertiesWithCacheAsync(IMailItem mail,
Dictionary<Guid, MailItemFolder> folderCache,
Dictionary<Guid, MailAccount> accountCache)
/// <summary>
/// This method should used for operations with multiple mailItems. Don't use this for single mail items.
/// Called method should provide own instances for caches.
/// </summary>
private async Task LoadAssignedPropertiesWithCacheAsync(IMailItem mail, Dictionary<Guid, MailItemFolder> folderCache, Dictionary<Guid, MailAccount> accountCache, Dictionary<string, AccountContact> contactCache)
{
if (mail is ThreadMailItem threadMailItem)
{
if (mail is ThreadMailItem threadMailItem)
foreach (var childMail in threadMailItem.ThreadItems)
{
foreach (var childMail in threadMailItem.ThreadItems)
await LoadAssignedPropertiesWithCacheAsync(childMail, folderCache, accountCache, contactCache).ConfigureAwait(false);
}
}
if (mail is MailCopy mailCopy)
{
var isFolderCached = folderCache.TryGetValue(mailCopy.FolderId, out MailItemFolder folderAssignment);
MailAccount accountAssignment = null;
if (!isFolderCached)
{
folderAssignment = await _folderService.GetFolderAsync(mailCopy.FolderId).ConfigureAwait(false);
folderCache.TryAdd(mailCopy.FolderId, folderAssignment);
}
if (folderAssignment != null)
{
var isAccountCached = accountCache.TryGetValue(folderAssignment.MailAccountId, out accountAssignment);
if (!isAccountCached)
{
await LoadAssignedPropertiesWithCacheAsync(childMail, folderCache, accountCache).ConfigureAwait(false);
accountAssignment = await _accountService.GetAccountAsync(folderAssignment.MailAccountId).ConfigureAwait(false);
accountCache.TryAdd(folderAssignment.MailAccountId, accountAssignment);
}
}
if (mail is MailCopy mailCopy)
AccountContact contactAssignment = null;
bool isContactCached = !string.IsNullOrEmpty(mailCopy.FromAddress) &&
contactCache.TryGetValue(mailCopy.FromAddress, out contactAssignment);
if (!isContactCached && accountAssignment != null)
{
MailAccount accountAssignment = null;
contactAssignment = await GetSenderContactForAccountAsync(accountAssignment, mailCopy.FromAddress).ConfigureAwait(false);
var isFolderCached = folderCache.TryGetValue(mailCopy.FolderId, out MailItemFolder folderAssignment);
accountAssignment = null;
if (!isFolderCached)
if (contactAssignment != null)
{
folderAssignment = await _folderService.GetFolderAsync(mailCopy.FolderId).ConfigureAwait(false);
if (!folderCache.ContainsKey(mailCopy.FolderId))
{
folderCache.Add(mailCopy.FolderId, folderAssignment);
}
contactCache.TryAdd(mailCopy.FromAddress, contactAssignment);
}
if (folderAssignment != null)
{
var isAccountCached = accountCache.TryGetValue(folderAssignment.MailAccountId, out accountAssignment);
if (!isAccountCached)
{
accountAssignment = await _accountService.GetAccountAsync(folderAssignment.MailAccountId).ConfigureAwait(false);
if (!accountCache.ContainsKey(folderAssignment.MailAccountId))
{
accountCache.Add(folderAssignment.MailAccountId, accountAssignment);
}
}
}
AccountContact contactAssignment = null;
bool isContactCached = !string.IsNullOrEmpty(mailCopy.FromAddress) ?
contactCache.TryGetValue(mailCopy.FromAddress, out contactAssignment) :
false;
if (!isContactCached && accountAssignment != null)
{
contactAssignment = await GetSenderContactForAccountAsync(accountAssignment, mailCopy.FromAddress).ConfigureAwait(false);
if (contactAssignment != null)
{
if (!contactCache.ContainsKey(mailCopy.FromAddress))
{
contactCache.Add(mailCopy.FromAddress, contactAssignment);
}
}
}
mailCopy.AssignedFolder = folderAssignment;
mailCopy.AssignedAccount = accountAssignment;
mailCopy.SenderContact = contactAssignment ?? CreateUnknownContact(mailCopy.FromName, mailCopy.FromAddress);
}
mailCopy.AssignedFolder = folderAssignment;
mailCopy.AssignedAccount = accountAssignment;
mailCopy.SenderContact = contactAssignment ?? CreateUnknownContact(mailCopy.FromName, mailCopy.FromAddress);
}
}
private AccountContact CreateUnknownContact(string fromName, string fromAddress)
private static AccountContact CreateUnknownContact(string fromName, string fromAddress)
{
if (string.IsNullOrEmpty(fromName) && string.IsNullOrEmpty(fromAddress))
{
@@ -1071,4 +1060,38 @@ public class MailService : BaseDatabaseService, IMailService
return new GmailArchiveComparisonResult(addedMails, removedMails);
}
public async Task<List<MailCopy>> GetMailItemsAsync(IEnumerable<string> mailCopyIds)
{
if (!mailCopyIds.Any()) return [];
var query = new Query("MailCopy")
.WhereIn("MailCopy.Id", mailCopyIds)
.SelectRaw("MailCopy.*")
.GetRawQuery();
var mailCopies = await Connection.QueryAsync<MailCopy>(query);
if (mailCopies?.Count == 0) return [];
Dictionary<Guid, MailItemFolder> folderCache = [];
Dictionary<Guid, MailAccount> accountCache = [];
Dictionary<string, AccountContact> contactCache = [];
foreach (var mail in mailCopies)
{
await LoadAssignedPropertiesWithCacheAsync(mail, folderCache, accountCache, contactCache).ConfigureAwait(false);
}
return mailCopies;
}
public async Task<List<string>> AreMailsExistsAsync(IEnumerable<string> mailCopyIds)
{
var query = new Query(nameof(MailCopy))
.WhereIn("Id", mailCopyIds)
.Select("Id")
.GetRawQuery();
return await Connection.QueryScalarsAsync<string>(query);
}
}