javier/Wino-Mail
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Online Search (#576)

Browse Source 
* Very basic online search for gmail.

* Server side of handling offline search and listing part in listing page.

* Default search mode implementation and search UI improvements.

* Online search for Outlook.

* Very basic online search for gmail.

* Server side of handling offline search and listing part in listing page.

* Default search mode implementation and search UI improvements.

* Online search for Outlook.

* Online search for imap without downloading the messages yet. TODO

* Completing imap search.
This commit is contained in:
Burak Kaan Köse
2025-02-22 00:22:00 +01:00
committed by GitHub
parent 42b695854b
commit f61bcb621b
30 changed files with 900 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Wino.Core/Integration/Json/ServerRequestTypeInfoResolver.cs
View File

@@ -1,7 +1,7 @@
using System.Text.Json.Serialization.Metadata;
using MailKit;
using Wino.Core.Domain.Entities.Mail;
using Wino.Core.Domain.Interfaces;
using Wino.Core.Domain.Models.Folders;
using Wino.Core.Domain.Models.MailItem;
using Wino.Core.Requests.Folder;
using Wino.Core.Requests.Mail;
@@ -45,7 +45,7 @@ public class ServerRequestTypeInfoResolver : DefaultJsonTypeInfoResolver
}
};
}
else if (t.Type == typeof(IMailFolder))
else if (t.Type == typeof(IMailItemFolder))
{
t.PolymorphismOptions = new JsonPolymorphismOptions()
{

32
Wino.Core/Integration/Processors/DefaultChangeProcessor.cs
View File

@@ -38,9 +38,17 @@ public interface IDefaultChangeProcessor
Task<List<MailItemFolder>> GetSynchronizationFoldersAsync(MailSynchronizationOptions options);
Task<bool> MapLocalDraftAsync(Guid accountId, Guid localDraftCopyUniqueId, string newMailCopyId, string newDraftId, string newThreadId);
Task UpdateFolderLastSyncDateAsync(Guid folderId);
Task<List<MailItemFolder>> GetExistingFoldersAsync(Guid accountId);
Task UpdateRemoteAliasInformationAsync(MailAccount account, List<RemoteAccountAlias> remoteAccountAliases);
/// <summary>
/// Interrupted initial synchronization may cause downloaded mails to be saved in the database twice.
/// Since downloading mime is costly in Outlook, we need to check if the actual copy of the message has been saved before.
/// This is also used in online search to prevent duplicate mails.
/// </summary>
/// <param name="messageId">MailCopyId of the message.</param>
/// <returns>Whether mail exists or not.</returns>
Task<bool> IsMailExistsAsync(string messageId);
// Calendar
Task<List<AccountCalendar>> GetAccountCalendarsAsync(Guid accountId);
@@ -51,6 +59,8 @@ public interface IDefaultChangeProcessor
Task UpdateAccountCalendarAsync(AccountCalendar accountCalendar);
Task UpdateCalendarDeltaSynchronizationToken(Guid calendarId, string deltaToken);
Task<MailCopy> GetMailCopyAsync(string mailCopyId);
Task CreateMailRawAsync(MailAccount account, MailItemFolder mailItemFolder, NewMailItemPackage package);
}
public interface IGmailChangeProcessor : IDefaultChangeProcessor
@@ -62,14 +72,6 @@ public interface IGmailChangeProcessor : IDefaultChangeProcessor
public interface IOutlookChangeProcessor : IDefaultChangeProcessor
{
/// <summary>
/// Interrupted initial synchronization may cause downloaded mails to be saved in the database twice.
/// Since downloading mime is costly in Outlook, we need to check if the actual copy of the message has been saved before.
/// </summary>
/// <param name="messageId">MailCopyId of the message.</param>
/// <returns>Whether the mime has b</returns>
Task<bool> IsMailExistsAsync(string messageId);
/// <summary>
/// Checks whether the mail exists in the folder.
/// When deciding Create or Update existing mail, we need to check if the mail exists in the folder.
@@ -141,22 +143,26 @@ public class DefaultChangeProcessor(IDatabaseService databaseService,
public Task ChangeFlagStatusAsync(string mailCopyId, bool isFlagged)
=> MailService.ChangeFlagStatusAsync(mailCopyId, isFlagged);
public Task<bool> IsMailExistsAsync(string messageId)
=> MailService.IsMailExistsAsync(messageId);
public Task<MailCopy> GetMailCopyAsync(string mailCopyId)
=> MailService.GetSingleMailItemAsync(mailCopyId);
public Task ChangeMailReadStatusAsync(string mailCopyId, bool isRead)
=> MailService.ChangeReadStatusAsync(mailCopyId, isRead);
public Task DeleteAssignmentAsync(Guid accountId, string mailCopyId, string remoteFolderId)
=> MailService.DeleteAssignmentAsync(accountId, mailCopyId, remoteFolderId);
public Task DeleteMailAsync(Guid accountId, string mailId)
=> MailService.DeleteMailAsync(accountId, mailId);
public Task<bool> CreateMailAsync(Guid accountId, NewMailItemPackage package)
=> MailService.CreateMailAsync(accountId, package);
public Task<List<MailItemFolder>> GetExistingFoldersAsync(Guid accountId)
=> FolderService.GetFoldersAsync(accountId);
public Task CreateMailRawAsync(MailAccount account, MailItemFolder mailItemFolder, NewMailItemPackage package)
=> MailService.CreateMailRawAsync(account, mailItemFolder, package);
public Task<bool> MapLocalDraftAsync(Guid accountId, Guid localDraftCopyUniqueId, string newMailCopyId, string newDraftId, string newThreadId)
=> MailService.MapLocalDraftAsync(accountId, localDraftCopyUniqueId, newMailCopyId, newDraftId, newThreadId);

2
Wino.Core/Integration/Processors/OutlookChangeProcessor.cs
View File

@@ -20,8 +20,6 @@ public class OutlookChangeProcessor(IDatabaseService databaseService,
IMimeFileService mimeFileService) : DefaultChangeProcessor(databaseService, folderService, mailService, calendarService, accountService, mimeFileService)
, IOutlookChangeProcessor
{
public Task<bool> IsMailExistsAsync(string messageId)
=> MailService.IsMailExistsAsync(messageId);
public Task<bool> IsMailExistsInFolderAsync(string messageId, Guid folderId)
=> MailService.IsMailExistsAsync(messageId, folderId);

72
Wino.Core/Synchronizers/GmailSynchronizer.cs
View File

@@ -24,6 +24,7 @@ using Wino.Core.Domain.Enums;
using Wino.Core.Domain.Exceptions;
using Wino.Core.Domain.Interfaces;
using Wino.Core.Domain.Models.Accounts;
using Wino.Core.Domain.Models.Folders;
using Wino.Core.Domain.Models.MailItem;
using Wino.Core.Domain.Models.Synchronization;
using Wino.Core.Extensions;
@@ -903,6 +904,77 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
return [new HttpRequestBundle<IClientServiceRequest>(networkCall, singleDraftRequest, singleDraftRequest)];
}
public override async Task<List<MailCopy>> OnlineSearchAsync(string queryText, List<IMailItemFolder> folders, CancellationToken cancellationToken = default)
{
var request = _gmailService.Users.Messages.List("me");
request.Q = queryText;
request.MaxResults = 500; // Max 500 is returned.
string pageToken = null;
var messagesToDownload = new List<Message>();
do
{
if (folders?.Any() ?? false)
{
request.LabelIds = folders.Select(a => a.RemoteFolderId).ToList();
}
if (!string.IsNullOrEmpty(pageToken))
{
request.PageToken = pageToken;
}
var response = await request.ExecuteAsync(cancellationToken);
if (response.Messages == null) break;
// Handle skipping manually
foreach (var message in response.Messages)
{
messagesToDownload.Add(message);
}
pageToken = response.NextPageToken;
} while (!string.IsNullOrEmpty(pageToken));
// Do not download messages that exists, but return them for listing.
var messageIds = messagesToDownload.Select(a => a.Id).ToList();
List<string> downloadRequireMessageIds = new();
foreach (var messageId in messageIds)
{
var exists = await _gmailChangeProcessor.IsMailExistsAsync(messageId).ConfigureAwait(false);
if (!exists)
{
downloadRequireMessageIds.Add(messageId);
}
}
// Download missing messages.
await BatchDownloadMessagesAsync(downloadRequireMessageIds, cancellationToken);
// Get results from database and return.
var searchResults = new List<MailCopy>();
foreach (var messageId in messageIds)
{
var copy = await _gmailChangeProcessor.GetMailCopyAsync(messageId).ConfigureAwait(false);
if (copy == null) continue;
searchResults.Add(copy);
}
return searchResults;
// TODO: Return the search result ids.
}
public override async Task DownloadMissingMimeMessageAsync(IMailItem mailItem,
ITransferProgress transferProgress = null,
CancellationToken cancellationToken = default)

63
Wino.Core/Synchronizers/ImapSync/ImapSynchronizationStrategyBase.cs
View File

@@ -83,38 +83,19 @@ public abstract class ImapSynchronizationStrategyBase : IImapSynchronizerStrateg
// Fetch the new mails in batch.
var batchedMessageIds = newMessageIds.Batch(50);
var batchedMessageIds = newMessageIds.Batch(50).ToList();
var downloadTasks = new List<Task>();
// Create tasks for each batch.
foreach (var group in batchedMessageIds)
{
var uniqueIdSet = new UniqueIdSet(group, SortOrder.Ascending);
var summaries = await remoteFolder.FetchAsync(uniqueIdSet, MailSynchronizationFlags, cancellationToken).ConfigureAwait(false);
foreach (var summary in summaries)
{
var mimeMessage = await remoteFolder.GetMessageAsync(summary.UniqueId, cancellationToken).ConfigureAwait(false);
var creationPackage = new ImapMessageCreationPackage(summary, mimeMessage);
var mailPackages = await synchronizer.CreateNewMailPackagesAsync(creationPackage, Folder, cancellationToken).ConfigureAwait(false);
if (mailPackages != null)
{
foreach (var package in mailPackages)
{
// Local draft is mapped. We don't need to create a new mail copy.
if (package == null) continue;
bool isCreatedNew = await MailService.CreateMailAsync(Folder.MailAccountId, package).ConfigureAwait(false);
// This is upsert. We are not interested in updated mails.
if (isCreatedNew) downloadedMessageIds.Add(package.Copy.Id);
}
}
}
downloadedMessageIds.AddRange(group.Select(a => MailkitClientExtensions.CreateUid(Folder.Id, a.Id)));
var task = DownloadMessagesAsync(synchronizer, remoteFolder, new UniqueIdSet(group), cancellationToken);
downloadTasks.Add(task);
}
// Wait for all batches to complete.
await Task.WhenAll(downloadTasks).ConfigureAwait(false);
return downloadedMessageIds;
}
@@ -183,4 +164,32 @@ public abstract class ImapSynchronizationStrategyBase : IImapSynchronizerStrateg
await HandleMessageDeletedAsync(deletedUids).ConfigureAwait(false);
}
}
public async Task DownloadMessagesAsync(IImapSynchronizer synchronizer,
IMailFolder folder,
UniqueIdSet uniqueIdSet,
CancellationToken cancellationToken = default)
{
var summaries = await folder.FetchAsync(uniqueIdSet, MailSynchronizationFlags, cancellationToken).ConfigureAwait(false);
foreach (var summary in summaries)
{
var mimeMessage = await folder.GetMessageAsync(summary.UniqueId, cancellationToken).ConfigureAwait(false);
var creationPackage = new ImapMessageCreationPackage(summary, mimeMessage);
var mailPackages = await synchronizer.CreateNewMailPackagesAsync(creationPackage, Folder, cancellationToken).ConfigureAwait(false);
if (mailPackages != null)
{
foreach (var package in mailPackages)
{
// Local draft is mapped. We don't need to create a new mail copy.
if (package == null) continue;
await MailService.CreateMailAsync(Folder.MailAccountId, package).ConfigureAwait(false);
}
}
}
}
}

76
Wino.Core/Synchronizers/ImapSynchronizer.cs
View File

@@ -8,6 +8,7 @@ using System.Threading.Tasks;
using CommunityToolkit.Mvvm.Messaging;
using MailKit;
using MailKit.Net.Imap;
using MailKit.Search;
using MoreLinq;
using Serilog;
using Wino.Core.Domain.Entities.Mail;
@@ -16,6 +17,7 @@ using Wino.Core.Domain.Enums;
using Wino.Core.Domain.Exceptions;
using Wino.Core.Domain.Interfaces;
using Wino.Core.Domain.Models.Connectivity;
using Wino.Core.Domain.Models.Folders;
using Wino.Core.Domain.Models.MailItem;
using Wino.Core.Domain.Models.Synchronization;
using Wino.Core.Extensions;
@@ -628,6 +630,80 @@ public class ImapSynchronizer : WinoSynchronizer<ImapRequest, ImapMessageCreatio
}
}
public override async Task<List<MailCopy>> OnlineSearchAsync(string queryText, List<IMailItemFolder> folders, CancellationToken cancellationToken = default)
{
IImapClient client = null;
IMailFolder activeFolder = null;
try
{
client = await _clientPool.GetClientAsync().ConfigureAwait(false);
var searchResults = new List<MailCopy>();
List<string> searchResultFolderMailUids = new();
foreach (var folder in folders)
{
var remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId).ConfigureAwait(false);
await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false);
// Look for subject and body.
var query = SearchQuery.BodyContains(queryText).Or(SearchQuery.SubjectContains(queryText));
var searchResultsInFolder = await remoteFolder.SearchAsync(query, cancellationToken).ConfigureAwait(false);
var nonExisttingUniqueIds = new List<UniqueId>();
foreach (var searchResultId in searchResultsInFolder)
{
var folderMailUid = MailkitClientExtensions.CreateUid(folder.Id, searchResultId.Id);
searchResultFolderMailUids.Add(folderMailUid);
bool exists = await _imapChangeProcessor.IsMailExistsAsync(folderMailUid);
if (!exists)
{
nonExisttingUniqueIds.Add(searchResultId);
}
}
if (nonExisttingUniqueIds.Any())
{
var syncStrategy = _imapSynchronizationStrategyProvider.GetSynchronizationStrategy(client);
await syncStrategy.DownloadMessagesAsync(this, remoteFolder, new UniqueIdSet(nonExisttingUniqueIds, SortOrder.Ascending), cancellationToken).ConfigureAwait(false);
}
await remoteFolder.CloseAsync().ConfigureAwait(false);
}
foreach (var messageId in searchResultFolderMailUids)
{
var copy = await _imapChangeProcessor.GetMailCopyAsync(messageId).ConfigureAwait(false);
if (copy == null) continue;
searchResults.Add(copy);
}
return searchResults;
}
catch (Exception ex)
{
Log.Error(ex, "Failed to perform online imap search.");
throw;
}
finally
{
if (activeFolder?.IsOpen ?? false)
{
await activeFolder.CloseAsync().ConfigureAwait(false);
}
_clientPool.Release(client);
}
return new List<MailCopy>();
}
private async Task<IEnumerable<string>> SynchronizeFolderInternalAsync(MailItemFolder folder, CancellationToken cancellationToken = default)
{
if (!folder.IsSynchronizationEnabled) return default;

134
Wino.Core/Synchronizers/OutlookSynchronizer.cs
View File

@@ -28,6 +28,7 @@ using Wino.Core.Domain.Enums;
using Wino.Core.Domain.Exceptions;
using Wino.Core.Domain.Interfaces;
using Wino.Core.Domain.Models.Accounts;
using Wino.Core.Domain.Models.Folders;
using Wino.Core.Domain.Models.MailItem;
using Wino.Core.Domain.Models.Synchronization;
using Wino.Core.Extensions;
@@ -42,7 +43,7 @@ namespace Wino.Core.Synchronizers.Mail;
[JsonSerializable(typeof(Microsoft.Graph.Me.Messages.Item.Move.MovePostRequestBody))]
[JsonSerializable(typeof(OutlookFileAttachment))]
public partial class OutlookSynchronizerJsonContext: JsonSerializerContext;
public partial class OutlookSynchronizerJsonContext : JsonSerializerContext;
public class OutlookSynchronizer : WinoSynchronizer<RequestInformation, Message, Event>
{
@@ -187,6 +188,33 @@ public class OutlookSynchronizer : WinoSynchronizer<RequestInformation, Message,
return MailSynchronizationResult.Completed(unreadNewItems);
}
public async Task DownloadSearchResultMessageAsync(string messageId, MailItemFolder assignedFolder, CancellationToken cancellationToken = default)
{
Log.Information("Downloading search result message {messageId} for {Name} - {FolderName}", messageId, Account.Name, assignedFolder.FolderName);
// Outlook message handling was a little strange.
// Instead of changing it from the scratch, we will just download the message and process it.
// Search results will only return Id for the messages.
// This method will download the raw mime, get the required enough metadata from the service and create
// the mail locally. Message ids passed to this method is expected to be non-existent locally.
var message = await _graphClient.Me.Messages[messageId].GetAsync((config) =>
{
config.QueryParameters.Select = outlookMessageSelectParameters;
}, cancellationToken).ConfigureAwait(false);
var mailPackages = await CreateNewMailPackagesAsync(message, assignedFolder, cancellationToken).ConfigureAwait(false);
if (mailPackages == null) return;
foreach (var package in mailPackages)
{
cancellationToken.ThrowIfCancellationRequested();
await _outlookChangeProcessor.CreateMailRawAsync(Account, assignedFolder, package).ConfigureAwait(false);
}
}
private async Task<IEnumerable<string>> SynchronizeFolderAsync(MailItemFolder folder, CancellationToken cancellationToken = default)
{
var downloadedMessageIds = new List<string>();
@@ -927,6 +955,109 @@ public class OutlookSynchronizer : WinoSynchronizer<RequestInformation, Message,
}
}
public override async Task<List<MailCopy>> OnlineSearchAsync(string queryText, List<IMailItemFolder> folders, CancellationToken cancellationToken = default)
{
bool isFoldersIncluded = folders?.Any() ?? false;
var messagesToDownload = new List<Message>();
// Perform search for each folder separately.
if (isFoldersIncluded)
{
var folderIds = folders.Select(a => a.RemoteFolderId);
var tasks = folderIds.Select(async folderId =>
{
var mailQuery = _graphClient.Me.MailFolders[folderId].Messages
.GetAsync(requestConfig =>
{
requestConfig.QueryParameters.Search = $"\"{queryText}\"";
requestConfig.QueryParameters.Select = ["Id, ParentFolderId"];
requestConfig.QueryParameters.Top = 1000;
});
var result = await mailQuery;
if (result?.Value != null)
{
lock (messagesToDownload)
{
messagesToDownload.AddRange(result.Value);
}
}
});
await Task.WhenAll(tasks);
}
else
{
// Perform search for all messages without folder data.
var mailQuery = _graphClient.Me.Messages
.GetAsync(requestConfig =>
{
requestConfig.QueryParameters.Search = $"\"{queryText}\"";
requestConfig.QueryParameters.Select = ["Id, ParentFolderId"];
requestConfig.QueryParameters.Top = 1000;
});
var result = await mailQuery;
if (result?.Value != null)
{
lock (messagesToDownload)
{
messagesToDownload.AddRange(result.Value);
}
}
}
// Do not download messages that exists, but return them for listing.
var localFolders = await _outlookChangeProcessor.GetLocalFoldersAsync(Account.Id).ConfigureAwait(false);
var existingMessageIds = new List<string>();
//Download missing messages.
foreach (var message in messagesToDownload)
{
var messageId = message.Id;
var parentFolderId = message.ParentFolderId;
if (!localFolders.Any(a => a.RemoteFolderId == parentFolderId))
{
Log.Warning($"Search result returned a message from a folder that is not synchronized.");
continue;
}
existingMessageIds.Add(messageId);
var exists = await _outlookChangeProcessor.IsMailExistsAsync(messageId).ConfigureAwait(false);
if (!exists)
{
// Check if folder exists. We can't download a mail without existing folder.
var localFolder = localFolders.Find(a => a.RemoteFolderId == parentFolderId);
await DownloadSearchResultMessageAsync(messageId, localFolder, cancellationToken).ConfigureAwait(false);
}
}
// Get results from database and return.
var searchResults = new List<MailCopy>();
foreach (var messageId in existingMessageIds)
{
var copy = await _outlookChangeProcessor.GetMailCopyAsync(messageId).ConfigureAwait(false);
if (copy == null) continue;
searchResults.Add(copy);
}
return searchResults;
}
private async Task<MimeMessage> DownloadMimeMessageAsync(string messageId, CancellationToken cancellationToken = default)
{
var mimeContentStream = await _graphClient.Me.Messages[messageId].Content.GetAsync(null, cancellationToken).ConfigureAwait(false);
@@ -935,7 +1066,6 @@ public class OutlookSynchronizer : WinoSynchronizer<RequestInformation, Message,
public override async Task<List<NewMailItemPackage>> CreateNewMailPackagesAsync(Message message, MailItemFolder assignedFolder, CancellationToken cancellationToken = default)
{
var mimeMessage = await DownloadMimeMessageAsync(message.Id, cancellationToken).ConfigureAwait(false);
var mailCopy = message.AsMailCopy();

11
Wino.Core/Synchronizers/WinoSynchronizer.cs
View File

@@ -14,6 +14,7 @@ using Wino.Core.Domain.Entities.Shared;
using Wino.Core.Domain.Enums;
using Wino.Core.Domain.Interfaces;
using Wino.Core.Domain.Models.Accounts;
using Wino.Core.Domain.Models.Folders;
using Wino.Core.Domain.Models.MailItem;
using Wino.Core.Domain.Models.Synchronization;
using Wino.Core.Requests.Bundles;
@@ -416,6 +417,16 @@ public abstract class WinoSynchronizer<TBaseRequest, TMessageType, TCalendarEven
/// <param name="cancellationToken">Cancellation token.</param>
public virtual Task DownloadMissingMimeMessageAsync(IMailItem mailItem, ITransferProgress transferProgress = null, CancellationToken cancellationToken = default) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType()));
/// <summary>
/// Performs an online search for the given query text in the given folders.
/// Downloads the missing messages from the server.
/// </summary>
/// <param name="queryText">Query to search for.</param>
/// <param name="folders">Which folders to include in.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <exception cref="NotSupportedException"></exception>
public virtual Task<List<MailCopy>> OnlineSearchAsync(string queryText, List<IMailItemFolder> folders, CancellationToken cancellationToken = default) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType()));
public List<IRequestBundle<ImapRequest>> CreateSingleTaskBundle(Func<IImapClient, IRequestBase, Task> action, IRequestBase request, IUIChangeRequest uIChangeRequest)
{
return [new ImapRequestBundle(new ImapRequest(action, request), request, uIChangeRequest)];