javier/Wino-Mail
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bumping some nugets. More on the imap synchronizers.

Browse Source
This commit is contained in:
Burak Kaan Köse
2025-01-28 22:56:19 +01:00
parent b7b51ac4e6
commit b73eb3efcb
25 changed files with 300 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
Wino.Core/Synchronizers/ImapSync/CondstoreSynchronizer.cs
View File

@@ -6,14 +6,10 @@ using System.Threading.Tasks;
using MailKit;
using MailKit.Net.Imap;
using MailKit.Search;
using MoreLinq;
using Serilog;
using Wino.Core.Domain.Entities.Mail;
using Wino.Core.Domain.Exceptions;
using Wino.Core.Domain.Interfaces;
using Wino.Core.Domain.Models.MailItem;
using Wino.Core.Integration;
using Wino.Services.Extensions;
using IMailService = Wino.Core.Domain.Interfaces.IMailService;
namespace Wino.Core.Synchronizers.ImapSync
@@ -41,6 +37,7 @@ namespace Wino.Core.Synchronizers.ImapSync
IMailFolder remoteFolder = null;
var downloadedMessageIds = new List<string>();
try
{
remoteFolder = await winoClient.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false);
@@ -57,92 +54,10 @@ namespace Wino.Core.Synchronizers.ImapSync
// the MODSEQ value for deleted messages.
if (remoteHighestModSeq > localHighestModSeq)
{
// Search for emails with a MODSEQ greater than the last known value.
// Use SORT extension if server supports.
IList<UniqueId> changedUids = null;
if (winoClient.Capabilities.HasFlag(ImapCapabilities.Sort))
{
changedUids = await remoteFolder.SortAsync(SearchQuery.ChangedSince(localHighestModSeq), [OrderBy.ReverseDate], cancellationToken).ConfigureAwait(false);
}
else
{
changedUids = await remoteFolder.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq), cancellationToken).ConfigureAwait(false);
}
// For initial synchronizations, take the first allowed number of items.
// For consequtive synchronizations, take all the items. We don't want to miss any changes.
if (isInitialSynchronization)
changedUids = changedUids.Take((int)synchronizer.InitialMessageDownloadCountPerFolder).ToList();
var changedUids = await GetChangedUidsAsync(client, folder, remoteFolder, synchronizer, cancellationToken).ConfigureAwait(false);
// Get locally exists mails for the returned UIDs.
var existingMails = await MailService.GetExistingMailsAsync(folder.Id, changedUids).ConfigureAwait(false);
var existingMailUids = existingMails.Select(m => MailkitClientExtensions.ResolveUidStruct(m.Id)).ToArray();
// These are the non-existing mails. They will be downloaded + processed.
var newMessageIds = changedUids.Except(existingMailUids).ToList();
// Fetch minimum data for the existing mails in one query.
var existingFlagData = await remoteFolder.FetchAsync(existingMailUids, MessageSummaryItems.Flags | MessageSummaryItems.UniqueId).ConfigureAwait(false);
foreach (var update in existingFlagData)
{
if (update.UniqueId == null)
{
Log.Warning($"Couldn't fetch UniqueId for the mail. FetchAsync failed.");
continue;
}
if (update.Flags == null)
{
Log.Warning($"Couldn't fetch flags for the mail with UID {update.UniqueId.Id}. FetchAsync failed.");
continue;
}
var existingMail = existingMails.FirstOrDefault(m => MailkitClientExtensions.ResolveUidStruct(m.Id).Id == update.UniqueId.Id);
if (existingMail == null)
{
Log.Warning($"Couldn't find the mail with UID {update.UniqueId.Id} in the local database. Flag update is ignored.");
continue;
}
await HandleMessageFlagsChangeAsync(existingMail, update.Flags.Value).ConfigureAwait(false);
}
// Fetch the new mails in batch.
var batchedMessageIds = newMessageIds.Batch(50);
foreach (var group in batchedMessageIds)
{
var summaries = await remoteFolder.FetchAsync(group, MailSynchronizationFlags, cancellationToken).ConfigureAwait(false);
foreach (var summary in summaries)
{
var mimeMessage = await remoteFolder.GetMessageAsync(summary.UniqueId, cancellationToken).ConfigureAwait(false);
var creationPackage = new ImapMessageCreationPackage(summary, mimeMessage);
var mailPackages = await synchronizer.CreateNewMailPackagesAsync(creationPackage, folder, cancellationToken).ConfigureAwait(false);
if (mailPackages != null)
{
foreach (var package in mailPackages)
{
// Local draft is mapped. We don't need to create a new mail copy.
if (package == null) continue;
bool isCreatedNew = await MailService.CreateMailAsync(folder.MailAccountId, package).ConfigureAwait(false);
// This is upsert. We are not interested in updated mails.
if (isCreatedNew) downloadedMessageIds.Add(package.Copy.Id);
}
}
}
}
downloadedMessageIds = await HandleChangedUIdsAsync(folder, synchronizer, remoteFolder, changedUids, cancellationToken).ConfigureAwait(false);
folder.HighestModeSeq = (long)remoteHighestModSeq;
@@ -177,5 +92,41 @@ namespace Wino.Core.Synchronizers.ImapSync
}
}
}
internal override async Task<IList<UniqueId>> GetChangedUidsAsync(IImapClient winoClient, MailItemFolder localFolder, IMailFolder remoteFolder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default)
{
var localHighestModSeq = (ulong)localFolder.HighestModeSeq;
var remoteHighestModSeq = remoteFolder.HighestModSeq;
// Search for emails with a MODSEQ greater than the last known value.
// Use SORT extension if server supports.
IList<UniqueId> changedUids = null;
// TODO: Temporarily disabled.
//if (winoClient.Capabilities.HasFlag(ImapCapabilities.Sort))
//{
// changedUids = await remoteFolder.SortAsync(SearchQuery.ChangedSince(localHighestModSeq), [OrderBy.ReverseDate], cancellationToken).ConfigureAwait(false);
//}
//else
//{
// changedUids = await remoteFolder.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq), cancellationToken).ConfigureAwait(false);
//}
changedUids = await remoteFolder.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq), cancellationToken).ConfigureAwait(false);
// For initial synchronizations, take the first allowed number of items.
// For consequtive synchronizations, take all the items. We don't want to miss any changes.
// Smaller uid means newer message. For initial sync, we need start taking items from the top.
bool isInitialSynchronization = localHighestModSeq == 0;
if (isInitialSynchronization)
{
changedUids = changedUids.OrderByDescending(a => a.Id).Take((int)synchronizer.InitialMessageDownloadCountPerFolder).ToList();
}
return changedUids;
}
}
}

79
Wino.Core/Synchronizers/ImapSync/ImapSynchronizationStrategyBase.cs
View File

@@ -5,8 +5,11 @@ using System.Threading.Tasks;
using MailKit;
using MailKit.Net.Imap;
using MailKit.Search;
using MoreLinq;
using Serilog;
using Wino.Core.Domain.Entities.Mail;
using Wino.Core.Domain.Interfaces;
using Wino.Core.Domain.Models.MailItem;
using Wino.Services.Extensions;
using IMailService = Wino.Core.Domain.Interfaces.IMailService;
@@ -36,6 +39,82 @@ namespace Wino.Core.Synchronizers.ImapSync
}
public abstract Task<List<string>> HandleSynchronizationAsync(IImapClient client, MailItemFolder folder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default);
internal abstract Task<IList<UniqueId>> GetChangedUidsAsync(IImapClient client, MailItemFolder localFolder, IMailFolder remoteFolder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default);
protected async Task<List<string>> HandleChangedUIdsAsync(MailItemFolder folder, IImapSynchronizer synchronizer, IMailFolder remoteFolder, IList<UniqueId> changedUids, CancellationToken cancellationToken)
{
List<string> downloadedMessageIds = new();
var existingMails = await MailService.GetExistingMailsAsync(folder.Id, changedUids).ConfigureAwait(false);
var existingMailUids = existingMails.Select(m => MailkitClientExtensions.ResolveUidStruct(m.Id)).ToArray();
// These are the non-existing mails. They will be downloaded + processed.
var newMessageIds = changedUids.Except(existingMailUids).ToList();
var deletedMessageIds = existingMailUids.Except(changedUids).ToList();
// Fetch minimum data for the existing mails in one query.
var existingFlagData = await remoteFolder.FetchAsync(existingMailUids, MessageSummaryItems.Flags | MessageSummaryItems.UniqueId).ConfigureAwait(false);
foreach (var update in existingFlagData)
{
if (update.UniqueId == null)
{
Log.Warning($"Couldn't fetch UniqueId for the mail. FetchAsync failed.");
continue;
}
if (update.Flags == null)
{
Log.Warning($"Couldn't fetch flags for the mail with UID {update.UniqueId.Id}. FetchAsync failed.");
continue;
}
var existingMail = existingMails.FirstOrDefault(m => MailkitClientExtensions.ResolveUidStruct(m.Id).Id == update.UniqueId.Id);
if (existingMail == null)
{
Log.Warning($"Couldn't find the mail with UID {update.UniqueId.Id} in the local database. Flag update is ignored.");
continue;
}
await HandleMessageFlagsChangeAsync(existingMail, update.Flags.Value).ConfigureAwait(false);
}
// Fetch the new mails in batch.
var batchedMessageIds = newMessageIds.Batch(50);
foreach (var group in batchedMessageIds)
{
var summaries = await remoteFolder.FetchAsync(group, MailSynchronizationFlags, cancellationToken).ConfigureAwait(false);
foreach (var summary in summaries)
{
var mimeMessage = await remoteFolder.GetMessageAsync(summary.UniqueId, cancellationToken).ConfigureAwait(false);
var creationPackage = new ImapMessageCreationPackage(summary, mimeMessage);
var mailPackages = await synchronizer.CreateNewMailPackagesAsync(creationPackage, folder, cancellationToken).ConfigureAwait(false);
if (mailPackages != null)
{
foreach (var package in mailPackages)
{
// Local draft is mapped. We don't need to create a new mail copy.
if (package == null) continue;
bool isCreatedNew = await MailService.CreateMailAsync(folder.MailAccountId, package).ConfigureAwait(false);
// This is upsert. We are not interested in updated mails.
if (isCreatedNew) downloadedMessageIds.Add(package.Copy.Id);
}
}
}
}
return downloadedMessageIds;
}
protected async Task HandleMessageFlagsChangeAsync(MailItemFolder folder, UniqueId? uniqueId, MessageFlags flags)
{

48
Wino.Core/Synchronizers/ImapSync/QResyncSynchronizer.cs
View File

@@ -1,6 +1,5 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
@@ -29,6 +28,8 @@ namespace Wino.Core.Synchronizers.ImapSync
IImapSynchronizer synchronizer,
CancellationToken cancellationToken = default)
{
var downloadedMessageIds = new List<string>();
if (client is not WinoImapClient winoClient)
throw new ImapSynchronizerStrategyException("Client must be of type WinoImapClient.");
@@ -54,7 +55,6 @@ namespace Wino.Core.Synchronizers.ImapSync
if (!isCacheValid)
{
// TODO: Remove all local data.
}
// Perform QRESYNC synchronization.
@@ -70,42 +70,52 @@ namespace Wino.Core.Synchronizers.ImapSync
await remoteFolder.OpenAsync(FolderAccess.ReadOnly, folder.UidValidity, localHighestModSeq, allUniqueIds).ConfigureAwait(false);
var changedUids = await remoteFolder.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq), cancellationToken).ConfigureAwait(false);
var changedUids = await GetChangedUidsAsync(client, folder, remoteFolder, synchronizer, cancellationToken).ConfigureAwait(false);
foreach (var uid in changedUids)
{
Debug.WriteLine($"Processing message with UID: {uid}");
// var message = await remoteFolder.GetMessageAsync(uid, cancellationToken).ConfigureAwait(false);
// TODO: Process the message.
}
downloadedMessageIds = await HandleChangedUIdsAsync(folder, synchronizer, remoteFolder, changedUids, cancellationToken).ConfigureAwait(false);
// Update the local folder with the new highest mod-seq and validity.
folder.HighestModeSeq = (long)remoteHighestModSeq;
folder.UidValidity = remoteFolder.UidValidity;
await ManageUUIdBasedDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false);
await FolderService.UpdateFolderAsync(folder).ConfigureAwait(false);
}
catch (Exception ex)
catch (FolderNotFoundException)
{
await FolderService.DeleteFolderAsync(folder.MailAccountId, folder.RemoteFolderId).ConfigureAwait(false);
return default;
}
catch (Exception)
{
throw;
}
finally
{
if (remoteFolder != null)
if (!cancellationToken.IsCancellationRequested)
{
remoteFolder.MessagesVanished -= async (c, r) => await HandleMessageDeletedAsync(folder, r.UniqueIds).ConfigureAwait(false);
remoteFolder.MessageFlagsChanged -= async (c, r) => await HandleMessageFlagsChangeAsync(folder, r.UniqueId, r.Flags).ConfigureAwait(false);
if (remoteFolder.IsOpen)
if (remoteFolder != null)
{
await remoteFolder.CloseAsync();
remoteFolder.MessagesVanished -= async (c, r) => await HandleMessageDeletedAsync(folder, r.UniqueIds).ConfigureAwait(false);
remoteFolder.MessageFlagsChanged -= async (c, r) => await HandleMessageFlagsChangeAsync(folder, r.UniqueId, r.Flags).ConfigureAwait(false);
if (remoteFolder.IsOpen)
{
await remoteFolder.CloseAsync();
}
}
}
}
return default;
return downloadedMessageIds;
}
internal override async Task<IList<UniqueId>> GetChangedUidsAsync(IImapClient client, MailItemFolder localFolder, IMailFolder remoteFolder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default)
{
var localHighestModSeq = (ulong)localFolder.HighestModeSeq;
return await remoteFolder.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq), cancellationToken).ConfigureAwait(false);
}
}
}

60
Wino.Core/Synchronizers/ImapSync/UidBasedSynchronizer.cs
View File

@@ -1,8 +1,11 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MailKit;
using MailKit.Net.Imap;
using MailKit.Search;
using Wino.Core.Domain.Entities.Mail;
using Wino.Core.Domain.Interfaces;
using Wino.Core.Integration;
@@ -12,14 +15,65 @@ namespace Wino.Core.Synchronizers.ImapSync
/// <summary>
/// Uid based IMAP Synchronization strategy.
/// </summary>
internal class UidBasedSynchronizer : IImapSynchronizerStrategy
internal class UidBasedSynchronizer : ImapSynchronizationStrategyBase
{
public Task<List<string>> HandleSynchronizationAsync(IImapClient client, MailItemFolder folder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default)
public UidBasedSynchronizer(IFolderService folderService, Domain.Interfaces.IMailService mailService) : base(folderService, mailService)
{
}
public override async Task<List<string>> HandleSynchronizationAsync(IImapClient client, MailItemFolder folder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default)
{
if (client is not WinoImapClient winoClient)
throw new ArgumentException("Client must be of type WinoImapClient.", nameof(client));
return Task.FromResult(new List<string>());
var downloadedMessageIds = new List<string>();
IMailFolder remoteFolder = null;
try
{
remoteFolder = await winoClient.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false);
await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false);
// Fetch UIDs from the remote folder
var remoteUids = await remoteFolder.SearchAsync(SearchQuery.All, cancellationToken).ConfigureAwait(false);
remoteUids = remoteUids.OrderByDescending(a => a.Id).Take((int)synchronizer.InitialMessageDownloadCountPerFolder).ToList();
await HandleChangedUIdsAsync(folder, synchronizer, remoteFolder, remoteUids, cancellationToken).ConfigureAwait(false);
await ManageUUIdBasedDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false);
}
catch (FolderNotFoundException)
{
await FolderService.DeleteFolderAsync(folder.MailAccountId, folder.RemoteFolderId).ConfigureAwait(false);
return default;
}
catch (Exception)
{
throw;
}
finally
{
if (!cancellationToken.IsCancellationRequested)
{
if (remoteFolder != null)
{
if (remoteFolder.IsOpen)
{
await remoteFolder.CloseAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
}
}
}
}
return downloadedMessageIds;
}
internal override async Task<IList<UniqueId>> GetChangedUidsAsync(IImapClient client, MailItemFolder localFolder, IMailFolder remoteFolder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default)
{
throw new NotImplementedException();
}
}
}

12
Wino.Core/Synchronizers/ImapSynchronizer.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;
@@ -31,8 +32,9 @@ namespace Wino.Core.Synchronizers.Mail
{
public class ImapSynchronizer : WinoSynchronizer<ImapRequest, ImapMessageCreationPackage, object>, IImapSynchronizer
{
[Obsolete("N/A")]
public override uint BatchModificationSize => 1000;
public override uint InitialMessageDownloadCountPerFolder => 250;
public override uint InitialMessageDownloadCountPerFolder => 500;
#region Idle Implementation
@@ -631,8 +633,6 @@ namespace Wino.Core.Synchronizers.Mail
{
if (!folder.IsSynchronizationEnabled) return default;
var downloadedMessageIds = new List<string>();
IImapClient availableClient = null;
retry:
@@ -666,7 +666,6 @@ namespace Wino.Core.Synchronizers.Mail
return new List<string>();
}
/// <summary>
/// Whether the local folder should be updated with the remote folder.
/// IMAP only compares folder name for now.
@@ -705,7 +704,6 @@ namespace Wino.Core.Synchronizers.Mail
// Setup idle client.
idleClient = client;
idleDoneTokenSource ??= new CancellationTokenSource();
idleCancellationTokenSource ??= new CancellationTokenSource();
@@ -716,6 +714,7 @@ namespace Wino.Core.Synchronizers.Mail
inboxFolder.CountChanged += IdleNotificationTriggered;
inboxFolder.MessageFlagsChanged += IdleNotificationTriggered;
inboxFolder.MessageExpunged += IdleNotificationTriggered;
inboxFolder.MessagesVanished += IdleNotificationTriggered;
await client.IdleAsync(idleDoneTokenSource.Token, idleCancellationTokenSource.Token);
}
@@ -745,6 +744,7 @@ namespace Wino.Core.Synchronizers.Mail
inboxFolder.CountChanged -= IdleNotificationTriggered;
inboxFolder.MessageFlagsChanged -= IdleNotificationTriggered;
inboxFolder.MessageExpunged -= IdleNotificationTriggered;
inboxFolder.MessagesVanished -= IdleNotificationTriggered;
}
if (idleDoneTokenSource != null)
@@ -776,6 +776,8 @@ namespace Wino.Core.Synchronizers.Mail
private void RequestIdleChangeSynchronization()
{
Debug.WriteLine("Detected idle change.");
// We don't really need to act on the count change in detail.
// Our synchronization should be enough to handle the changes with on-demand sync.
// We can just trigger a sync here IMAPIdle type.

28
Wino.Core/Wino.Core.csproj
View File

@@ -16,34 +16,34 @@
<ItemGroup>
<PackageReference Include="CommunityToolkit.Diagnostics" Version="8.3.2" />
<PackageReference Include="CommunityToolkit.Mvvm" Version="8.3.2" />
<PackageReference Include="CommunityToolkit.Diagnostics" Version="8.4.0" />
<PackageReference Include="CommunityToolkit.Mvvm" Version="8.4.0" />
<PackageReference Include="Google.Apis.Calendar.v3" Version="1.68.0.3592" />
<PackageReference Include="Google.Apis.Gmail.v1" Version="1.68.0.3427" />
<PackageReference Include="Google.Apis.PeopleService.v1" Version="1.68.0.3359" />
<PackageReference Include="HtmlAgilityPack" Version="1.11.70" />
<PackageReference Include="HtmlKit" Version="1.1.0" />
<PackageReference Include="HtmlAgilityPack" Version="1.11.72" />
<PackageReference Include="HtmlKit" Version="1.2.0" />
<PackageReference Include="IsExternalInit" Version="1.0.3">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="MailKit" Version="4.9.0" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
<PackageReference Include="Microsoft.Graph" Version="5.67.0" />
<PackageReference Include="Microsoft.Identity.Client" Version="4.66.2" />
<PackageReference Include="Microsoft.Identity.Client.Broker" Version="4.66.2" />
<PackageReference Include="Microsoft.Identity.Client.Extensions.Msal" Version="4.66.2" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="9.0.1" />
<PackageReference Include="Microsoft.Graph" Version="5.69.0" />
<PackageReference Include="Microsoft.Identity.Client" Version="4.67.2" />
<PackageReference Include="Microsoft.Identity.Client.Broker" Version="4.67.2" />
<PackageReference Include="Microsoft.Identity.Client.Extensions.Msal" Version="4.67.2" />
<PackageReference Include="MimeKit" Version="4.9.0" />
<PackageReference Include="morelinq" Version="4.3.0" />
<PackageReference Include="morelinq" Version="4.4.0" />
<PackageReference Include="Nito.AsyncEx.Tasks" Version="5.1.2" />
<PackageReference Include="NodaTime" Version="3.2.0" />
<PackageReference Include="Serilog" Version="4.1.0" />
<PackageReference Include="NodaTime" Version="3.2.1" />
<PackageReference Include="Serilog" Version="4.2.0" />
<PackageReference Include="Serilog.Exceptions" Version="8.4.0" />
<PackageReference Include="Serilog.Sinks.Debug" Version="3.0.0" />
<PackageReference Include="Serilog.Sinks.File" Version="6.0.0" />
<PackageReference Include="SkiaSharp" Version="2.88.9" />
<PackageReference Include="SkiaSharp" Version="3.116.1" />
<PackageReference Include="SqlKata" Version="2.4.0" />
<PackageReference Include="System.Text.Encoding.CodePages" Version="8.0.0" />
<PackageReference Include="System.Text.Encoding.CodePages" Version="9.0.1" />
</ItemGroup>
<ItemGroup>