From e0f87f1374eab64fd387a6edd8c10067de6b04fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Kaan=20K=C3=B6se?= Date: Sun, 19 Jan 2025 20:35:41 +0100 Subject: [PATCH] IDLE implementation, imap synchronization strategies basics and condstore synchronization. --- .../Enums/MailSynchronizationType.cs | 4 +- .../ImapSynchronizerStrategyException.cs | 10 + .../ICustomFolderSynchronizationRequest.cs | 5 + .../IImapSynchronizationStrategyProvider.cs | 12 + .../Interfaces/IImapSynchronizer.cs | 15 + .../Interfaces/IImapSynchronizerStrategy.cs | 22 + Wino.Core.Domain/Interfaces/IMailService.cs | 9 + .../MailItem/ImapMessageCreationPackage.cs | 20 + .../MailSynchronizationOptions.cs | 8 +- Wino.Core/CoreContainerSetup.cs | 6 + Wino.Core/Integration/ImapClientPool.cs | 72 ++- Wino.Core/Integration/WinoImapClient.cs | 60 ++ Wino.Core/Mime/ImapMessageCreationPackage.cs | 9 - .../Requests/Bundles/TaskRequestBundle.cs | 6 +- .../Requests/Folder/EmptyFolderRequest.cs | 1 + .../Folder/MarkFolderAsReadRequest.cs | 2 + Wino.Core/Requests/Mail/ArchiveRequest.cs | 1 + Wino.Core/Requests/Mail/ChangeFlagRequest.cs | 2 + Wino.Core/Requests/Mail/CreateDraftRequest.cs | 2 + Wino.Core/Requests/Mail/DeleteRequest.cs | 2 +- Wino.Core/Requests/Mail/MarkReadRequest.cs | 2 + Wino.Core/Requests/Mail/MoveRequest.cs | 2 +- Wino.Core/Requests/Mail/SendDraftRequest.cs | 2 + Wino.Core/Services/SynchronizerFactory.cs | 12 +- .../ImapSync/CondstoreSynchronizer.cs | 152 +++++ .../ImapSynchronizationStrategyBase.cs | 99 ++++ .../ImapSynchronizationStrategyProvider.cs | 31 + .../ImapSync/QResyncSynchronizer.cs | 111 ++++ .../ImapSync/UidBasedSynchronizer.cs | 25 + Wino.Core/Synchronizers/ImapSynchronizer.cs | 556 +++++------------- Wino.Core/Synchronizers/WinoSynchronizer.cs | 223 ++++--- Wino.Core/Wino.Core.csproj | 4 + Wino.Mail.ViewModels/AppShellViewModel.cs | 2 + Wino.Mail/App.xaml.cs | 1 + Wino.Mail/Views/MailListPage.xaml | 5 - .../Extensions/MailkitClientExtensions.cs | 4 +- Wino.Services/FolderService.cs | 33 +- Wino.Services/MailService.cs | 11 + 38 files changed, 980 insertions(+), 563 deletions(-) create mode 100644 Wino.Core.Domain/Exceptions/ImapSynchronizerStrategyException.cs create mode 100644 Wino.Core.Domain/Interfaces/IImapSynchronizationStrategyProvider.cs create mode 100644 Wino.Core.Domain/Interfaces/IImapSynchronizer.cs create mode 100644 Wino.Core.Domain/Interfaces/IImapSynchronizerStrategy.cs create mode 100644 Wino.Core.Domain/Models/MailItem/ImapMessageCreationPackage.cs create mode 100644 Wino.Core/Integration/WinoImapClient.cs delete mode 100644 Wino.Core/Mime/ImapMessageCreationPackage.cs create mode 100644 Wino.Core/Synchronizers/ImapSync/CondstoreSynchronizer.cs create mode 100644 Wino.Core/Synchronizers/ImapSync/ImapSynchronizationStrategyBase.cs create mode 100644 Wino.Core/Synchronizers/ImapSync/ImapSynchronizationStrategyProvider.cs create mode 100644 Wino.Core/Synchronizers/ImapSync/QResyncSynchronizer.cs create mode 100644 Wino.Core/Synchronizers/ImapSync/UidBasedSynchronizer.cs diff --git a/Wino.Core.Domain/Enums/MailSynchronizationType.cs b/Wino.Core.Domain/Enums/MailSynchronizationType.cs index 5fd16667..e7dfe826 100644 --- a/Wino.Core.Domain/Enums/MailSynchronizationType.cs +++ b/Wino.Core.Domain/Enums/MailSynchronizationType.cs @@ -2,13 +2,13 @@ { public enum MailSynchronizationType { - // Shared UpdateProfile, // Only update profile information ExecuteRequests, // Run the queued requests, and then synchronize if needed. FoldersOnly, // Only synchronize folder metadata. - InboxOnly, // Only Inbox, Sent and Draft folders. + InboxOnly, // Only Inbox, Sent, Draft and Deleted folders. CustomFolders, // Only sync folders that are specified in the options. FullFolders, // Synchronize all folders. This won't update profile or alias information. Alias, // Only update alias information + IMAPIdle // Idle client triggered synchronization. } } diff --git a/Wino.Core.Domain/Exceptions/ImapSynchronizerStrategyException.cs b/Wino.Core.Domain/Exceptions/ImapSynchronizerStrategyException.cs new file mode 100644 index 00000000..e59927ab --- /dev/null +++ b/Wino.Core.Domain/Exceptions/ImapSynchronizerStrategyException.cs @@ -0,0 +1,10 @@ +namespace Wino.Core.Domain.Exceptions +{ + public class ImapSynchronizerStrategyException : System.Exception + { + public ImapSynchronizerStrategyException(string message) : base(message) + { + + } + } +} diff --git a/Wino.Core.Domain/Interfaces/ICustomFolderSynchronizationRequest.cs b/Wino.Core.Domain/Interfaces/ICustomFolderSynchronizationRequest.cs index b1784d08..36c90040 100644 --- a/Wino.Core.Domain/Interfaces/ICustomFolderSynchronizationRequest.cs +++ b/Wino.Core.Domain/Interfaces/ICustomFolderSynchronizationRequest.cs @@ -13,5 +13,10 @@ namespace Wino.Core.Domain.Interfaces /// Which folders to sync after this operation? /// List SynchronizationFolderIds { get; } + + /// + /// If true, additional folders like Sent, Drafts and Deleted will not be synchronized + /// + bool ExcludeMustHaveFolders { get; } } } diff --git a/Wino.Core.Domain/Interfaces/IImapSynchronizationStrategyProvider.cs b/Wino.Core.Domain/Interfaces/IImapSynchronizationStrategyProvider.cs new file mode 100644 index 00000000..6292285f --- /dev/null +++ b/Wino.Core.Domain/Interfaces/IImapSynchronizationStrategyProvider.cs @@ -0,0 +1,12 @@ +using MailKit.Net.Imap; + +namespace Wino.Core.Domain.Interfaces +{ + /// + /// Provides a synchronization strategy for synchronizing IMAP folders based on the server capabilities. + /// + public interface IImapSynchronizationStrategyProvider + { + IImapSynchronizerStrategy GetSynchronizationStrategy(IImapClient client); + } +} diff --git a/Wino.Core.Domain/Interfaces/IImapSynchronizer.cs b/Wino.Core.Domain/Interfaces/IImapSynchronizer.cs new file mode 100644 index 00000000..d3c152f6 --- /dev/null +++ b/Wino.Core.Domain/Interfaces/IImapSynchronizer.cs @@ -0,0 +1,15 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Wino.Core.Domain.Entities.Mail; +using Wino.Core.Domain.Models.MailItem; + +namespace Wino.Core.Domain.Interfaces +{ + public interface IImapSynchronizer + { + Task> CreateNewMailPackagesAsync(ImapMessageCreationPackage message, MailItemFolder assignedFolder, CancellationToken cancellationToken = default); + Task StartIdleClientAsync(); + Task StopIdleClientAsync(); + } +} diff --git a/Wino.Core.Domain/Interfaces/IImapSynchronizerStrategy.cs b/Wino.Core.Domain/Interfaces/IImapSynchronizerStrategy.cs new file mode 100644 index 00000000..0b1f7aff --- /dev/null +++ b/Wino.Core.Domain/Interfaces/IImapSynchronizerStrategy.cs @@ -0,0 +1,22 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using MailKit.Net.Imap; +using Wino.Core.Domain.Entities.Mail; + +namespace Wino.Core.Domain.Interfaces +{ + public interface IImapSynchronizerStrategy + { + /// + /// Synchronizes given folder with the ImapClient client from the client pool. + /// + /// Client to perform sync with. I love Mira and Jasminka + /// Folder to synchronize. + /// Imap synchronizer that downloads messages. + /// Cancellation token. + /// List of new downloaded message ids that don't exist locally. + Task> HandleSynchronizationAsync(IImapClient client, MailItemFolder folder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default); + } +} + diff --git a/Wino.Core.Domain/Interfaces/IMailService.cs b/Wino.Core.Domain/Interfaces/IMailService.cs index 1867ff5c..3474d8ea 100644 --- a/Wino.Core.Domain/Interfaces/IMailService.cs +++ b/Wino.Core.Domain/Interfaces/IMailService.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using MailKit; using Wino.Core.Domain.Entities.Mail; using Wino.Core.Domain.Entities.Shared; using Wino.Core.Domain.Models.MailItem; @@ -108,5 +109,13 @@ namespace Wino.Core.Domain.Interfaces /// Options like new email/forward/draft. /// Draft MailCopy and Draft MimeMessage as base64. Task<(MailCopy draftMailCopy, string draftBase64MimeMessage)> CreateDraftAsync(Guid accountId, DraftCreationOptions draftCreationOptions); + + /// + /// Returns ids + /// + /// + /// + /// + Task> GetExistingMailsAsync(Guid folderId, IEnumerable uniqueIds); } } diff --git a/Wino.Core.Domain/Models/MailItem/ImapMessageCreationPackage.cs b/Wino.Core.Domain/Models/MailItem/ImapMessageCreationPackage.cs new file mode 100644 index 00000000..1fb23688 --- /dev/null +++ b/Wino.Core.Domain/Models/MailItem/ImapMessageCreationPackage.cs @@ -0,0 +1,20 @@ +using MailKit; +using MimeKit; + +namespace Wino.Core.Domain.Models.MailItem +{ + /// + /// Encapsulates all required information to create a MimeMessage for IMAP synchronizer. + /// + public class ImapMessageCreationPackage + { + public IMessageSummary MessageSummary { get; } + public MimeMessage MimeMessage { get; } + + public ImapMessageCreationPackage(IMessageSummary messageSummary, MimeMessage mimeMessage) + { + MessageSummary = messageSummary; + MimeMessage = mimeMessage; + } + } +} diff --git a/Wino.Core.Domain/Models/Synchronization/MailSynchronizationOptions.cs b/Wino.Core.Domain/Models/Synchronization/MailSynchronizationOptions.cs index acbf611a..a0130237 100644 --- a/Wino.Core.Domain/Models/Synchronization/MailSynchronizationOptions.cs +++ b/Wino.Core.Domain/Models/Synchronization/MailSynchronizationOptions.cs @@ -26,6 +26,12 @@ namespace Wino.Core.Domain.Models.Synchronization /// public List SynchronizationFolderIds { get; set; } + /// + /// If true, additional folders like Sent,Drafts and Deleted will not be synchronized + /// with InboxOnly and CustomFolders sync type. + /// + public bool ExcludeMustHaveFolders { get; set; } + /// /// When doing a linked inbox synchronization, we must ignore reporting completion to the caller for each folder. /// This Id will help tracking that. Id is unique, but this one can be the same for all sync requests @@ -33,6 +39,6 @@ namespace Wino.Core.Domain.Models.Synchronization /// public Guid? GroupedSynchronizationTrackingId { get; set; } - public override string ToString() => $"Type: {Type}, Folders: {(SynchronizationFolderIds == null ? "All" : string.Join(",", SynchronizationFolderIds))}"; + public override string ToString() => $"Type: {Type}"; } } diff --git a/Wino.Core/CoreContainerSetup.cs b/Wino.Core/CoreContainerSetup.cs index 4db0a3f0..4015d057 100644 --- a/Wino.Core/CoreContainerSetup.cs +++ b/Wino.Core/CoreContainerSetup.cs @@ -4,6 +4,7 @@ using Wino.Authentication; using Wino.Core.Domain.Interfaces; using Wino.Core.Integration.Processors; using Wino.Core.Services; +using Wino.Core.Synchronizers.ImapSync; namespace Wino.Core { @@ -28,6 +29,11 @@ namespace Wino.Core services.AddTransient(); services.AddTransient(); services.AddTransient(); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); } } } diff --git a/Wino.Core/Integration/ImapClientPool.cs b/Wino.Core/Integration/ImapClientPool.cs index b31ff6d2..761d6e7b 100644 --- a/Wino.Core/Integration/ImapClientPool.cs +++ b/Wino.Core/Integration/ImapClientPool.cs @@ -48,10 +48,11 @@ namespace Wino.Core.Integration public bool ThrowOnSSLHandshakeCallback { get; set; } public ImapClientPoolOptions ImapClientPoolOptions { get; } + internal WinoImapClient IdleClient { get; set; } private readonly int MinimumPoolSize = 5; - private readonly ConcurrentStack _clients = []; + private readonly ConcurrentStack _clients = []; private readonly SemaphoreSlim _semaphore; private readonly CustomServerInformation _customServerInformation; private readonly Stream _protocolLogStream; @@ -74,7 +75,7 @@ namespace Wino.Core.Integration /// Reconnects and reauthenticates if necessary. /// /// Whether the client has been newly created. - private async Task EnsureCapabilitiesAsync(ImapClient client, bool isCreatedNew) + private async Task EnsureCapabilitiesAsync(IImapClient client, bool isCreatedNew) { try { @@ -84,6 +85,9 @@ namespace Wino.Core.Integration if ((isCreatedNew || isReconnected) && client.IsConnected) { + if (client.Capabilities.HasFlag(ImapCapabilities.Compress)) + await client.CompressAsync(); + // Identify if the server supports ID extension. // Some servers require it pre-authentication, some post-authentication. // We'll observe the response here and do it after authentication if needed. @@ -113,10 +117,11 @@ namespace Wino.Core.Integration // Activate post-auth capabilities. if (client.Capabilities.HasFlag(ImapCapabilities.QuickResync)) - await client.EnableQuickResyncAsync(); + { + await client.EnableQuickResyncAsync().ConfigureAwait(false); - if (client.Capabilities.HasFlag(ImapCapabilities.Compress)) - await client.CompressAsync(); + if (client is WinoImapClient winoImapClient) winoImapClient.IsQResyncEnabled = true; + } } } catch (Exception ex) @@ -145,11 +150,11 @@ namespace Wino.Core.Integration return reader.ReadToEnd(); } - public async Task GetClientAsync() + public async Task GetClientAsync() { await _semaphore.WaitAsync(); - if (_clients.TryPop(out ImapClient item)) + if (_clients.TryPop(out IImapClient item)) { await EnsureCapabilitiesAsync(item, false); @@ -163,17 +168,21 @@ namespace Wino.Core.Integration return client; } - public void Release(ImapClient item, bool destroyClient = false) + public void Release(IImapClient item, bool destroyClient = false) { if (item != null) { if (destroyClient) { - lock (item.SyncRoot) + if (item.IsConnected) { - item.Disconnect(true); + lock (item.SyncRoot) + { + item.Disconnect(quit: true); + } } + _clients.TryPop(out _); item.Dispose(); } else @@ -185,23 +194,15 @@ namespace Wino.Core.Integration } } - public void DestroyClient(ImapClient client) + private IImapClient CreateNewClient() { - if (client == null) return; - - client.Disconnect(true); - client.Dispose(); - } - - private ImapClient CreateNewClient() - { - ImapClient client = null; + WinoImapClient client = null; // Make sure to create a ImapClient with a protocol logger if enabled. client = _protocolLogStream != null - ? new ImapClient(new ProtocolLogger(_protocolLogStream)) - : new ImapClient(); + ? new WinoImapClient(new ProtocolLogger(_protocolLogStream)) + : new WinoImapClient(); HttpProxyClient proxyClient = null; @@ -213,7 +214,7 @@ namespace Wino.Core.Integration client.ProxyClient = proxyClient; - _logger.Debug("Created new ImapClient. Current clients: {Count}", _clients.Count); + _logger.Debug("Creating new ImapClient. Current clients: {Count}", _clients.Count); return client; } @@ -229,7 +230,7 @@ namespace Wino.Core.Integration }; /// True if the connection is newly established. - public async Task EnsureConnectedAsync(ImapClient client) + public async Task EnsureConnectedAsync(IImapClient client) { if (client.IsConnected) return false; @@ -263,8 +264,20 @@ namespace Wino.Core.Integration { if (_protocolLogStream == null) return; - var messageBytes = Encoding.UTF8.GetBytes($"W: {message}\n"); - _protocolLogStream.Write(messageBytes, 0, messageBytes.Length); + try + { + var messageBytes = Encoding.UTF8.GetBytes($"W: {message}\n"); + _protocolLogStream.Write(messageBytes, 0, messageBytes.Length); + } + catch (ObjectDisposedException) + { + Log.Warning($"Protocol log stream is disposed. Cannot write to it."); + } + catch (Exception) + { + + throw; + } } bool MyServerCertificateValidationCallback(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors sslPolicyErrors) @@ -281,7 +294,7 @@ namespace Wino.Core.Integration return true; } - public async Task EnsureAuthenticatedAsync(ImapClient client) + public async Task EnsureAuthenticatedAsync(IImapClient client) { if (client.IsAuthenticated) return; @@ -336,10 +349,7 @@ namespace Wino.Core.Integration _clients.Clear(); - if (_protocolLogStream != null) - { - _protocolLogStream.Dispose(); - } + _protocolLogStream?.Dispose(); } } } diff --git a/Wino.Core/Integration/WinoImapClient.cs b/Wino.Core/Integration/WinoImapClient.cs new file mode 100644 index 00000000..2d471d90 --- /dev/null +++ b/Wino.Core/Integration/WinoImapClient.cs @@ -0,0 +1,60 @@ +using MailKit; +using MailKit.Net.Imap; +using Serilog; + +namespace Wino.Core.Integration +{ + /// + /// Extended class for ImapClient that is used in Wino. + /// + internal class WinoImapClient : ImapClient + { + /// + /// Gets or internally sets whether the QRESYNC extension is enabled. + /// It is set by ImapClientPool immidiately after the authentication. + /// + public bool IsQResyncEnabled { get; internal set; } + + public WinoImapClient() + { + HookEvents(); + } + + public WinoImapClient(IProtocolLogger protocolLogger) : base(protocolLogger) + { + HookEvents(); + } + + private void HookEvents() + { + Disconnected += ClientDisconnected; + } + + private void UnhookEvents() + { + Disconnected -= ClientDisconnected; + } + + private void ClientDisconnected(object sender, DisconnectedEventArgs e) + { + if (e.IsRequested) + { + Log.Debug("Imap client is disconnected on request."); + } + else + { + Log.Debug("Imap client connection is dropped by server."); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + + if (disposing) + { + UnhookEvents(); + } + } + } +} diff --git a/Wino.Core/Mime/ImapMessageCreationPackage.cs b/Wino.Core/Mime/ImapMessageCreationPackage.cs deleted file mode 100644 index ae482d98..00000000 --- a/Wino.Core/Mime/ImapMessageCreationPackage.cs +++ /dev/null @@ -1,9 +0,0 @@ -using MailKit; - -namespace Wino.Core.Mime -{ - /// - /// Encapsulates all required information to create a MimeMessage for IMAP synchronizer. - /// - public record ImapMessageCreationPackage(IMessageSummary MessageSummary, IMailFolder MailFolder); -} diff --git a/Wino.Core/Requests/Bundles/TaskRequestBundle.cs b/Wino.Core/Requests/Bundles/TaskRequestBundle.cs index 96e0bb69..0aa8c903 100644 --- a/Wino.Core/Requests/Bundles/TaskRequestBundle.cs +++ b/Wino.Core/Requests/Bundles/TaskRequestBundle.cs @@ -7,10 +7,10 @@ namespace Wino.Core.Requests.Bundles { public class ImapRequest { - public Func IntegratorTask { get; } + public Func IntegratorTask { get; } public IRequestBase Request { get; } - public ImapRequest(Func integratorTask, IRequestBase request) + public ImapRequest(Func integratorTask, IRequestBase request) { IntegratorTask = integratorTask; Request = request; @@ -19,7 +19,7 @@ namespace Wino.Core.Requests.Bundles public class ImapRequest : ImapRequest where TRequestBaseType : IRequestBase { - public ImapRequest(Func integratorTask, TRequestBaseType request) + public ImapRequest(Func integratorTask, TRequestBaseType request) : base((client, request) => integratorTask(client, (TRequestBaseType)request), request) { } diff --git a/Wino.Core/Requests/Folder/EmptyFolderRequest.cs b/Wino.Core/Requests/Folder/EmptyFolderRequest.cs index 8ae559c2..f4d434c7 100644 --- a/Wino.Core/Requests/Folder/EmptyFolderRequest.cs +++ b/Wino.Core/Requests/Folder/EmptyFolderRequest.cs @@ -11,6 +11,7 @@ namespace Wino.Core.Requests.Folder { public record EmptyFolderRequest(MailItemFolder Folder, List MailsToDelete) : FolderRequestBase(Folder, FolderSynchronizerOperation.EmptyFolder), ICustomFolderSynchronizationRequest { + public bool ExcludeMustHaveFolders => false; public override void ApplyUIChanges() { foreach (var item in MailsToDelete) diff --git a/Wino.Core/Requests/Folder/MarkFolderAsReadRequest.cs b/Wino.Core/Requests/Folder/MarkFolderAsReadRequest.cs index 17e989b2..4aee879d 100644 --- a/Wino.Core/Requests/Folder/MarkFolderAsReadRequest.cs +++ b/Wino.Core/Requests/Folder/MarkFolderAsReadRequest.cs @@ -32,5 +32,7 @@ namespace Wino.Core.Requests.Folder } public List SynchronizationFolderIds => [Folder.Id]; + + public bool ExcludeMustHaveFolders => true; } } diff --git a/Wino.Core/Requests/Mail/ArchiveRequest.cs b/Wino.Core/Requests/Mail/ArchiveRequest.cs index bc37bff3..94d6275c 100644 --- a/Wino.Core/Requests/Mail/ArchiveRequest.cs +++ b/Wino.Core/Requests/Mail/ArchiveRequest.cs @@ -21,6 +21,7 @@ namespace Wino.Core.Requests.Mail public record ArchiveRequest(bool IsArchiving, MailCopy Item, MailItemFolder FromFolder, MailItemFolder ToFolder = null) : MailRequestBase(Item), ICustomFolderSynchronizationRequest { + public bool ExcludeMustHaveFolders => false; public List SynchronizationFolderIds { get diff --git a/Wino.Core/Requests/Mail/ChangeFlagRequest.cs b/Wino.Core/Requests/Mail/ChangeFlagRequest.cs index 56789c31..53a25602 100644 --- a/Wino.Core/Requests/Mail/ChangeFlagRequest.cs +++ b/Wino.Core/Requests/Mail/ChangeFlagRequest.cs @@ -14,6 +14,8 @@ namespace Wino.Core.Requests.Mail { public List SynchronizationFolderIds => [Item.FolderId]; + public bool ExcludeMustHaveFolders => true; + public override MailSynchronizerOperation Operation => MailSynchronizerOperation.ChangeFlag; public override void ApplyUIChanges() diff --git a/Wino.Core/Requests/Mail/CreateDraftRequest.cs b/Wino.Core/Requests/Mail/CreateDraftRequest.cs index beb09fa6..2c77f802 100644 --- a/Wino.Core/Requests/Mail/CreateDraftRequest.cs +++ b/Wino.Core/Requests/Mail/CreateDraftRequest.cs @@ -13,6 +13,8 @@ namespace Wino.Core.Requests.Mail : MailRequestBase(DraftPreperationRequest.CreatedLocalDraftCopy), ICustomFolderSynchronizationRequest { + public bool ExcludeMustHaveFolders => false; + public List SynchronizationFolderIds => [ DraftPreperationRequest.CreatedLocalDraftCopy.AssignedFolder.Id diff --git a/Wino.Core/Requests/Mail/DeleteRequest.cs b/Wino.Core/Requests/Mail/DeleteRequest.cs index de6e9ffb..ce7295db 100644 --- a/Wino.Core/Requests/Mail/DeleteRequest.cs +++ b/Wino.Core/Requests/Mail/DeleteRequest.cs @@ -17,7 +17,7 @@ namespace Wino.Core.Requests.Mail ICustomFolderSynchronizationRequest { public List SynchronizationFolderIds => [Item.FolderId]; - + public bool ExcludeMustHaveFolders => false; public override MailSynchronizerOperation Operation => MailSynchronizerOperation.Delete; public override void ApplyUIChanges() diff --git a/Wino.Core/Requests/Mail/MarkReadRequest.cs b/Wino.Core/Requests/Mail/MarkReadRequest.cs index e5b80e44..c3f18a8d 100644 --- a/Wino.Core/Requests/Mail/MarkReadRequest.cs +++ b/Wino.Core/Requests/Mail/MarkReadRequest.cs @@ -15,6 +15,8 @@ namespace Wino.Core.Requests.Mail public override MailSynchronizerOperation Operation => MailSynchronizerOperation.MarkRead; + public bool ExcludeMustHaveFolders => true; + public override void ApplyUIChanges() { Item.IsRead = IsRead; diff --git a/Wino.Core/Requests/Mail/MoveRequest.cs b/Wino.Core/Requests/Mail/MoveRequest.cs index 8fc198ef..489e0d02 100644 --- a/Wino.Core/Requests/Mail/MoveRequest.cs +++ b/Wino.Core/Requests/Mail/MoveRequest.cs @@ -13,7 +13,7 @@ namespace Wino.Core.Requests.Mail : MailRequestBase(Item), ICustomFolderSynchronizationRequest { public List SynchronizationFolderIds => new() { FromFolder.Id, ToFolder.Id }; - + public bool ExcludeMustHaveFolders => false; public override MailSynchronizerOperation Operation => MailSynchronizerOperation.Move; public override void ApplyUIChanges() diff --git a/Wino.Core/Requests/Mail/SendDraftRequest.cs b/Wino.Core/Requests/Mail/SendDraftRequest.cs index 0dfc56fe..621f269a 100644 --- a/Wino.Core/Requests/Mail/SendDraftRequest.cs +++ b/Wino.Core/Requests/Mail/SendDraftRequest.cs @@ -28,6 +28,8 @@ namespace Wino.Core.Requests.Mail } } + public bool ExcludeMustHaveFolders => false; + public override MailSynchronizerOperation Operation => MailSynchronizerOperation.Send; public override void ApplyUIChanges() diff --git a/Wino.Core/Services/SynchronizerFactory.cs b/Wino.Core/Services/SynchronizerFactory.cs index 62814e5d..e5f15487 100644 --- a/Wino.Core/Services/SynchronizerFactory.cs +++ b/Wino.Core/Services/SynchronizerFactory.cs @@ -13,6 +13,7 @@ namespace Wino.Core.Services private bool isInitialized = false; private readonly IAccountService _accountService; + private readonly IImapSynchronizationStrategyProvider _imapSynchronizationStrategyProvider; private readonly IApplicationConfiguration _applicationConfiguration; private readonly IOutlookChangeProcessor _outlookChangeProcessor; private readonly IGmailChangeProcessor _gmailChangeProcessor; @@ -28,6 +29,7 @@ namespace Wino.Core.Services IOutlookAuthenticator outlookAuthenticator, IGmailAuthenticator gmailAuthenticator, IAccountService accountService, + IImapSynchronizationStrategyProvider imapSynchronizationStrategyProvider, IApplicationConfiguration applicationConfiguration) { _outlookChangeProcessor = outlookChangeProcessor; @@ -36,6 +38,7 @@ namespace Wino.Core.Services _outlookAuthenticator = outlookAuthenticator; _gmailAuthenticator = gmailAuthenticator; _accountService = accountService; + _imapSynchronizationStrategyProvider = imapSynchronizationStrategyProvider; _applicationConfiguration = applicationConfiguration; } @@ -51,6 +54,7 @@ namespace Wino.Core.Services { synchronizer = CreateNewSynchronizer(account); + return await GetAccountSynchronizerAsync(accountId); } } @@ -70,7 +74,7 @@ namespace Wino.Core.Services case Domain.Enums.MailProviderType.Gmail: return new GmailSynchronizer(mailAccount, _gmailAuthenticator, _gmailChangeProcessor); case Domain.Enums.MailProviderType.IMAP4: - return new ImapSynchronizer(mailAccount, _imapChangeProcessor, _applicationConfiguration); + return new ImapSynchronizer(mailAccount, _imapChangeProcessor, _imapSynchronizationStrategyProvider, _applicationConfiguration); default: break; } @@ -82,6 +86,12 @@ namespace Wino.Core.Services { var synchronizer = CreateIntegratorWithDefaultProcessor(account); + if (synchronizer is IImapSynchronizer imapSynchronizer) + { + // Start the idle client for IMAP synchronizer. + _ = imapSynchronizer.StartIdleClientAsync(); + } + synchronizerCache.Add(synchronizer); return synchronizer; diff --git a/Wino.Core/Synchronizers/ImapSync/CondstoreSynchronizer.cs b/Wino.Core/Synchronizers/ImapSync/CondstoreSynchronizer.cs new file mode 100644 index 00000000..414fd94c --- /dev/null +++ b/Wino.Core/Synchronizers/ImapSync/CondstoreSynchronizer.cs @@ -0,0 +1,152 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MailKit; +using MailKit.Net.Imap; +using MailKit.Search; +using Serilog; +using Wino.Core.Domain.Entities.Mail; +using Wino.Core.Domain.Exceptions; +using Wino.Core.Domain.Interfaces; +using Wino.Core.Domain.Models.MailItem; +using Wino.Core.Integration; +using Wino.Services.Extensions; +using IMailService = Wino.Core.Domain.Interfaces.IMailService; + +namespace Wino.Core.Synchronizers.ImapSync +{ + /// + /// RFC 4551 CONDSTORE IMAP Synchronization strategy. + /// + internal class CondstoreSynchronizer : ImapSynchronizationStrategyBase + { + public CondstoreSynchronizer(IFolderService folderService, IMailService mailService) : base(folderService, mailService) + { + } + + public async override Task> HandleSynchronizationAsync(IImapClient client, + MailItemFolder folder, + IImapSynchronizer synchronizer, + CancellationToken cancellationToken = default) + { + if (client is not WinoImapClient winoClient) + throw new ArgumentException("Client must be of type WinoImapClient.", nameof(client)); + + if (!client.Capabilities.HasFlag(ImapCapabilities.CondStore)) + throw new ImapSynchronizerStrategyException("Server does not support CONDSTORE."); + + IMailFolder remoteFolder = null; + + var downloadedMessageIds = new List(); + try + { + remoteFolder = await winoClient.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false); + + await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false); + + var localHighestModSeq = (ulong)folder.HighestModeSeq; + var remoteHighestModSeq = remoteFolder.HighestModSeq; + + // There are some changes on new messages or flag changes. + // Deletions are tracked separately because some servers do not increase + // the MODSEQ value for deleted messages. + if (remoteHighestModSeq > localHighestModSeq) + { + // Search for emails with a MODSEQ greater than the last known value + var changedUids = await remoteFolder.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq)).ConfigureAwait(false); + + // Get locally exists mails for the returned UIDs. + var existingMails = await MailService.GetExistingMailsAsync(folder.Id, changedUids); + var existingMailUids = existingMails.Select(m => MailkitClientExtensions.ResolveUidStruct(m.Id)).ToArray(); + + // These are the non-existing mails. They will be downloaded + processed. + var newMessageIds = changedUids.Except(existingMailUids).ToList(); + + // Fetch minimum data for the existing mails in one query. + var existingFlagData = await remoteFolder.FetchAsync(existingMailUids, MessageSummaryItems.Flags | MessageSummaryItems.UniqueId).ConfigureAwait(false); + + foreach (var update in existingFlagData) + { + if (update.UniqueId == null) + { + Log.Warning($"Couldn't fetch UniqueId for the mail. FetchAsync failed."); + continue; + } + + if (update.Flags == null) + { + Log.Warning($"Couldn't fetch flags for the mail with UID {update.UniqueId.Id}. FetchAsync failed."); + continue; + } + + var existingMail = existingMails.FirstOrDefault(m => MailkitClientExtensions.ResolveUidStruct(m.Id).Id == update.UniqueId.Id); + + if (existingMail == null) + { + Log.Warning($"Couldn't find the mail with UID {update.UniqueId.Id} in the local database. Flag update is ignored."); + continue; + } + + await HandleMessageFlagsChangeAsync(existingMail, update.Flags.Value).ConfigureAwait(false); + } + + // Fetch the new mails. + var summaries = await remoteFolder.FetchAsync(newMessageIds, MailSynchronizationFlags, cancellationToken).ConfigureAwait(false); + + foreach (var summary in summaries) + { + var mimeMessage = await remoteFolder.GetMessageAsync(summary.UniqueId, cancellationToken).ConfigureAwait(false); + + var creationPackage = new ImapMessageCreationPackage(summary, mimeMessage); + + var mailPackages = await synchronizer.CreateNewMailPackagesAsync(creationPackage, folder, cancellationToken).ConfigureAwait(false); + + if (mailPackages != null) + { + foreach (var package in mailPackages) + { + // Local draft is mapped. We don't need to create a new mail copy. + if (package == null) continue; + + bool isCreatedNew = await MailService.CreateMailAsync(folder.MailAccountId, package).ConfigureAwait(false); + + // This is upsert. We are not interested in updated mails. + if (isCreatedNew) downloadedMessageIds.Add(package.Copy.Id); + } + } + } + + folder.HighestModeSeq = (long)remoteHighestModSeq; + + await FolderService.UpdateFolderAsync(folder).ConfigureAwait(false); + } + + await ManageUUIdBasedDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false); + + return downloadedMessageIds; + } + catch (FolderNotFoundException) + { + await FolderService.DeleteFolderAsync(folder.MailAccountId, folder.RemoteFolderId).ConfigureAwait(false); + + return default; + } + catch (Exception) + { + throw; + } + finally + { + if (remoteFolder != null) + { + if (remoteFolder.IsOpen) + { + await remoteFolder.CloseAsync().ConfigureAwait(false); + } + } + } + } + } +} diff --git a/Wino.Core/Synchronizers/ImapSync/ImapSynchronizationStrategyBase.cs b/Wino.Core/Synchronizers/ImapSync/ImapSynchronizationStrategyBase.cs new file mode 100644 index 00000000..7f721a68 --- /dev/null +++ b/Wino.Core/Synchronizers/ImapSync/ImapSynchronizationStrategyBase.cs @@ -0,0 +1,99 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MailKit; +using MailKit.Net.Imap; +using MailKit.Search; +using Wino.Core.Domain.Entities.Mail; +using Wino.Core.Domain.Interfaces; +using Wino.Services.Extensions; +using IMailService = Wino.Core.Domain.Interfaces.IMailService; + +namespace Wino.Core.Synchronizers.ImapSync +{ + public abstract class ImapSynchronizationStrategyBase : IImapSynchronizerStrategy + { + // Minimum summary items to Fetch for mail synchronization from IMAP. + protected readonly MessageSummaryItems MailSynchronizationFlags = + MessageSummaryItems.Flags | + MessageSummaryItems.UniqueId | + MessageSummaryItems.ThreadId | + MessageSummaryItems.EmailId | + MessageSummaryItems.Headers | + MessageSummaryItems.PreviewText | + MessageSummaryItems.GMailThreadId | + MessageSummaryItems.References | + MessageSummaryItems.ModSeq; + + protected IFolderService FolderService { get; } + protected IMailService MailService { get; } + + protected ImapSynchronizationStrategyBase(IFolderService folderService, IMailService mailService) + { + FolderService = folderService; + MailService = mailService; + } + + public abstract Task> HandleSynchronizationAsync(IImapClient client, MailItemFolder folder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default); + + protected async Task HandleMessageFlagsChangeAsync(MailItemFolder folder, UniqueId? uniqueId, MessageFlags flags) + { + if (folder == null) return; + if (uniqueId == null) return; + + var localMailCopyId = MailkitClientExtensions.CreateUid(folder.Id, uniqueId.Value.Id); + + var isFlagged = MailkitClientExtensions.GetIsFlagged(flags); + var isRead = MailkitClientExtensions.GetIsRead(flags); + + await MailService.ChangeReadStatusAsync(localMailCopyId, isRead).ConfigureAwait(false); + await MailService.ChangeFlagStatusAsync(localMailCopyId, isFlagged).ConfigureAwait(false); + } + + protected async Task HandleMessageFlagsChangeAsync(MailCopy mailCopy, MessageFlags flags) + { + if (mailCopy == null) return; + + var isFlagged = MailkitClientExtensions.GetIsFlagged(flags); + var isRead = MailkitClientExtensions.GetIsRead(flags); + + if (isFlagged != mailCopy.IsFlagged) + { + await MailService.ChangeFlagStatusAsync(mailCopy.Id, isFlagged).ConfigureAwait(false); + } + + if (isRead != mailCopy.IsRead) + { + await MailService.ChangeReadStatusAsync(mailCopy.Id, isRead).ConfigureAwait(false); + } + } + + protected async Task HandleMessageDeletedAsync(MailItemFolder folder, IList uniqueIds) + { + if (folder == null) return; + if (uniqueIds == null || uniqueIds.Count == 0) return; + + foreach (var uniqueId in uniqueIds) + { + if (uniqueId == null) continue; + var localMailCopyId = MailkitClientExtensions.CreateUid(folder.Id, uniqueId.Id); + + await MailService.DeleteMailAsync(folder.MailAccountId, localMailCopyId).ConfigureAwait(false); + } + } + + protected async Task ManageUUIdBasedDeletedMessagesAsync(MailItemFolder localFolder, IMailFolder remoteFolder, CancellationToken cancellationToken = default) + { + var allUids = (await FolderService.GetKnownUidsForFolderAsync(localFolder.Id)).Select(a => new UniqueId(a)).ToList(); + + if (allUids.Count > 0) + { + var remoteAllUids = await remoteFolder.SearchAsync(SearchQuery.All, cancellationToken); + var deletedUids = allUids.Except(remoteAllUids).ToList(); + + await HandleMessageDeletedAsync(localFolder, deletedUids).ConfigureAwait(false); + } + } + } +} diff --git a/Wino.Core/Synchronizers/ImapSync/ImapSynchronizationStrategyProvider.cs b/Wino.Core/Synchronizers/ImapSync/ImapSynchronizationStrategyProvider.cs new file mode 100644 index 00000000..b6fef92c --- /dev/null +++ b/Wino.Core/Synchronizers/ImapSync/ImapSynchronizationStrategyProvider.cs @@ -0,0 +1,31 @@ +using MailKit.Net.Imap; +using Wino.Core.Domain.Interfaces; +using Wino.Core.Integration; + +namespace Wino.Core.Synchronizers.ImapSync +{ + internal class ImapSynchronizationStrategyProvider : IImapSynchronizationStrategyProvider + { + private readonly QResyncSynchronizer _qResyncSynchronizer; + private readonly CondstoreSynchronizer _condstoreSynchronizer; + private readonly UidBasedSynchronizer _uidBasedSynchronizer; + + public ImapSynchronizationStrategyProvider(QResyncSynchronizer qResyncSynchronizer, CondstoreSynchronizer condstoreSynchronizer, UidBasedSynchronizer uidBasedSynchronizer) + { + _qResyncSynchronizer = qResyncSynchronizer; + _condstoreSynchronizer = condstoreSynchronizer; + _uidBasedSynchronizer = uidBasedSynchronizer; + } + + public IImapSynchronizerStrategy GetSynchronizationStrategy(IImapClient client) + { + if (client is not WinoImapClient winoImapClient) + throw new System.ArgumentException("Client must be of type WinoImapClient.", nameof(client)); + + // if (client.Capabilities.HasFlag(ImapCapabilities.QuickResync) && winoImapClient.IsQResyncEnabled) return _qResyncSynchronizer; + if (client.Capabilities.HasFlag(ImapCapabilities.CondStore)) return _condstoreSynchronizer; + + return _uidBasedSynchronizer; + } + } +} diff --git a/Wino.Core/Synchronizers/ImapSync/QResyncSynchronizer.cs b/Wino.Core/Synchronizers/ImapSync/QResyncSynchronizer.cs new file mode 100644 index 00000000..356a9b9a --- /dev/null +++ b/Wino.Core/Synchronizers/ImapSync/QResyncSynchronizer.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MailKit; +using MailKit.Net.Imap; +using MailKit.Search; +using Wino.Core.Domain.Entities.Mail; +using Wino.Core.Domain.Exceptions; +using Wino.Core.Domain.Interfaces; +using Wino.Core.Integration; +using IMailService = Wino.Core.Domain.Interfaces.IMailService; + +namespace Wino.Core.Synchronizers.ImapSync +{ + /// + /// RFC 5162 QRESYNC IMAP Synchronization strategy. + /// + internal class QResyncSynchronizer : ImapSynchronizationStrategyBase + { + public QResyncSynchronizer(IFolderService folderService, IMailService mailService) : base(folderService, mailService) + { + } + + public override async Task> HandleSynchronizationAsync(IImapClient client, + MailItemFolder folder, + IImapSynchronizer synchronizer, + CancellationToken cancellationToken = default) + { + if (client is not WinoImapClient winoClient) + throw new ImapSynchronizerStrategyException("Client must be of type WinoImapClient."); + + if (!client.Capabilities.HasFlag(ImapCapabilities.QuickResync)) + throw new ImapSynchronizerStrategyException("Server does not support QRESYNC."); + + if (!winoClient.IsQResyncEnabled) + throw new ImapSynchronizerStrategyException("QRESYNC is not enabled for WinoImapClient."); + + // Ready to implement QRESYNC synchronization. + + IMailFolder remoteFolder = null; + + try + { + remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false); + + // Check the Uid validity first. + // If they don't match, clear all the local data and perform full-resync. + + bool isCacheValid = remoteFolder.UidValidity == folder.UidValidity; + + if (!isCacheValid) + { + // TODO: Remove all local data. + + } + + // Perform QRESYNC synchronization. + + var remoteHighestModSeq = remoteFolder.HighestModSeq; + var localHighestModSeq = (ulong)folder.HighestModeSeq; + + remoteFolder.MessagesVanished += async (c, r) => await HandleMessageDeletedAsync(folder, r.UniqueIds).ConfigureAwait(false); + remoteFolder.MessageFlagsChanged += async (c, r) => await HandleMessageFlagsChangeAsync(folder, r.UniqueId, r.Flags).ConfigureAwait(false); + + var allUids = await FolderService.GetKnownUidsForFolderAsync(folder.Id); + var allUniqueIds = allUids.Select(a => new UniqueId(a)).ToList(); + + await remoteFolder.OpenAsync(FolderAccess.ReadOnly, folder.UidValidity, localHighestModSeq, allUniqueIds).ConfigureAwait(false); + + var changedUids = await remoteFolder.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq), cancellationToken).ConfigureAwait(false); + + foreach (var uid in changedUids) + { + Debug.WriteLine($"Processing message with UID: {uid}"); + + // var message = await remoteFolder.GetMessageAsync(uid, cancellationToken).ConfigureAwait(false); + + // TODO: Process the message. + } + + // Update the local folder with the new highest mod-seq and validity. + folder.HighestModeSeq = (long)remoteHighestModSeq; + folder.UidValidity = remoteFolder.UidValidity; + + await FolderService.UpdateFolderAsync(folder).ConfigureAwait(false); + } + catch (Exception ex) + { + throw; + } + finally + { + if (remoteFolder != null) + { + remoteFolder.MessagesVanished -= async (c, r) => await HandleMessageDeletedAsync(folder, r.UniqueIds).ConfigureAwait(false); + remoteFolder.MessageFlagsChanged -= async (c, r) => await HandleMessageFlagsChangeAsync(folder, r.UniqueId, r.Flags).ConfigureAwait(false); + + if (remoteFolder.IsOpen) + { + await remoteFolder.CloseAsync(); + } + } + } + + return default; + } + } +} diff --git a/Wino.Core/Synchronizers/ImapSync/UidBasedSynchronizer.cs b/Wino.Core/Synchronizers/ImapSync/UidBasedSynchronizer.cs new file mode 100644 index 00000000..bec3f93d --- /dev/null +++ b/Wino.Core/Synchronizers/ImapSync/UidBasedSynchronizer.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using MailKit.Net.Imap; +using Wino.Core.Domain.Entities.Mail; +using Wino.Core.Domain.Interfaces; +using Wino.Core.Integration; + +namespace Wino.Core.Synchronizers.ImapSync +{ + /// + /// Uid based IMAP Synchronization strategy. + /// + internal class UidBasedSynchronizer : IImapSynchronizerStrategy + { + public Task> HandleSynchronizationAsync(IImapClient client, MailItemFolder folder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default) + { + if (client is not WinoImapClient winoClient) + throw new ArgumentException("Client must be of type WinoImapClient.", nameof(client)); + + return Task.FromResult(new List()); + } + } +} diff --git a/Wino.Core/Synchronizers/ImapSynchronizer.cs b/Wino.Core/Synchronizers/ImapSynchronizer.cs index eed08707..0a3a94e2 100644 --- a/Wino.Core/Synchronizers/ImapSynchronizer.cs +++ b/Wino.Core/Synchronizers/ImapSynchronizer.cs @@ -7,7 +7,6 @@ using System.Threading.Tasks; using CommunityToolkit.Mvvm.Messaging; using MailKit; using MailKit.Net.Imap; -using MailKit.Search; using MimeKit; using MoreLinq; using Serilog; @@ -22,7 +21,6 @@ using Wino.Core.Domain.Models.Synchronization; using Wino.Core.Extensions; using Wino.Core.Integration; using Wino.Core.Integration.Processors; -using Wino.Core.Mime; using Wino.Core.Requests.Bundles; using Wino.Core.Requests.Folder; using Wino.Core.Requests.Mail; @@ -31,56 +29,37 @@ using Wino.Services.Extensions; namespace Wino.Core.Synchronizers.Mail { - public class ImapSynchronizer : WinoSynchronizer + public class ImapSynchronizer : WinoSynchronizer, IImapSynchronizer { - private CancellationTokenSource idleDoneToken; - private CancellationTokenSource cancelInboxListeningToken = new CancellationTokenSource(); + public override uint BatchModificationSize => 1000; + public override uint InitialMessageDownloadCountPerFolder => 250; - private IMailFolder inboxFolder; + #region Idle Implementation + + private CancellationTokenSource idleCancellationTokenSource; + private CancellationTokenSource idleDoneTokenSource; + + #endregion private readonly ILogger _logger = Log.ForContext(); private readonly ImapClientPool _clientPool; private readonly IImapChangeProcessor _imapChangeProcessor; + private readonly IImapSynchronizationStrategyProvider _imapSynchronizationStrategyProvider; private readonly IApplicationConfiguration _applicationConfiguration; - // Minimum summary items to Fetch for mail synchronization from IMAP. - private readonly MessageSummaryItems mailSynchronizationFlags = - MessageSummaryItems.Flags | - MessageSummaryItems.UniqueId | - MessageSummaryItems.ThreadId | - MessageSummaryItems.EmailId | - MessageSummaryItems.Headers | - MessageSummaryItems.PreviewText | - MessageSummaryItems.GMailThreadId | - MessageSummaryItems.References | - MessageSummaryItems.ModSeq; - - /// - /// Timer that keeps the alive for the lifetime of the pool. - /// Sends NOOP command to the server periodically. - /// - private Timer _noOpTimer; - - /// - /// ImapClient that keeps the Inbox folder opened all the time for listening notifications. - /// - private ImapClient _inboxIdleClient; - - public override uint BatchModificationSize => 1000; - public override uint InitialMessageDownloadCountPerFolder => 250; - public ImapSynchronizer(MailAccount account, IImapChangeProcessor imapChangeProcessor, + IImapSynchronizationStrategyProvider imapSynchronizationStrategyProvider, IApplicationConfiguration applicationConfiguration) : base(account) { // Create client pool with account protocol log. _imapChangeProcessor = imapChangeProcessor; + _imapSynchronizationStrategyProvider = imapSynchronizationStrategyProvider; _applicationConfiguration = applicationConfiguration; var poolOptions = ImapClientPoolOptions.CreateDefault(Account.ServerInformation, CreateAccountProtocolLogFileStream()); _clientPool = new ImapClientPool(poolOptions); - idleDoneToken = new CancellationTokenSource(); } private Stream CreateAccountProtocolLogFileStream() @@ -95,136 +74,10 @@ namespace Wino.Core.Synchronizers.Mail return new FileStream(logFile, FileMode.CreateNew); } - // TODO - // private async void NoOpTimerTriggered(object state) => await AwaitInboxIdleAsync(); - - private async Task AwaitInboxIdleAsync() - { - if (_inboxIdleClient == null) - { - _logger.Warning("InboxClient is null. Cannot send NOOP command."); - return; - } - - await _clientPool.EnsureConnectedAsync(_inboxIdleClient); - await _clientPool.EnsureAuthenticatedAsync(_inboxIdleClient); - - try - { - if (inboxFolder == null) - { - inboxFolder = _inboxIdleClient.Inbox; - await inboxFolder.OpenAsync(FolderAccess.ReadOnly, cancelInboxListeningToken.Token); - } - - idleDoneToken = new CancellationTokenSource(); - - await _inboxIdleClient.IdleAsync(idleDoneToken.Token, cancelInboxListeningToken.Token); - } - finally - { - idleDoneToken.Dispose(); - idleDoneToken = null; - } - } - - private async Task StopInboxListeningAsync() - { - if (inboxFolder != null) - { - inboxFolder.CountChanged -= InboxFolderCountChanged; - inboxFolder.MessageExpunged -= InboxFolderMessageExpunged; - inboxFolder.MessageFlagsChanged -= InboxFolderMessageFlagsChanged; - } - - if (_noOpTimer != null) - { - _noOpTimer.Dispose(); - _noOpTimer = null; - } - - if (idleDoneToken != null) - { - idleDoneToken.Cancel(); - idleDoneToken.Dispose(); - idleDoneToken = null; - } - - if (_inboxIdleClient != null) - { - await _inboxIdleClient.DisconnectAsync(true); - _inboxIdleClient.Dispose(); - _inboxIdleClient = null; - } - } - - /// - /// Tries to connect & authenticate with the given credentials. - /// Prepares synchronizer for active listening of Inbox folder. - /// - public async Task StartInboxListeningAsync() - { - _inboxIdleClient = await _clientPool.GetClientAsync(); - - // Run it every 8 minutes after 1 minute delay. - // _noOpTimer = new Timer(NoOpTimerTriggered, null, 60000, 8 * 60 * 1000); - - await _clientPool.EnsureConnectedAsync(_inboxIdleClient); - await _clientPool.EnsureAuthenticatedAsync(_inboxIdleClient); - - if (!_inboxIdleClient.Capabilities.HasFlag(ImapCapabilities.Idle)) - { - _logger.Information("Imap server does not support IDLE command. Listening live changes is not supported for {Name}", Account.Name); - return; - } - - inboxFolder = _inboxIdleClient.Inbox; - - if (inboxFolder == null) - { - _logger.Information("Inbox folder is null. Cannot listen for changes."); - return; - } - - inboxFolder.CountChanged += InboxFolderCountChanged; - inboxFolder.MessageExpunged += InboxFolderMessageExpunged; - inboxFolder.MessageFlagsChanged += InboxFolderMessageFlagsChanged; - - while (!cancelInboxListeningToken.IsCancellationRequested) - { - await AwaitInboxIdleAsync(); - } - - await StopInboxListeningAsync(); - } - - private void InboxFolderMessageFlagsChanged(object sender, MessageFlagsChangedEventArgs e) - { - Console.WriteLine("Flags have changed for message #{0} ({1}).", e.Index, e.Flags); - } - - private void InboxFolderMessageExpunged(object sender, MessageEventArgs e) - { - _logger.Information("Inbox folder message expunged"); - } - - private void InboxFolderCountChanged(object sender, EventArgs e) - { - _logger.Information("Inbox folder count changed."); - } - - /// - /// Parses List of string of mail copy ids and return valid uIds. - /// Follow the rules for creating arbitrary unique id for mail copies. - /// - private UniqueIdSet GetUniqueIds(IEnumerable mailCopyIds) - => new(mailCopyIds.Select(a => new UniqueId(MailkitClientExtensions.ResolveUid(a)))); - /// /// Returns UniqueId for the given mail copy id. /// - private UniqueId GetUniqueId(string mailCopyId) - => new(MailkitClientExtensions.ResolveUid(mailCopyId)); + private UniqueId GetUniqueId(string mailCopyId) => new(MailkitClientExtensions.ResolveUid(mailCopyId)); #region Mail Integrations @@ -400,11 +253,7 @@ namespace Wino.Core.Synchronizers.Mail public override async Task> CreateNewMailPackagesAsync(ImapMessageCreationPackage message, MailItemFolder assignedFolder, CancellationToken cancellationToken = default) { - var imapFolder = message.MailFolder; - var summary = message.MessageSummary; - - var mimeMessage = await imapFolder.GetMessageAsync(summary.UniqueId, cancellationToken).ConfigureAwait(false); - var mailCopy = summary.GetMailDetails(assignedFolder, mimeMessage); + var mailCopy = message.MessageSummary.GetMailDetails(assignedFolder, message.MimeMessage); // Draft folder message updates must be updated as IsDraft. // I couldn't find it in MimeMessage... @@ -414,8 +263,8 @@ namespace Wino.Core.Synchronizers.Mail // Check draft mapping. // This is the same implementation as in the OutlookSynchronizer. - if (mimeMessage.Headers.Contains(Domain.Constants.WinoLocalDraftHeader) - && Guid.TryParse(mimeMessage.Headers[Domain.Constants.WinoLocalDraftHeader], out Guid localDraftCopyUniqueId)) + if (message.MimeMessage.Headers.Contains(Domain.Constants.WinoLocalDraftHeader) + && Guid.TryParse(message.MimeMessage.Headers[Domain.Constants.WinoLocalDraftHeader], out Guid localDraftCopyUniqueId)) { // This message belongs to existing local draft copy. // We don't need to create a new mail copy for this message, just update the existing one. @@ -427,7 +276,7 @@ namespace Wino.Core.Synchronizers.Mail // Local copy doesn't exists. Continue execution to insert mail copy. } - var package = new NewMailItemPackage(mailCopy, mimeMessage, assignedFolder.RemoteFolderId); + var package = new NewMailItemPackage(mailCopy, message.MimeMessage, assignedFolder.RemoteFolderId); return [ @@ -463,6 +312,7 @@ namespace Wino.Core.Synchronizers.Mail PublishSynchronizationProgress(progress); var folderDownloadedMessageIds = await SynchronizeFolderInternalAsync(folder, cancellationToken).ConfigureAwait(false); + downloadedMessageIds.AddRange(folderDownloadedMessageIds); } } @@ -497,7 +347,7 @@ namespace Wino.Core.Synchronizers.Mail // At this point this client is ready to execute async commands. // Each task bundle will await and execution will continue in case of error. - ImapClient executorClient = null; + IImapClient executorClient = null; bool isCrashed = false; @@ -550,7 +400,7 @@ namespace Wino.Core.Synchronizers.Mail /// ImapClient from the pool /// Assigning remote folder. /// Assigning local folder. - private void AssignSpecialFolderType(ImapClient executorClient, IMailFolder remoteFolder, MailItemFolder localFolder) + private void AssignSpecialFolderType(IImapClient executorClient, IMailFolder remoteFolder, MailItemFolder localFolder) { // Inbox is awlawys available. Don't miss it for assignment even though XList or SpecialUser is not supported. if (executorClient.Inbox == remoteFolder) @@ -592,7 +442,7 @@ namespace Wino.Core.Synchronizers.Mail var localFolders = await _imapChangeProcessor.GetLocalFoldersAsync(Account.Id).ConfigureAwait(false); - ImapClient executorClient = null; + IImapClient executorClient = null; try { @@ -768,262 +618,31 @@ namespace Wino.Core.Synchronizers.Mail } } - - private async Task> SynchronizeFolderInternalAsync(MailItemFolder folder, CancellationToken cancellationToken = default) { if (!folder.IsSynchronizationEnabled) return default; var downloadedMessageIds = new List(); - // STEP1: Ask for flag changes for older mails. - // STEP2: Get new mail changes. - // https://www.rfc-editor.org/rfc/rfc4549 - Section 4.3 - - var _synchronizationClient = await _clientPool.GetClientAsync(); - - IMailFolder imapFolder = null; - - var knownMailIds = new UniqueIdSet(); - var locallyKnownMailUids = await _imapChangeProcessor.GetKnownUidsForFolderAsync(folder.Id); - knownMailIds.AddRange(locallyKnownMailUids.Select(a => new UniqueId(a))); - - var highestUniqueId = Math.Max(0, locallyKnownMailUids.Count == 0 ? 0 : locallyKnownMailUids.Max()); - - var missingMailIds = new UniqueIdSet(); - - var uidValidity = folder.UidValidity; - var highestModeSeq = folder.HighestModeSeq; - - var logger = Log.ForContext("FolderName", folder.FolderName); - - logger.Verbose("HighestModeSeq: {HighestModeSeq}, HighestUniqueId: {HighestUniqueId}, UIDValidity: {UIDValidity}", highestModeSeq, highestUniqueId, uidValidity); - - // Event handlers are placed here to handle existing MailItemFolder and IIMailFolder from MailKit. - // MailKit doesn't expose folder data when these events are emitted. - - // Use local folder's UidValidty because cache might've been expired for remote IMAP folder. - // That will make our mail copy id invalid. - - EventHandler MessageVanishedHandler = async (s, e) => - { - if (imapFolder == null) return; - - foreach (var uniqueId in e.UniqueIds) - { - var localMailCopyId = MailkitClientExtensions.CreateUid(folder.Id, uniqueId.Id); - - await _imapChangeProcessor.DeleteMailAsync(Account.Id, localMailCopyId); - } - }; - - EventHandler MessageFlagsChangedHandler = async (s, e) => - { - if (imapFolder == null) return; - if (e.UniqueId == null) return; - - var localMailCopyId = MailkitClientExtensions.CreateUid(folder.Id, e.UniqueId.Value.Id); - - var isFlagged = MailkitClientExtensions.GetIsFlagged(e.Flags); - var isRead = MailkitClientExtensions.GetIsRead(e.Flags); - - await _imapChangeProcessor.ChangeMailReadStatusAsync(localMailCopyId, isRead); - await _imapChangeProcessor.ChangeFlagStatusAsync(localMailCopyId, isFlagged); - }; - - EventHandler MessageExpungedHandler = async (s, e) => - { - if (imapFolder == null) return; - if (e.UniqueId == null) return; - - var localMailCopyId = MailkitClientExtensions.CreateUid(folder.Id, e.UniqueId.Value.Id); - await _imapChangeProcessor.DeleteMailAsync(Account.Id, localMailCopyId); - }; + IImapClient availableClient = null; try { - imapFolder = await _synchronizationClient.GetFolderAsync(folder.RemoteFolderId, cancellationToken); + availableClient = await _clientPool.GetClientAsync().ConfigureAwait(false); - imapFolder.MessageFlagsChanged += MessageFlagsChangedHandler; - - // TODO: Bug: Enabling quick re-sync actually doesn't enable it. - - var qsyncEnabled = false; // _synchronizationClient.Capabilities.HasFlag(ImapCapabilities.QuickResync); - var condStoreEnabled = _synchronizationClient.Capabilities.HasFlag(ImapCapabilities.CondStore); - - if (qsyncEnabled) - { - - imapFolder.MessagesVanished += MessageVanishedHandler; - - await imapFolder.OpenAsync(FolderAccess.ReadWrite, uidValidity, (ulong)highestModeSeq, knownMailIds, cancellationToken); - - // Check the folder validity. - // We'll delete our existing cache if it's not. - - // Get all messages after the last successful synchronization date. - // This is fine for Wino synchronization because we're not really looking to - // synchronize all folder. - - var allMessageIds = await imapFolder.SearchAsync(SearchQuery.All, cancellationToken); - - if (uidValidity != imapFolder.UidValidity) - { - // TODO: Cache is invalid. Delete all local cache. - //await ChangeProcessor.FolderService.ClearImapFolderCacheAsync(folder.Id); - - folder.UidValidity = imapFolder.UidValidity; - missingMailIds.AddRange(allMessageIds); - } - else - { - // Cache is valid. - // Add missing mails only. - - missingMailIds.AddRange(allMessageIds.Except(knownMailIds).Where(a => a.Id > highestUniqueId)); - } - } - else - { - // QSYNC extension is not enabled for the server. - // We rely on ConditionalStore. - - imapFolder.MessageExpunged += MessageExpungedHandler; - await imapFolder.OpenAsync(FolderAccess.ReadWrite, cancellationToken); - - // Get all messages after the last succesful synchronization date. - // This is fine for Wino synchronization because we're not really looking to - // synchronize all folder. - - var allMessageIds = await imapFolder.SearchAsync(SearchQuery.All, cancellationToken); - - if (uidValidity != imapFolder.UidValidity) - { - // TODO: Cache is invalid. Delete all local cache. - // await ChangeProcessor.FolderService.ClearImapFolderCacheAsync(folder.Id); - - folder.UidValidity = imapFolder.UidValidity; - missingMailIds.AddRange(allMessageIds); - } - else - { - // Cache is valid. - - var purgedMessages = knownMailIds.Except(allMessageIds); - - foreach (var purgedMessage in purgedMessages) - { - var mailId = MailkitClientExtensions.CreateUid(folder.Id, purgedMessage.Id); - - await _imapChangeProcessor.DeleteMailAsync(Account.Id, mailId); - } - - IList changed; - - if (knownMailIds.Count > 0) - { - // CONDSTORE enabled. Fetch items with highest mode seq for known items - // to track flag changes. Otherwise just get changes without the mode seq. - - if (condStoreEnabled) - changed = await imapFolder.FetchAsync(knownMailIds, (ulong)highestModeSeq, MessageSummaryItems.Flags | MessageSummaryItems.ModSeq | MessageSummaryItems.UniqueId); - else - changed = await imapFolder.FetchAsync(knownMailIds, MessageSummaryItems.Flags | MessageSummaryItems.UniqueId); - - foreach (var changedItem in changed) - { - var localMailCopyId = MailkitClientExtensions.CreateUid(folder.Id, changedItem.UniqueId.Id); - - var isFlagged = changedItem.Flags.GetIsFlagged(); - var isRead = changedItem.Flags.GetIsRead(); - - await _imapChangeProcessor.ChangeMailReadStatusAsync(localMailCopyId, isRead); - await _imapChangeProcessor.ChangeFlagStatusAsync(localMailCopyId, isFlagged); - } - } - - // We're only interested in items that has highier known uid than we fetched before. - // Others are just older messages. - - missingMailIds.AddRange(allMessageIds.Except(knownMailIds).Where(a => a.Id > highestUniqueId)); - } - } - - // Fetch completely missing new items in the end. - - // Limit check. - if (missingMailIds.Count > InitialMessageDownloadCountPerFolder) - { - missingMailIds = new UniqueIdSet(missingMailIds.TakeLast((int)InitialMessageDownloadCountPerFolder)); - } - - // In case of the high input, we'll batch them by 50 to reflect changes quickly. - var batchedMissingMailIds = missingMailIds.Batch(50).Select(a => new UniqueIdSet(a, SortOrder.Ascending)); - - foreach (var batchMissingMailIds in batchedMissingMailIds) - { - var summaries = await imapFolder.FetchAsync(batchMissingMailIds, mailSynchronizationFlags, cancellationToken).ConfigureAwait(false); - - foreach (var summary in summaries) - { - // We pass the opened folder and summary to retrieve raw MimeMessage. - - var creationPackage = new ImapMessageCreationPackage(summary, imapFolder); - var createdMailPackages = await CreateNewMailPackagesAsync(creationPackage, folder, cancellationToken).ConfigureAwait(false); - - // Local draft is mapped. We don't need to create a new mail copy. - if (createdMailPackages == null) - continue; - - foreach (var mailPackage in createdMailPackages) - { - bool isCreated = await _imapChangeProcessor.CreateMailAsync(Account.Id, mailPackage).ConfigureAwait(false); - - if (isCreated) - { - downloadedMessageIds.Add(mailPackage.Copy.Id); - } - } - } - } - - if (folder.HighestModeSeq != (long)imapFolder.HighestModSeq) - { - folder.HighestModeSeq = (long)imapFolder.HighestModSeq; - - await _imapChangeProcessor.UpdateFolderAsync(folder).ConfigureAwait(false); - } - - // Update last synchronization date for the folder.. - - await _imapChangeProcessor.UpdateFolderLastSyncDateAsync(folder.Id).ConfigureAwait(false); - - return downloadedMessageIds; - } - catch (FolderNotFoundException) - { - await _imapChangeProcessor.DeleteFolderAsync(Account.Id, folder.RemoteFolderId).ConfigureAwait(false); - - return default; + var strategy = _imapSynchronizationStrategyProvider.GetSynchronizationStrategy(availableClient); + return await strategy.HandleSynchronizationAsync(availableClient, folder, this, cancellationToken).ConfigureAwait(false); } catch (Exception) { - throw; + } finally { - if (imapFolder != null) - { - imapFolder.MessageFlagsChanged -= MessageFlagsChangedHandler; - imapFolder.MessageExpunged -= MessageExpungedHandler; - imapFolder.MessagesVanished -= MessageVanishedHandler; - - if (imapFolder.IsOpen) - await imapFolder.CloseAsync(); - } - - _clientPool.Release(_synchronizationClient); + _clientPool.Release(availableClient, false); } + + return new List(); } @@ -1037,8 +656,123 @@ namespace Wino.Core.Synchronizers.Mail => !localFolder.FolderName.Equals(remoteFolder.Name, StringComparison.OrdinalIgnoreCase); protected override Task SynchronizeCalendarEventsInternalAsync(CalendarSynchronizationOptions options, CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public async Task StartIdleClientAsync() { - throw new NotImplementedException(); + IImapClient idleClient = null; + IMailFolder inboxFolder = null; + + bool? reconnect = null; + + try + { + var client = await _clientPool.GetClientAsync().ConfigureAwait(false); + + if (!client.Capabilities.HasFlag(ImapCapabilities.Idle)) + { + Log.Debug($"{Account.Name} does not support Idle command. Ignored."); + return; + } + + if (client.Inbox == null) + { + Log.Warning($"{Account.Name} does not have an Inbox folder for idle client to track. Ignored."); + return; + } + + // Setup idle client. + idleClient = client; + + + idleDoneTokenSource ??= new CancellationTokenSource(); + idleCancellationTokenSource ??= new CancellationTokenSource(); + + inboxFolder = client.Inbox; + + await inboxFolder.OpenAsync(FolderAccess.ReadOnly, idleCancellationTokenSource.Token); + + inboxFolder.CountChanged += IdleNotificationTriggered; + inboxFolder.MessageFlagsChanged += IdleNotificationTriggered; + inboxFolder.MessageExpunged += IdleNotificationTriggered; + + await client.IdleAsync(idleDoneTokenSource.Token, idleCancellationTokenSource.Token); + } + catch (ImapProtocolException protocolException) + { + Log.Warning(protocolException, "Idle client received protocol exception."); + reconnect = true; + } + catch (IOException ioException) + { + Log.Warning(ioException, "Idle client received IO exception."); + reconnect = true; + } + catch (Exception ex) + { + Log.Warning(ex, "Idle client failed to start."); + reconnect = false; + } + finally + { + if (inboxFolder != null) + { + inboxFolder.CountChanged -= IdleNotificationTriggered; + inboxFolder.MessageFlagsChanged -= IdleNotificationTriggered; + inboxFolder.MessageExpunged -= IdleNotificationTriggered; + } + + if (idleDoneTokenSource != null) + { + idleDoneTokenSource.Dispose(); + idleDoneTokenSource = null; + } + + if (idleClient != null) + { + // Killing the client is not necessary. We can re-use it later. + _clientPool.Release(idleClient, destroyClient: false); + + idleClient = null; + } + + if (reconnect == true) + { + Log.Information("Idle client is reconnecting."); + + _ = StartIdleClientAsync(); + } + else if (reconnect == false) + { + Log.Information("Finalized idle client."); + } + } + } + + private void RequestIdleChangeSynchronization() + { + // We don't really need to act on the count change in detail. + // Our synchronization should be enough to handle the changes with on-demand sync. + // We can just trigger a sync here IMAPIdle type. + + var options = new MailSynchronizationOptions() + { + AccountId = Account.Id, + Type = MailSynchronizationType.IMAPIdle + }; + + _ = SynchronizeMailsAsync(options); + } + + private void IdleNotificationTriggered(object sender, EventArgs e) + => RequestIdleChangeSynchronization(); + + public Task StopIdleClientAsync() + { + idleDoneTokenSource?.Cancel(); + idleCancellationTokenSource?.Cancel(); + + return Task.CompletedTask; } } } diff --git a/Wino.Core/Synchronizers/WinoSynchronizer.cs b/Wino.Core/Synchronizers/WinoSynchronizer.cs index e9ee1848..d23d8fe1 100644 --- a/Wino.Core/Synchronizers/WinoSynchronizer.cs +++ b/Wino.Core/Synchronizers/WinoSynchronizer.cs @@ -25,6 +25,8 @@ namespace Wino.Core.Synchronizers { public abstract class WinoSynchronizer : BaseSynchronizer, IWinoSynchronizerBase { + protected Dictionary PendingSynchronizationRequest = new(); + protected ILogger Logger = Log.ForContext>(); protected WinoSynchronizer(MailAccount account) : base(account) @@ -83,94 +85,128 @@ namespace Wino.Core.Synchronizers { try { - activeSynchronizationCancellationToken = cancellationToken; - - State = AccountSynchronizerState.ExecutingRequests; - - List> nativeRequests = new(); - - List requestCopies = new(changeRequestQueue); - - var keys = changeRequestQueue.GroupBy(a => a.GroupingKey()); - - foreach (var group in keys) + if (!ShouldQueueMailSynchronization(options)) { - var key = group.Key; - - if (key is MailSynchronizerOperation mailSynchronizerOperation) - { - switch (mailSynchronizerOperation) - { - case MailSynchronizerOperation.MarkRead: - nativeRequests.AddRange(MarkRead(new BatchMarkReadRequest(group.Cast()))); - break; - case MailSynchronizerOperation.Move: - nativeRequests.AddRange(Move(new BatchMoveRequest(group.Cast()))); - break; - case MailSynchronizerOperation.Delete: - nativeRequests.AddRange(Delete(new BatchDeleteRequest(group.Cast()))); - break; - case MailSynchronizerOperation.CreateDraft: - nativeRequests.AddRange(CreateDraft(group.ElementAt(0) as CreateDraftRequest)); - break; - case MailSynchronizerOperation.Send: - nativeRequests.AddRange(SendDraft(group.ElementAt(0) as SendDraftRequest)); - break; - case MailSynchronizerOperation.ChangeFlag: - nativeRequests.AddRange(ChangeFlag(new BatchChangeFlagRequest(group.Cast()))); - break; - case MailSynchronizerOperation.AlwaysMoveTo: - nativeRequests.AddRange(AlwaysMoveTo(new BatchAlwaysMoveToRequest(group.Cast()))); - break; - case MailSynchronizerOperation.MoveToFocused: - nativeRequests.AddRange(MoveToFocused(new BatchMoveToFocusedRequest(group.Cast()))); - break; - case MailSynchronizerOperation.Archive: - nativeRequests.AddRange(Archive(new BatchArchiveRequest(group.Cast()))); - break; - default: - break; - } - } - else if (key is FolderSynchronizerOperation folderSynchronizerOperation) - { - switch (folderSynchronizerOperation) - { - case FolderSynchronizerOperation.RenameFolder: - nativeRequests.AddRange(RenameFolder(group.ElementAt(0) as RenameFolderRequest)); - break; - case FolderSynchronizerOperation.EmptyFolder: - nativeRequests.AddRange(EmptyFolder(group.ElementAt(0) as EmptyFolderRequest)); - break; - case FolderSynchronizerOperation.MarkFolderRead: - nativeRequests.AddRange(MarkFolderAsRead(group.ElementAt(0) as MarkFolderAsReadRequest)); - break; - default: - break; - } - } + Log.Debug($"{options.Type} synchronization is ignored."); + return MailSynchronizationResult.Canceled; } - changeRequestQueue.Clear(); + PendingSynchronizationRequest.Add(options, cancellationToken); - Console.WriteLine($"Prepared {nativeRequests.Count()} native requests"); + activeSynchronizationCancellationToken = cancellationToken; + + await synchronizationSemaphore.WaitAsync(activeSynchronizationCancellationToken); PublishSynchronizationProgress(1); - await ExecuteNativeRequestsAsync(nativeRequests, activeSynchronizationCancellationToken); + // ImapSynchronizer will send this type when an Idle client receives a notification of changes. + // We should not execute requests in this case. + bool shouldExecuteRequests = options.Type != MailSynchronizationType.IMAPIdle; - PublishUnreadItemChanges(); + bool shouldDelayExecution = false; + int maxExecutionDelay = 0; - // Execute request sync options should be re-calculated after execution. - // This is the part we decide which individual folders must be synchronized - // after the batch request execution. - if (options.Type == MailSynchronizationType.ExecuteRequests) - options = GetSynchronizationOptionsAfterRequestExecution(requestCopies); + if (shouldExecuteRequests && changeRequestQueue.Any()) + { + State = AccountSynchronizerState.ExecutingRequests; + + List> nativeRequests = new(); + + List requestCopies = new(changeRequestQueue); + + var keys = changeRequestQueue.GroupBy(a => a.GroupingKey()); + + foreach (var group in keys) + { + var key = group.Key; + + if (key is MailSynchronizerOperation mailSynchronizerOperation) + { + switch (mailSynchronizerOperation) + { + case MailSynchronizerOperation.MarkRead: + nativeRequests.AddRange(MarkRead(new BatchMarkReadRequest(group.Cast()))); + break; + case MailSynchronizerOperation.Move: + nativeRequests.AddRange(Move(new BatchMoveRequest(group.Cast()))); + break; + case MailSynchronizerOperation.Delete: + nativeRequests.AddRange(Delete(new BatchDeleteRequest(group.Cast()))); + break; + case MailSynchronizerOperation.CreateDraft: + nativeRequests.AddRange(CreateDraft(group.ElementAt(0) as CreateDraftRequest)); + break; + case MailSynchronizerOperation.Send: + nativeRequests.AddRange(SendDraft(group.ElementAt(0) as SendDraftRequest)); + break; + case MailSynchronizerOperation.ChangeFlag: + nativeRequests.AddRange(ChangeFlag(new BatchChangeFlagRequest(group.Cast()))); + break; + case MailSynchronizerOperation.AlwaysMoveTo: + nativeRequests.AddRange(AlwaysMoveTo(new BatchAlwaysMoveToRequest(group.Cast()))); + break; + case MailSynchronizerOperation.MoveToFocused: + nativeRequests.AddRange(MoveToFocused(new BatchMoveToFocusedRequest(group.Cast()))); + break; + case MailSynchronizerOperation.Archive: + nativeRequests.AddRange(Archive(new BatchArchiveRequest(group.Cast()))); + break; + default: + break; + } + } + else if (key is FolderSynchronizerOperation folderSynchronizerOperation) + { + switch (folderSynchronizerOperation) + { + case FolderSynchronizerOperation.RenameFolder: + nativeRequests.AddRange(RenameFolder(group.ElementAt(0) as RenameFolderRequest)); + break; + case FolderSynchronizerOperation.EmptyFolder: + nativeRequests.AddRange(EmptyFolder(group.ElementAt(0) as EmptyFolderRequest)); + break; + case FolderSynchronizerOperation.MarkFolderRead: + nativeRequests.AddRange(MarkFolderAsRead(group.ElementAt(0) as MarkFolderAsReadRequest)); + break; + default: + break; + } + } + } + + changeRequestQueue.Clear(); + + Console.WriteLine($"Prepared {nativeRequests.Count()} native requests"); + + await ExecuteNativeRequestsAsync(nativeRequests, activeSynchronizationCancellationToken).ConfigureAwait(false); + + PublishUnreadItemChanges(); + + // Execute request sync options should be re-calculated after execution. + // This is the part we decide which individual folders must be synchronized + // after the batch request execution. + if (options.Type == MailSynchronizationType.ExecuteRequests) + options = GetSynchronizationOptionsAfterRequestExecution(requestCopies); + + // Let servers to finish their job. Sometimes the servers doesn't respond immediately. + // Bug: if Outlook can't create the message in Sent Items folder before this delay, + // message will not appear in user's inbox since it's not in the Sent Items folder. + + shouldDelayExecution = + (Account.ProviderType == MailProviderType.Outlook || Account.ProviderType == MailProviderType.Office365) + && requestCopies.Any(a => a.ResynchronizationDelay > 0); + + if (shouldDelayExecution) + { + maxExecutionDelay = requestCopies.Aggregate(0, (max, next) => Math.Max(max, next.ResynchronizationDelay)); + } + + // In terms of flag/read changes, there is no point of synchronizing must have folders. + options.ExcludeMustHaveFolders = requestCopies.All(a => a is ICustomFolderSynchronizationRequest request && request.ExcludeMustHaveFolders); + } State = AccountSynchronizerState.Synchronizing; - await synchronizationSemaphore.WaitAsync(activeSynchronizationCancellationToken); - // Handle special synchronization types. // Profile information sync. @@ -213,19 +249,9 @@ namespace Wino.Core.Synchronizers } } - // Let servers to finish their job. Sometimes the servers doesn't respond immediately. - // Bug: if Outlook can't create the message in Sent Items folder before this delay, - // message will not appear in user's inbox since it's not in the Sent Items folder. - - bool shouldDelayExecution = - (Account.ProviderType == MailProviderType.Outlook || Account.ProviderType == MailProviderType.Office365) - && requestCopies.Any(a => a.ResynchronizationDelay > 0); - if (shouldDelayExecution) { - var maxDelay = requestCopies.Aggregate(0, (max, next) => Math.Max(max, next.ResynchronizationDelay)); - - await Task.Delay(maxDelay); + await Task.Delay(maxExecutionDelay); } // Start the internal synchronization. @@ -249,6 +275,8 @@ namespace Wino.Core.Synchronizers } finally { + PendingSynchronizationRequest.Remove(options); + // Reset account progress to hide the progress. PublishSynchronizationProgress(0); @@ -317,6 +345,23 @@ namespace Wino.Core.Synchronizers return options; } + /// + /// Checks if the mail synchronization should be queued or not. + /// + /// New mail sync request. + /// Whether sync should be queued or not. + private bool ShouldQueueMailSynchronization(MailSynchronizationOptions options) + { + // Multiple IMAPIdle requests are ignored. + if (options.Type == MailSynchronizationType.IMAPIdle && + PendingSynchronizationRequest.Any(a => a.Key.Type == MailSynchronizationType.IMAPIdle)) + { + return false; + } + + return true; + } + #region Mail/Folder Operations public virtual bool DelaySendOperationSynchronization() => false; @@ -349,12 +394,12 @@ namespace Wino.Core.Synchronizers /// Cancellation token. public virtual Task DownloadMissingMimeMessageAsync(IMailItem mailItem, ITransferProgress transferProgress = null, CancellationToken cancellationToken = default) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType())); - public List> CreateSingleTaskBundle(Func action, IRequestBase request, IUIChangeRequest uIChangeRequest) + public List> CreateSingleTaskBundle(Func action, IRequestBase request, IUIChangeRequest uIChangeRequest) { return [new ImapRequestBundle(new ImapRequest(action, request), request, uIChangeRequest)]; } - public List> CreateTaskBundle(Func value, + public List> CreateTaskBundle(Func value, List requests) where TSingeRequestType : IRequestBase, IUIChangeRequest { diff --git a/Wino.Core/Wino.Core.csproj b/Wino.Core/Wino.Core.csproj index 529f4a39..e632e9fb 100644 --- a/Wino.Core/Wino.Core.csproj +++ b/Wino.Core/Wino.Core.csproj @@ -52,4 +52,8 @@ + + + + diff --git a/Wino.Mail.ViewModels/AppShellViewModel.cs b/Wino.Mail.ViewModels/AppShellViewModel.cs index 9b7fc454..e07f8776 100644 --- a/Wino.Mail.ViewModels/AppShellViewModel.cs +++ b/Wino.Mail.ViewModels/AppShellViewModel.cs @@ -243,7 +243,9 @@ namespace Wino.Mail.ViewModels await RecreateMenuItemsAsync(); await ProcessLaunchOptionsAsync(); +#if !DEBUG await ForceAllAccountSynchronizationsAsync(); +#endif await MakeSureEnableStartupLaunchAsync(); await ConfigureBackgroundTasksAsync(); } diff --git a/Wino.Mail/App.xaml.cs b/Wino.Mail/App.xaml.cs index 4154323a..2bbadfc6 100644 --- a/Wino.Mail/App.xaml.cs +++ b/Wino.Mail/App.xaml.cs @@ -233,6 +233,7 @@ namespace Wino } catch (WinoServerException serverException) { + // TODO: Exception context is lost. var dialogService = Services.GetService(); dialogService.InfoBarMessage(Translator.Info_SyncFailedTitle, serverException.Message, InfoBarMessageType.Error); diff --git a/Wino.Mail/Views/MailListPage.xaml b/Wino.Mail/Views/MailListPage.xaml index bc852d85..67d60efd 100644 --- a/Wino.Mail/Views/MailListPage.xaml +++ b/Wino.Mail/Views/MailListPage.xaml @@ -472,11 +472,6 @@ - diff --git a/Wino.Services/Extensions/MailkitClientExtensions.cs b/Wino.Services/Extensions/MailkitClientExtensions.cs index e4c31750..93e1ab8d 100644 --- a/Wino.Services/Extensions/MailkitClientExtensions.cs +++ b/Wino.Services/Extensions/MailkitClientExtensions.cs @@ -5,7 +5,6 @@ using MimeKit; using Wino.Core.Domain; using Wino.Core.Domain.Entities.Mail; using Wino.Core.Domain.Enums; -using Wino.Services.Extensions; namespace Wino.Services.Extensions { @@ -22,6 +21,9 @@ namespace Wino.Services.Extensions throw new ArgumentOutOfRangeException(nameof(mailCopyId), mailCopyId, "Invalid mailCopyId format."); } + public static UniqueId ResolveUidStruct(string mailCopyId) + => new UniqueId(ResolveUid(mailCopyId)); + public static string CreateUid(Guid folderId, uint messageUid) => $"{folderId}{MailCopyUidSeparator}{messageUid}"; diff --git a/Wino.Services/FolderService.cs b/Wino.Services/FolderService.cs index 3b970e11..05e69390 100644 --- a/Wino.Services/FolderService.cs +++ b/Wino.Services/FolderService.cs @@ -367,16 +367,14 @@ namespace Wino.Services public async Task> GetKnownUidsForFolderAsync(Guid folderId) { - var folder = await GetFolderAsync(folderId); - - if (folder == null) return default; - var mailCopyIds = await GetMailCopyIdsByFolderIdAsync(folderId); // Make sure we don't include Ids that doesn't have uid separator. // Local drafts might not have it for example. - return new List(mailCopyIds.Where(a => a.Contains(MailkitClientExtensions.MailCopyUidSeparator)).Select(a => MailkitClientExtensions.ResolveUid(a))); + return new List(mailCopyIds + .Where(a => a.Contains(MailkitClientExtensions.MailCopyUidSeparator)) + .Select(a => MailkitClientExtensions.ResolveUid(a))); } public async Task UpdateSystemFolderConfigurationAsync(Guid accountId, SystemFolderConfiguration configuration) @@ -546,7 +544,19 @@ namespace Wino.Services { var folders = new List(); - if (options.Type == MailSynchronizationType.FullFolders) + if (options.Type == MailSynchronizationType.IMAPIdle) + { + // Type Inbox will include Sent, Drafts and Deleted folders as well. + // For IMAP idle sync, we must include only Inbox folder. + + var inboxFolder = await GetSpecialFolderByAccountIdAsync(options.AccountId, SpecialFolderType.Inbox); + + if (inboxFolder != null) + { + folders.Add(inboxFolder); + } + } + else if (options.Type == MailSynchronizationType.FullFolders) { // Only get sync enabled folders. @@ -570,12 +580,19 @@ namespace Wino.Services } else if (options.Type == MailSynchronizationType.CustomFolders) { - // Only get the specified and enabled folders. + // Only get the specified folders. var synchronizationFolders = await Connection.Table() - .Where(a => a.MailAccountId == options.AccountId && options.SynchronizationFolderIds.Contains(a.Id)) + .Where(a => + a.MailAccountId == options.AccountId && + options.SynchronizationFolderIds.Contains(a.Id)) .ToListAsync(); + if (options.ExcludeMustHaveFolders) + { + return synchronizationFolders; + } + // Order is important for moving. // By implementation, removing mail folders must be synchronized first. Requests are made in that order for custom sync. // eg. Moving item from Folder A to Folder B. If we start syncing Folder B first, we might miss adding assignment for Folder A. diff --git a/Wino.Services/MailService.cs b/Wino.Services/MailService.cs index e5f02cec..b01f3313 100644 --- a/Wino.Services/MailService.cs +++ b/Wino.Services/MailService.cs @@ -984,6 +984,17 @@ namespace Wino.Services public Task IsMailExistsAsync(string mailCopyId) => Connection.ExecuteScalarAsync("SELECT EXISTS(SELECT 1 FROM MailCopy WHERE Id = ?)", mailCopyId); + public async Task> GetExistingMailsAsync(Guid folderId, IEnumerable uniqueIds) + { + var localMailIds = uniqueIds.Where(a => a != null).Select(a => MailkitClientExtensions.CreateUid(folderId, a.Id)).ToArray(); + + var query = new Query(nameof(MailCopy)) + .WhereIn("Id", localMailIds) + .GetRawQuery(); + + return await Connection.QueryAsync(query); + } + public Task IsMailExistsAsync(string mailCopyId, Guid folderId) => Connection.ExecuteScalarAsync("SELECT EXISTS(SELECT 1 FROM MailCopy WHERE Id = ? AND FolderId = ?)", mailCopyId, folderId); }