using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using CommunityToolkit.Mvvm.Messaging; using MailKit; using MailKit.Net.Imap; using MailKit.Search; using MimeKit; using Serilog; using Wino.Core.Domain.Entities.Mail; using Wino.Core.Domain.Entities.Shared; using Wino.Core.Domain.Enums; using Wino.Core.Domain.Exceptions; using Wino.Core.Domain.Interfaces; using Wino.Core.Domain.Models.Connectivity; using Wino.Core.Domain.Models.Folders; using Wino.Core.Domain.Models.MailItem; using Wino.Core.Domain.Models.Synchronization; using Wino.Core.Extensions; using Wino.Core.Integration; using Wino.Core.Integration.Processors; using Wino.Core.Requests.Bundles; using Wino.Core.Requests.Folder; using Wino.Core.Requests.Mail; using Wino.Core.Synchronizers.ImapSync; using Wino.Messaging.Server; using Wino.Messaging.UI; using Wino.Services.Extensions; namespace Wino.Core.Synchronizers.Mail; public class ImapSynchronizer : WinoSynchronizer, IImapSynchronizer { /// /// N/A for IMAP as it doesn't support batch modifications natively. /// public override uint BatchModificationSize => 1000; public override uint InitialMessageDownloadCountPerFolder => 500; #region Idle Implementation private static readonly Random IdleReconnectJitter = new(); private readonly object _idleDebounceLock = new(); private CancellationTokenSource _idleLoopCancellationTokenSource; private Task _idleLoopTask; private int _lastIdleInboxCount = -1; private DateTime _lastIdleSyncRequestUtc = DateTime.MinValue; private readonly TimeSpan _idleSyncDebounceWindow = TimeSpan.FromSeconds(15); #endregion private readonly ILogger _logger = Log.ForContext(); private readonly ImapClientPool _clientPool; private readonly IImapChangeProcessor _imapChangeProcessor; private readonly IApplicationConfiguration _applicationConfiguration; private readonly UnifiedImapSynchronizer _unifiedSynchronizer; private readonly IImapSynchronizerErrorHandlerFactory _errorHandlerFactory; public ImapSynchronizer(MailAccount account, IImapChangeProcessor imapChangeProcessor, IApplicationConfiguration applicationConfiguration, UnifiedImapSynchronizer unifiedSynchronizer, IImapSynchronizerErrorHandlerFactory errorHandlerFactory) : base(account, WeakReferenceMessenger.Default) { // Create client pool with account protocol log. _imapChangeProcessor = imapChangeProcessor; _applicationConfiguration = applicationConfiguration; _unifiedSynchronizer = unifiedSynchronizer; _errorHandlerFactory = errorHandlerFactory; var protocolLogStream = CreateAccountProtocolLogFileStream(); var poolOptions = ImapClientPoolOptions.CreateDefault(Account.ServerInformation, protocolLogStream); _clientPool = new ImapClientPool(poolOptions); } private Stream CreateAccountProtocolLogFileStream() { if (Account == null) throw new ArgumentNullException(nameof(Account)); var logFile = Path.Combine(_applicationConfiguration.ApplicationDataFolderPath, $"Protocol_{Account.Address}_{Account.Id}.log"); // Each session should start a new log. if (File.Exists(logFile)) File.Delete(logFile); return new FileStream(logFile, FileMode.CreateNew); } /// /// Returns UniqueId for the given mail copy id. /// private UniqueId GetUniqueId(string mailCopyId) => new(MailkitClientExtensions.ResolveUid(mailCopyId)); #region Mail Integrations // Items are grouped before being passed to this method. // Meaning that all items will come from and to the same folder. // It's fine to assume that here. public override List> Move(BatchMoveRequest requests) { return CreateTaskBundle(async (client, item) => { var sourceFolder = await client.GetFolderAsync(item.FromFolder.RemoteFolderId); var destinationFolder = await client.GetFolderAsync(item.ToFolder.RemoteFolderId); // Only opening source folder is enough. await sourceFolder.OpenAsync(FolderAccess.ReadWrite).ConfigureAwait(false); await sourceFolder.MoveToAsync(GetUniqueId(item.Item.Id), destinationFolder).ConfigureAwait(false); await sourceFolder.CloseAsync().ConfigureAwait(false); }, requests); } public override List> ChangeFlag(BatchChangeFlagRequest requests) { return CreateTaskBundle(async (client, item) => { var folder = item.Item.AssignedFolder; var remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId); await remoteFolder.OpenAsync(FolderAccess.ReadWrite).ConfigureAwait(false); await remoteFolder.StoreAsync(GetUniqueId(item.Item.Id), new StoreFlagsRequest(item.Item.IsFlagged ? StoreAction.Add : StoreAction.Remove, MessageFlags.Flagged) { Silent = true }).ConfigureAwait(false); await remoteFolder.CloseAsync().ConfigureAwait(false); }, requests); } public override List> Delete(BatchDeleteRequest requests) { return CreateTaskBundle(async (client, request) => { var folder = request.Item.AssignedFolder; var remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId).ConfigureAwait(false); await remoteFolder.OpenAsync(FolderAccess.ReadWrite).ConfigureAwait(false); await remoteFolder.AddFlagsAsync(GetUniqueId(request.Item.Id), MessageFlags.Deleted, true); await remoteFolder.ExpungeAsync().ConfigureAwait(false); await remoteFolder.CloseAsync().ConfigureAwait(false); }, requests); } public override List> MarkRead(BatchMarkReadRequest requests) { return CreateTaskBundle(async (client, request) => { var folder = request.Item.AssignedFolder; var remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId); await remoteFolder.OpenAsync(FolderAccess.ReadWrite).ConfigureAwait(false); await remoteFolder.StoreAsync(GetUniqueId(request.Item.Id), new StoreFlagsRequest(request.IsRead ? StoreAction.Add : StoreAction.Remove, MessageFlags.Seen) { Silent = true }).ConfigureAwait(false); await remoteFolder.CloseAsync().ConfigureAwait(false); }, requests); } public override List> CreateDraft(CreateDraftRequest request) { return CreateSingleTaskBundle(async (client, item) => { var remoteDraftFolder = await client.GetFolderAsync(request.DraftPreperationRequest.CreatedLocalDraftCopy.AssignedFolder.RemoteFolderId).ConfigureAwait(false); await remoteDraftFolder.OpenAsync(FolderAccess.ReadWrite).ConfigureAwait(false); await remoteDraftFolder.AppendAsync(request.DraftPreperationRequest.CreatedLocalDraftMimeMessage, MessageFlags.Draft).ConfigureAwait(false); await remoteDraftFolder.CloseAsync().ConfigureAwait(false); }, request, request); } public override List> Archive(BatchArchiveRequest request) { var batchMoveRequest = new BatchMoveRequest(request.Select(item => new MoveRequest(item.Item, item.FromFolder, item.ToFolder))); return Move(batchMoveRequest); } public override List> EmptyFolder(EmptyFolderRequest request) => Delete(new BatchDeleteRequest(request.MailsToDelete.Select(a => new DeleteRequest(a)))); public override List> MarkFolderAsRead(MarkFolderAsReadRequest request) => MarkRead(new BatchMarkReadRequest(request.MailsToMarkRead.Select(a => new MarkReadRequest(a, true)))); public override List> SendDraft(SendDraftRequest request) { return CreateSingleTaskBundle(async (client, item) => { // Batch sending is not supported. It will always be a single request therefore no need for a loop here. var singleRequest = request.Request; using var smtpClient = new MailKit.Net.Smtp.SmtpClient(); if (smtpClient.IsConnected && client.IsAuthenticated) return; if (!smtpClient.IsConnected) await smtpClient.ConnectAsync(Account.ServerInformation.OutgoingServer, int.Parse(Account.ServerInformation.OutgoingServerPort), MailKit.Security.SecureSocketOptions.Auto); if (!smtpClient.IsAuthenticated) await smtpClient.AuthenticateAsync(Account.ServerInformation.OutgoingServerUsername, Account.ServerInformation.OutgoingServerPassword); // Remove local draft header before sending to prevent leaking to recipients. singleRequest.Mime.Headers.Remove(Domain.Constants.WinoLocalDraftHeader); // TODO: Transfer progress implementation as popup in the UI. await smtpClient.SendAsync(singleRequest.Mime, default); await smtpClient.DisconnectAsync(true); // SMTP sent the message, but we need to remove it from the Draft folder. var draftFolder = singleRequest.MailItem.AssignedFolder; var folder = await client.GetFolderAsync(draftFolder.RemoteFolderId); await folder.OpenAsync(FolderAccess.ReadWrite); await folder.AddFlagsAsync(new UniqueId(MailkitClientExtensions.ResolveUid(singleRequest.MailItem.Id)), MessageFlags.Deleted, true); await folder.ExpungeAsync(); await folder.CloseAsync(); // Check whether we need to create a copy of the message to Sent folder. // This comes from the account preferences. if (singleRequest.AccountPreferences.ShouldAppendMessagesToSentFolder && singleRequest.SentFolder != null) { var sentFolder = await client.GetFolderAsync(singleRequest.SentFolder.RemoteFolderId); await sentFolder.OpenAsync(FolderAccess.ReadWrite); await sentFolder.AppendAsync(singleRequest.Mime, MessageFlags.Seen); await sentFolder.CloseAsync(); } }, request, request); } public override async Task DownloadMissingMimeMessageAsync(MailCopy mailItem, ITransferProgress transferProgress = null, CancellationToken cancellationToken = default) { var folder = mailItem.AssignedFolder; var remoteFolderId = folder.RemoteFolderId; var client = await _clientPool.GetClientAsync().ConfigureAwait(false); try { var remoteFolder = await client.GetFolderAsync(remoteFolderId, cancellationToken).ConfigureAwait(false); var uniqueId = new UniqueId(MailkitClientExtensions.ResolveUid(mailItem.Id)); await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false); var message = await remoteFolder.GetMessageAsync(uniqueId, cancellationToken, transferProgress).ConfigureAwait(false); await _imapChangeProcessor.SaveMimeFileAsync(mailItem.FileId, message, Account.Id).ConfigureAwait(false); await remoteFolder.CloseAsync(false, cancellationToken).ConfigureAwait(false); } catch (FolderNotFoundException ex) { _logger.Warning("IMAP folder {FolderId} not found during MIME download for {MailId}. Deleting locally.", remoteFolderId, mailItem.Id); await _imapChangeProcessor.DeleteMailAsync(Account.Id, mailItem.Id).ConfigureAwait(false); throw new SynchronizerEntityNotFoundException(ex.Message); } catch (ImapCommandException ex) when (ex.Response == ImapCommandResponse.No) { _logger.Warning("IMAP message {MailId} not found during MIME download (NO response). Deleting locally.", mailItem.Id); await _imapChangeProcessor.DeleteMailAsync(Account.Id, mailItem.Id).ConfigureAwait(false); throw new SynchronizerEntityNotFoundException(ex.Message); } finally { _clientPool.Release(client); } } public override Task DownloadCalendarAttachmentAsync( Wino.Core.Domain.Entities.Calendar.CalendarItem calendarItem, Wino.Core.Domain.Entities.Calendar.CalendarAttachment attachment, string localFilePath, CancellationToken cancellationToken = default) { // IMAP protocol doesn't support calendar operations natively // Calendar functionality would require CalDAV protocol _logger.Warning("IMAP protocol does not support calendar attachments. CalDAV would be required."); throw new NotSupportedException("IMAP does not support calendar attachments. Use Outlook or Gmail for calendar functionality."); } public override List> RenameFolder(RenameFolderRequest request) { return CreateSingleTaskBundle(async (client, item) => { var folder = await client.GetFolderAsync(request.Folder.RemoteFolderId).ConfigureAwait(false); await folder.RenameAsync(folder.ParentFolder, request.NewFolderName).ConfigureAwait(false); }, request, request); } public override List> DeleteFolder(DeleteFolderRequest request) { return CreateSingleTaskBundle(async (client, item) => { var folder = await client.GetFolderAsync(request.Folder.RemoteFolderId).ConfigureAwait(false); await folder.DeleteAsync().ConfigureAwait(false); }, request, request); } public override List> CreateSubFolder(CreateSubFolderRequest request) { return CreateSingleTaskBundle(async (client, item) => { var parentFolder = await client.GetFolderAsync(request.Folder.RemoteFolderId).ConfigureAwait(false); await parentFolder.CreateAsync(request.NewFolderName, true).ConfigureAwait(false); }, request, request); } #endregion public override async Task> CreateNewMailPackagesAsync(ImapMessageCreationPackage message, MailItemFolder assignedFolder, CancellationToken cancellationToken = default) { var mailCopy = message.MessageSummary.GetMailDetails(assignedFolder, message.MimeMessage); // Draft folder message updates must be updated as IsDraft. // I couldn't find it in MimeMesssage... mailCopy.IsDraft = assignedFolder.SpecialFolderType == SpecialFolderType.Draft; // Check draft mapping. // This is the same implementation as in the OutlookSynchronizer. string draftHeaderValue = null; if (message.MimeMessage?.Headers?.Contains(Domain.Constants.WinoLocalDraftHeader) == true) { draftHeaderValue = message.MimeMessage.Headers[Domain.Constants.WinoLocalDraftHeader]; } else if (message.MessageSummary?.Headers?.Contains(Domain.Constants.WinoLocalDraftHeader) == true) { draftHeaderValue = message.MessageSummary.Headers[Domain.Constants.WinoLocalDraftHeader]; } if (Guid.TryParse(draftHeaderValue, out Guid localDraftCopyUniqueId)) { // This message belongs to existing local draft copy. // We don't need to create a new mail copy for this message, just update the existing one. bool isMappingSuccessful = await _imapChangeProcessor.MapLocalDraftAsync(Account.Id, localDraftCopyUniqueId, mailCopy.Id, draftHeaderValue, mailCopy.ThreadId); if (isMappingSuccessful) return null; // Local copy doesn't exists. Continue execution to insert mail copy. } var contacts = message.MimeMessage != null ? ExtractContactsFromMimeMessage(message.MimeMessage) : ExtractContactsFromMessageSummary(message.MessageSummary); var package = new NewMailItemPackage(mailCopy, message.MimeMessage, assignedFolder.RemoteFolderId, contacts); return [ package ]; } private static IReadOnlyList ExtractContactsFromMimeMessage(MimeMessage mimeMessage) { if (mimeMessage == null) return []; var contacts = new Dictionary(StringComparer.OrdinalIgnoreCase); AddFromInternetAddressList(mimeMessage.From); AddFromInternetAddressList(mimeMessage.To); AddFromInternetAddressList(mimeMessage.Cc); AddFromInternetAddressList(mimeMessage.Bcc); AddFromInternetAddressList(mimeMessage.ReplyTo); if (mimeMessage.Sender is MailboxAddress senderMailbox) { AddContact(senderMailbox.Address, senderMailbox.Name); } return contacts.Values.ToList(); void AddFromInternetAddressList(InternetAddressList addresses) { if (addresses == null) return; foreach (var mailbox in addresses.Mailboxes) { AddContact(mailbox.Address, mailbox.Name); } } void AddContact(string address, string name) { var trimmedAddress = address?.Trim(); if (string.IsNullOrWhiteSpace(trimmedAddress)) return; var displayName = string.IsNullOrWhiteSpace(name) ? trimmedAddress : name.Trim(); contacts[trimmedAddress] = new AccountContact { Address = trimmedAddress, Name = displayName }; } } private static IReadOnlyList ExtractContactsFromMessageSummary(IMessageSummary summary) { if (summary?.Envelope == null) return []; var contacts = new Dictionary(StringComparer.OrdinalIgnoreCase); AddFromInternetAddressList(summary.Envelope.From); AddFromInternetAddressList(summary.Envelope.To); AddFromInternetAddressList(summary.Envelope.Cc); AddFromInternetAddressList(summary.Envelope.Bcc); AddFromInternetAddressList(summary.Envelope.ReplyTo); var senderMailbox = summary.Envelope.Sender?.Mailboxes?.FirstOrDefault(); if (senderMailbox != null) { AddContact(senderMailbox.Address, senderMailbox.Name); } return contacts.Values.ToList(); void AddFromInternetAddressList(InternetAddressList addresses) { if (addresses == null) return; foreach (var mailbox in addresses.Mailboxes) { AddContact(mailbox.Address, mailbox.Name); } } void AddContact(string address, string name) { var trimmedAddress = address?.Trim(); if (string.IsNullOrWhiteSpace(trimmedAddress)) return; var displayName = string.IsNullOrWhiteSpace(name) ? trimmedAddress : name.Trim(); contacts[trimmedAddress] = new AccountContact { Address = trimmedAddress, Name = displayName }; } } protected override async Task SynchronizeMailsInternalAsync(MailSynchronizationOptions options, CancellationToken cancellationToken = default) { var downloadedMessageIds = new List(); var folderResults = new List(); _logger.Information("Internal synchronization started for {Name}", Account.Name); _logger.Information("Options: {Options}", options); try { // Set indeterminate progress initially UpdateSyncProgress(0, 0, "Synchronizing..."); bool shouldDoFolderSync = options.Type == MailSynchronizationType.FullFolders || options.Type == MailSynchronizationType.FoldersOnly; if (shouldDoFolderSync) { await SynchronizeFoldersAsync(cancellationToken).ConfigureAwait(false); } if (options.Type != MailSynchronizationType.FoldersOnly) { var synchronizationFolders = await _imapChangeProcessor.GetSynchronizationFoldersAsync(options).ConfigureAwait(false); var totalFolders = synchronizationFolders.Count; const int maxParallelFolderSyncClients = 3; var folderSyncSemaphore = new SemaphoreSlim(maxParallelFolderSyncClients, maxParallelFolderSyncClients); using var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); var linkedToken = linkedCancellationTokenSource.Token; var resultLock = new object(); int completedFolders = 0; var syncTasks = synchronizationFolders.Select(async folder => { await folderSyncSemaphore.WaitAsync(linkedToken).ConfigureAwait(false); try { IImapClient client = null; try { client = await _clientPool.GetClientAsync(linkedToken).ConfigureAwait(false); var folderResult = await _unifiedSynchronizer .SynchronizeFolderAsync(client, folder, this, Account.ServerInformation?.IncomingServer, linkedToken) .ConfigureAwait(false); List folderDownloadedIds = null; if (folderResult.Success && folderResult.DownloadedCount > 0) { folderDownloadedIds = await GetDownloadedIdsForFolderAsync(folder, folderResult.DownloadedCount).ConfigureAwait(false); } lock (resultLock) { folderResults.Add(folderResult); if (folderDownloadedIds != null && folderDownloadedIds.Count > 0) { downloadedMessageIds.AddRange(folderDownloadedIds); } } } finally { if (client != null) { _clientPool.Release(client); } } } catch (OperationCanceledException) { throw; } catch (Exception ex) { var errorContext = new SynchronizerErrorContext { Account = Account, ErrorMessage = ex.Message, Exception = ex, FolderId = folder.Id, FolderName = folder.FolderName, OperationType = "ImapFolderSync" }; _ = await _errorHandlerFactory.HandleErrorAsync(errorContext).ConfigureAwait(false); var failedResult = FolderSyncResult.Failed(folder.Id, folder.FolderName, errorContext); lock (resultLock) { folderResults.Add(failedResult); } if (!errorContext.CanContinueSync) { _logger.Error(ex, "Folder {FolderName} sync failed with fatal error", folder.FolderName); linkedCancellationTokenSource.Cancel(); throw; } _logger.Warning(ex, "Folder {FolderName} sync failed, continuing with other folders", folder.FolderName); } finally { folderSyncSemaphore.Release(); var completed = Interlocked.Increment(ref completedFolders); UpdateSyncProgress(totalFolders, totalFolders - completed, $"Syncing {folder.FolderName}..."); } }).ToList(); await Task.WhenAll(syncTasks).ConfigureAwait(false); if (cancellationToken.IsCancellationRequested) return MailSynchronizationResult.Canceled; } } catch (OperationCanceledException) { _logger.Information("Synchronization was canceled for {Name}", Account.Name); return MailSynchronizationResult.Canceled; } catch (Exception ex) { _logger.Error(ex, "Synchronization failed for {Name}", Account.Name); return MailSynchronizationResult.Failed(ex); } finally { // Reset progress ResetSyncProgress(); } // Get all unread new downloaded items and return in the result. // This is primarily used in notifications. var unreadNewItems = await _imapChangeProcessor.GetDownloadedUnreadMailsAsync(Account.Id, downloadedMessageIds).ConfigureAwait(false); return MailSynchronizationResult.CompletedWithFolderResults(unreadNewItems, folderResults); } /// /// Gets the most recent downloaded message IDs for a folder. /// Used for notification purposes after sync completes. /// private async Task> GetDownloadedIdsForFolderAsync(MailItemFolder folder, int count) { // Get the most recent mail IDs from the folder var recentMails = await _imapChangeProcessor.GetRecentMailIdsForFolderAsync(folder.Id, count).ConfigureAwait(false); return recentMails?.ToList() ?? new List(); } public override async Task ExecuteNativeRequestsAsync(List> batchedRequests, CancellationToken cancellationToken = default) { // First apply the UI changes for each bundle. // This is important to reflect changes to the UI before the network call is done. foreach (var item in batchedRequests) { item.Request.ApplyUIChanges(); } // All task bundles will execute on the same client. // Tasks themselves don't pull the client from the pool // because exception handling is easier this way. // Also we might parallelize these bundles later on for additional performance. foreach (var item in batchedRequests) { // At this point this client is ready to execute async commands. // Each task bundle will await and execution will continue in case of error. IImapClient executorClient = null; bool isCrashed = false; try { executorClient = await _clientPool.GetClientAsync(); } catch (ImapClientPoolException) { // Client pool failed to get a client. // Requests may not be executed at this point. item.Request.RevertUIChanges(); isCrashed = true; throw; } finally { // Make sure that the client is released from the pool for next usages if error occurs. if (isCrashed && executorClient != null) { _clientPool.Release(executorClient); } } try { await item.NativeRequest.IntegratorTask(executorClient, item.Request).ConfigureAwait(false); } catch (Exception ex) { var errorContext = new SynchronizerErrorContext { Account = Account, ErrorCode = ex is FolderNotFoundException ? 404 : null, ErrorMessage = ex.Message, Exception = ex, RequestBundle = item, OperationType = "RequestExecution" }; var handled = await _errorHandlerFactory.HandleErrorAsync(errorContext).ConfigureAwait(false); if (!handled) { item.Request.RevertUIChanges(); throw; } } finally { _clientPool.Release(executorClient); } } } /// /// Assigns special folder type for the given local folder. /// If server doesn't support special folders, we can't determine the type. MailKit will throw for GetFolder. /// Default type is Other. /// /// ImapClient from the pool /// Assigning remote folder. /// Assigning local folder. private void AssignSpecialFolderType(IImapClient executorClient, IMailFolder remoteFolder, MailItemFolder localFolder) { // Inbox is awlawys available. Don't miss it for assignment even though XList or SpecialUser is not supported. if (executorClient.Inbox == remoteFolder) { localFolder.SpecialFolderType = SpecialFolderType.Inbox; return; } bool isSpecialFoldersSupported = executorClient.Capabilities.HasFlag(ImapCapabilities.SpecialUse) || executorClient.Capabilities.HasFlag(ImapCapabilities.XList); if (!isSpecialFoldersSupported) { localFolder.SpecialFolderType = SpecialFolderType.Other; return; } if (remoteFolder == executorClient.Inbox) localFolder.SpecialFolderType = SpecialFolderType.Inbox; else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Drafts)) localFolder.SpecialFolderType = SpecialFolderType.Draft; else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Junk)) localFolder.SpecialFolderType = SpecialFolderType.Junk; else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Trash)) localFolder.SpecialFolderType = SpecialFolderType.Deleted; else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Sent)) localFolder.SpecialFolderType = SpecialFolderType.Sent; else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Archive)) localFolder.SpecialFolderType = SpecialFolderType.Archive; else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Important)) localFolder.SpecialFolderType = SpecialFolderType.Important; else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Flagged)) localFolder.SpecialFolderType = SpecialFolderType.Starred; } private async Task SynchronizeFoldersAsync(CancellationToken cancellationToken = default) { // https://www.rfc-editor.org/rfc/rfc4549#section-1.1 var localFolders = await _imapChangeProcessor.GetLocalFoldersAsync(Account.Id).ConfigureAwait(false); IImapClient executorClient = null; try { List insertedFolders = new(); List updatedFolders = new(); List deletedFolders = new(); executorClient = await _clientPool.GetClientAsync().ConfigureAwait(false); var remoteFolders = (await executorClient.GetFoldersAsync(executorClient.PersonalNamespaces[0], cancellationToken: cancellationToken)).ToList(); // 1. First check deleted folders. // 1.a If local folder doesn't exists remotely, delete it. // 1.b If local folder exists remotely, check if it is still a valid folder. If UidValidity is changed, delete it. foreach (var localFolder in localFolders) { IMailFolder remoteFolder = null; try { remoteFolder = remoteFolders.FirstOrDefault(a => a.FullName == localFolder.RemoteFolderId); bool shouldDeleteLocalFolder = false; // Check UidValidity of the remote folder if exists. if (remoteFolder != null) { // UidValidity won't be available until it's opened. await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false); shouldDeleteLocalFolder = remoteFolder.UidValidity != localFolder.UidValidity; } else { // Remote folder doesn't exist. Delete it. shouldDeleteLocalFolder = true; } if (shouldDeleteLocalFolder) { await _imapChangeProcessor.DeleteFolderAsync(Account.Id, localFolder.RemoteFolderId).ConfigureAwait(false); deletedFolders.Add(localFolder); } } catch (Exception) { throw; } finally { if (remoteFolder != null) { await remoteFolder.CloseAsync().ConfigureAwait(false); } } } deletedFolders.ForEach(a => localFolders.Remove(a)); // 2. Get all remote folders and insert/update each of them. var nameSpace = executorClient.PersonalNamespaces[0]; IMailFolder inbox = executorClient.Inbox; // Sometimes Inbox is the root namespace. We need to check for that. if (inbox != null && !remoteFolders.Contains(inbox)) remoteFolders.Add(inbox); foreach (var remoteFolder in remoteFolders) { // Namespaces are not needed as folders. // Non-existed folders don't need to be synchronized. if (remoteFolder.IsNamespace && !remoteFolder.Attributes.HasFlag(FolderAttributes.Inbox) || !remoteFolder.Exists) continue; // Ignore folders that can't be opened. if (!remoteFolder.CanOpen) continue; var existingLocalFolder = localFolders.FirstOrDefault(a => a.RemoteFolderId == remoteFolder.FullName); if (existingLocalFolder == null) { // Folder doesn't exist locally. Insert it. var localFolder = remoteFolder.GetLocalFolder(); // Check whether this is a special folder. AssignSpecialFolderType(executorClient, remoteFolder, localFolder); bool isSystemFolder = localFolder.SpecialFolderType != SpecialFolderType.Other; localFolder.IsSynchronizationEnabled = isSystemFolder; localFolder.IsSticky = isSystemFolder; // By default, all special folders update unread count in the UI except Trash. localFolder.ShowUnreadCount = localFolder.SpecialFolderType != SpecialFolderType.Deleted || localFolder.SpecialFolderType != SpecialFolderType.Other; localFolder.MailAccountId = Account.Id; // Sometimes sub folders are parented under Inbox. // Even though this makes sense in server level, in the client it sucks. // That will make sub folders to be parented under Inbox in the client. // Instead, we will mark them as non-parented folders. // This is better. Model allows personalized folder structure anyways // even though we don't have the page/control to adjust it. if (remoteFolder.ParentFolder == executorClient.Inbox) localFolder.ParentRemoteFolderId = string.Empty; // Set UidValidity for cache expiration. // Folder must be opened for this. await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken); localFolder.UidValidity = remoteFolder.UidValidity; await remoteFolder.CloseAsync(cancellationToken: cancellationToken); insertedFolders.Add(localFolder); } else { // Update existing folder. Right now we only update the name. // TODO: Moving folders around different parents. This is not supported right now. // We will need more comphrensive folder update mechanism to support this. if (ShouldUpdateFolder(remoteFolder, existingLocalFolder)) { existingLocalFolder.FolderName = remoteFolder.Name; updatedFolders.Add(existingLocalFolder); } else { // Remove it from the local folder list to skip additional folder updates. localFolders.Remove(existingLocalFolder); } } } // Process changes in order-> Insert, Update. Deleted ones are already processed. foreach (var folder in insertedFolders) { await _imapChangeProcessor.InsertFolderAsync(folder).ConfigureAwait(false); } foreach (var folder in updatedFolders) { await _imapChangeProcessor.UpdateFolderAsync(folder).ConfigureAwait(false); } if (insertedFolders.Any() || deletedFolders.Any() || updatedFolders.Any()) { WeakReferenceMessenger.Default.Send(new AccountFolderConfigurationUpdated(Account.Id)); } } catch (Exception ex) { _logger.Error(ex, "Synchronizing IMAP folders failed."); throw; } finally { if (executorClient != null) { _clientPool.Release(executorClient); } } } public override async Task> OnlineSearchAsync(string queryText, List folders, CancellationToken cancellationToken = default) { IImapClient client = null; try { client = await _clientPool.GetClientAsync().ConfigureAwait(false); List searchResults = []; List searchResultFolderMailUids = []; foreach (var folder in folders) { if (folder is not MailItemFolder localFolder) continue; var remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false); await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false); // Look for subject and body. var query = SearchQuery.BodyContains(queryText).Or(SearchQuery.SubjectContains(queryText)); var searchResultsInFolder = await remoteFolder.SearchAsync(query, cancellationToken).ConfigureAwait(false); Dictionary searchResultsIdsInFolder = []; foreach (var searchResultId in searchResultsInFolder) { var folderMailUid = MailkitClientExtensions.CreateUid(folder.Id, searchResultId.Id); searchResultFolderMailUids.Add(folderMailUid); searchResultsIdsInFolder.Add(folderMailUid, searchResultId); } // Populate no foundIds var foundIds = await _imapChangeProcessor.AreMailsExistsAsync(searchResultsIdsInFolder.Select(a => a.Key)); var notFoundIds = searchResultsIdsInFolder.Keys.Except(foundIds); List nonExistingUniqueIds = []; foreach (var nonExistingId in notFoundIds) { nonExistingUniqueIds.Add(searchResultsIdsInFolder[nonExistingId]); } if (nonExistingUniqueIds.Count != 0) { await _unifiedSynchronizer .DownloadMessagesByUidsAsync(client, remoteFolder, localFolder, nonExistingUniqueIds, this, cancellationToken) .ConfigureAwait(false); } await remoteFolder.CloseAsync(cancellationToken: cancellationToken).ConfigureAwait(false); } return await _imapChangeProcessor.GetMailCopiesAsync(searchResultFolderMailUids); } catch (Exception ex) { Log.Error(ex, "Failed to perform online imap search."); throw; } finally { _clientPool.Release(client); } } /// /// Whether the local folder should be updated with the remote folder. /// IMAP only compares folder name for now. /// /// Remote folder /// Local folder. public bool ShouldUpdateFolder(IMailFolder remoteFolder, MailItemFolder localFolder) => !localFolder.FolderName.Equals(remoteFolder.Name, StringComparison.OrdinalIgnoreCase); protected override Task SynchronizeCalendarEventsInternalAsync(CalendarSynchronizationOptions options, CancellationToken cancellationToken = default) => throw new NotImplementedException(); public Task StartIdleClientAsync() { if (IsDisposing) return Task.CompletedTask; if (_idleLoopTask != null && !_idleLoopTask.IsCompleted) return Task.CompletedTask; _idleLoopCancellationTokenSource = new CancellationTokenSource(); _idleLoopTask = RunIdleLoopAsync(_idleLoopCancellationTokenSource.Token); return Task.CompletedTask; } private async Task RunIdleLoopAsync(CancellationToken cancellationToken) { int reconnectAttempt = 0; while (!cancellationToken.IsCancellationRequested && !IsDisposing) { IImapClient idleClient = null; IMailFolder inboxFolder = null; bool shouldReconnect = false; try { idleClient = await _clientPool.GetIdleClientAsync(cancellationToken).ConfigureAwait(false); if (idleClient == null) { _logger.Warning("Dedicated IDLE client could not be allocated for {AccountName}.", Account.Name); return; } if (!idleClient.Capabilities.HasFlag(ImapCapabilities.Idle)) { _logger.Information("{AccountName} does not support IMAP IDLE. Automatic updates rely on global sync interval.", Account.Name); return; } if (idleClient.Inbox == null) { _logger.Warning("{AccountName} does not expose Inbox for IDLE listening.", Account.Name); return; } inboxFolder = idleClient.Inbox; await inboxFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false); _lastIdleInboxCount = inboxFolder.Count; inboxFolder.CountChanged += IdleInboxCountChanged; reconnectAttempt = 0; _logger.Debug("Started dedicated IDLE loop for {AccountName}.", Account.Name); while (!cancellationToken.IsCancellationRequested && !IsDisposing && idleClient.IsConnected) { using var idleDoneTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(9)); await idleClient.IdleAsync(idleDoneTokenSource.Token, cancellationToken).ConfigureAwait(false); } } catch (ImapProtocolException protocolException) { _logger.Information(protocolException, "Idle client received protocol exception for {AccountName}.", Account.Name); shouldReconnect = true; } catch (IOException ioException) { _logger.Information(ioException, "Idle client received IO exception for {AccountName}.", Account.Name); shouldReconnect = true; } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested || IsDisposing) { break; } catch (OperationCanceledException) { shouldReconnect = true; } catch (Exception ex) { _logger.Error(ex, "Idle client loop failed for {AccountName}.", Account.Name); shouldReconnect = true; } finally { if (inboxFolder != null) { inboxFolder.CountChanged -= IdleInboxCountChanged; if (inboxFolder.IsOpen && !cancellationToken.IsCancellationRequested) { await inboxFolder.CloseAsync(cancellationToken: cancellationToken).ConfigureAwait(false); } } _clientPool.ReleaseIdleClient(isFaulted: shouldReconnect); } if (!shouldReconnect) { break; } reconnectAttempt++; var reconnectDelay = GetIdleReconnectDelay(reconnectAttempt); _logger.Information("Reconnecting IDLE client for {AccountName} in {Delay}.", Account.Name, reconnectDelay); try { await Task.Delay(reconnectDelay, cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) { break; } } } private static TimeSpan GetIdleReconnectDelay(int attempt) { var backoffSeconds = Math.Min(60, Math.Pow(2, Math.Min(attempt, 6))); int jitterMs; lock (IdleReconnectJitter) { jitterMs = IdleReconnectJitter.Next(250, 1250); } return TimeSpan.FromSeconds(backoffSeconds) + TimeSpan.FromMilliseconds(jitterMs); } private void RequestIdleChangeSynchronization() { if (!ShouldTriggerIdleSynchronization(DateTime.UtcNow)) return; var options = new MailSynchronizationOptions() { AccountId = Account.Id, Type = MailSynchronizationType.IMAPIdle }; WeakReferenceMessenger.Default.Send(new NewMailSynchronizationRequested(options)); } internal bool ShouldTriggerIdleSynchronization(DateTime nowUtc) { lock (_idleDebounceLock) { if (nowUtc - _lastIdleSyncRequestUtc < _idleSyncDebounceWindow) { return false; } _lastIdleSyncRequestUtc = nowUtc; return true; } } private void IdleInboxCountChanged(object sender, EventArgs e) { if (sender is not IMailFolder inboxFolder) return; var currentCount = inboxFolder.Count; var previousCount = _lastIdleInboxCount; _lastIdleInboxCount = currentCount; if (currentCount > previousCount) { RequestIdleChangeSynchronization(); } } public async Task StopIdleClientAsync() { if (_idleLoopCancellationTokenSource != null) { _idleLoopCancellationTokenSource.Cancel(); } if (_idleLoopTask != null) { try { await _idleLoopTask.ConfigureAwait(false); } catch (OperationCanceledException) { // no-op } } _idleLoopCancellationTokenSource?.Dispose(); _idleLoopCancellationTokenSource = null; _idleLoopTask = null; } public override async Task KillSynchronizerAsync() { await base.KillSynchronizerAsync(); await StopIdleClientAsync(); // Make sure the client pool safely disconnects all ImapClients. _clientPool.Dispose(); } public Task PreWarmClientPoolAsync() => _clientPool.PreWarmPoolAsync(); }