using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Net.Http; using System.Threading; using System.Threading.Tasks; using CommunityToolkit.Mvvm.Messaging; using MailKit; using Serilog; using Wino.Core.Domain; using Wino.Core.Domain.Entities; using Wino.Core.Domain.Enums; using Wino.Core.Domain.Interfaces; using Wino.Core.Domain.Models.MailItem; using Wino.Core.Domain.Models.Synchronization; using Wino.Core.Integration; using Wino.Core.Misc; using Wino.Core.Requests; using Wino.Messaging.UI; namespace Wino.Core.Synchronizers { public abstract class BaseSynchronizer : BaseMailIntegrator, IBaseSynchronizer { private SemaphoreSlim synchronizationSemaphore = new(1); private CancellationToken activeSynchronizationCancellationToken; protected ConcurrentBag changeRequestQueue = []; protected ILogger Logger = Log.ForContext>(); protected BaseSynchronizer(MailAccount account) { Account = account; } public MailAccount Account { get; } private AccountSynchronizerState state; public AccountSynchronizerState State { get { return state; } private set { state = value; WeakReferenceMessenger.Default.Send(new AccountSynchronizerStateChanged(Account.Id, value)); } } /// /// Queues a single request to be executed in the next synchronization. /// /// Request to execute. public void QueueRequest(IRequestBase request) => changeRequestQueue.Add(request); /// /// Creates a new Wino Mail Item package out of native message type with full Mime. /// /// Native message type for the synchronizer. /// Cancellation token /// Package that encapsulates downloaded Mime and additional information for adding new mail. public abstract Task> CreateNewMailPackagesAsync(TMessageType message, MailItemFolder assignedFolder, CancellationToken cancellationToken = default); /// /// Runs existing queued requests in the queue. /// /// Batched requests to execute. Integrator methods will only receive batched requests. /// Cancellation token public abstract Task ExecuteNativeRequestsAsync(IEnumerable> batchedRequests, CancellationToken cancellationToken = default); /// /// Refreshed remote mail account profile if possible. /// Aliases, profile pictures, mailbox settings will be handled in this step. /// protected virtual Task SynchronizeProfileInformationAsync() => Task.CompletedTask; /// /// Returns the base64 encoded profile picture of the account from the given URL. /// /// URL to retrieve picture from. /// base64 encoded profile picture protected async Task GetProfilePictureBase64EncodedAsync(string url) { using var client = new HttpClient(); var response = await client.GetAsync(url).ConfigureAwait(false); var byteContent = await response.Content.ReadAsByteArrayAsync().ConfigureAwait(false); return Convert.ToBase64String(byteContent); } /// /// Internally synchronizes the account with the given options. /// Not exposed and overriden for each synchronizer. /// /// Synchronization options. /// Cancellation token. /// Synchronization result that contains summary of the sync. protected abstract Task SynchronizeInternalAsync(SynchronizationOptions options, CancellationToken cancellationToken = default); /// /// Batches network requests, executes them, and does the needed synchronization after the batch request execution. /// /// Synchronization options. /// Cancellation token. /// Synchronization result that contains summary of the sync. public async Task SynchronizeAsync(SynchronizationOptions options, CancellationToken cancellationToken = default) { try { activeSynchronizationCancellationToken = cancellationToken; var batches = CreateBatchRequests().Distinct(); if (batches.Any()) { Logger.Information($"{batches?.Count() ?? 0} batched requests"); State = AccountSynchronizerState.ExecutingRequests; var nativeRequests = CreateNativeRequestBundles(batches); Console.WriteLine($"Prepared {nativeRequests.Count()} native requests"); await ExecuteNativeRequestsAsync(nativeRequests, activeSynchronizationCancellationToken); PublishUnreadItemChanges(); // Execute request sync options should be re-calculated after execution. // This is the part we decide which individual folders must be synchronized // after the batch request execution. if (options.Type == SynchronizationType.ExecuteRequests) options = GetSynchronizationOptionsAfterRequestExecution(batches); } State = AccountSynchronizerState.Synchronizing; await synchronizationSemaphore.WaitAsync(activeSynchronizationCancellationToken); if (options.Type == SynchronizationType.Full) { // Refresh profile information and mailbox settings on full synchronization. // Exceptions here is not critical. Therefore, they are ignored. await SynchronizeProfileInformationAsync(); } // Let servers to finish their job. Sometimes the servers doesn't respond immediately. bool shouldDelayExecution = batches.Any(a => a.DelayExecution); if (shouldDelayExecution) { await Task.Delay(2000); } // Start the internal synchronization. var synchronizationResult = await SynchronizeInternalAsync(options, activeSynchronizationCancellationToken).ConfigureAwait(false); PublishUnreadItemChanges(); return synchronizationResult; } catch (OperationCanceledException) { Logger.Warning("Synchronization canceled."); return SynchronizationResult.Canceled; } catch (Exception ex) { Logger.Error(ex, "Synchronization failed for {Name}", Account.Name); Debugger.Break(); throw; } finally { // Reset account progress to hide the progress. PublishSynchronizationProgress(0); State = AccountSynchronizerState.Idle; synchronizationSemaphore.Release(); } } /// /// Updates unread item counts for some folders and account. /// Sends a message that shell can pick up and update the UI. /// private void PublishUnreadItemChanges() => WeakReferenceMessenger.Default.Send(new RefreshUnreadCountsMessage(Account.Id)); /// /// Sends a message to the shell to update the synchronization progress. /// /// Percentage of the progress. public void PublishSynchronizationProgress(double progress) => WeakReferenceMessenger.Default.Send(new AccountSynchronizationProgressUpdatedMessage(Account.Id, progress)); /// /// 1. Group all requests by operation type. /// 2. Group all individual operation type requests with equality check. /// Equality comparison in the records are done with RequestComparer /// to ignore Item property. Each request can have their own logic for comparison. /// For example, move requests for different mails from the same folder to the same folder /// must be dispatched in the same batch. This is much faster for the server. Specially IMAP /// since all folders must be asynchronously opened/closed. /// /// Batch request collection for all these single requests. private List CreateBatchRequests() { var batchList = new List(); var comparer = new RequestComparer(); while (changeRequestQueue.Count > 0) { if (changeRequestQueue.TryPeek(out IRequestBase request)) { // Mail request, must be batched. if (request is IRequest mailRequest) { var equalItems = changeRequestQueue .Where(a => a is IRequest && comparer.Equals(a, request)) .Cast() .ToList(); batchList.Add(mailRequest.CreateBatch(equalItems)); // Remove these items from the queue. foreach (var item in equalItems) { changeRequestQueue.TryTake(out _); } } else if (changeRequestQueue.TryTake(out request)) { // This is a folder operation. // There is no need to batch them since Users can't do folder ops in bulk. batchList.Add(request); } } } return batchList; } /// /// Converts batched requests into HTTP/Task calls that derived synchronizers can execute. /// /// Batch requests to be converted. /// Collection of native requests for individual synchronizer type. private IEnumerable> CreateNativeRequestBundles(IEnumerable batchChangeRequests) { IEnumerable>> GetNativeRequests() { foreach (var item in batchChangeRequests) { switch (item.Operation) { case MailSynchronizerOperation.Send: yield return SendDraft((BatchSendDraftRequestRequest)item); break; case MailSynchronizerOperation.MarkRead: yield return MarkRead((BatchMarkReadRequest)item); break; case MailSynchronizerOperation.Move: yield return Move((BatchMoveRequest)item); break; case MailSynchronizerOperation.Delete: yield return Delete((BatchDeleteRequest)item); break; case MailSynchronizerOperation.ChangeFlag: yield return ChangeFlag((BatchChangeFlagRequest)item); break; case MailSynchronizerOperation.AlwaysMoveTo: yield return AlwaysMoveTo((BatchAlwaysMoveToRequest)item); break; case MailSynchronizerOperation.MoveToFocused: yield return MoveToFocused((BatchMoveToFocusedRequest)item); break; case MailSynchronizerOperation.CreateDraft: yield return CreateDraft((BatchCreateDraftRequest)item); break; case MailSynchronizerOperation.RenameFolder: yield return RenameFolder((RenameFolderRequest)item); break; case MailSynchronizerOperation.EmptyFolder: yield return EmptyFolder((EmptyFolderRequest)item); break; case MailSynchronizerOperation.MarkFolderRead: yield return MarkFolderAsRead((MarkFolderAsReadRequest)item); break; case MailSynchronizerOperation.Archive: yield return Archive((BatchArchiveRequest)item); break; } } }; return GetNativeRequests().SelectMany(collections => collections); } /// /// Attempts to find out the best possible synchronization options after the batch request execution. /// /// Batch requests to run in synchronization. /// New synchronization options with minimal HTTP effort. private SynchronizationOptions GetSynchronizationOptionsAfterRequestExecution(IEnumerable requests) { List synchronizationFolderIds = new(); if (requests.All(a => a is IBatchChangeRequest)) { var requestsInsideBatches = requests.Cast().SelectMany(b => b.Items); // Gather FolderIds to synchronize. synchronizationFolderIds = requestsInsideBatches .Where(a => a is ICustomFolderSynchronizationRequest) .Cast() .SelectMany(a => a.SynchronizationFolderIds) .ToList(); } var options = new SynchronizationOptions() { AccountId = Account.Id, }; if (synchronizationFolderIds.Count > 0) { // Gather FolderIds to synchronize. options.Type = SynchronizationType.Custom; options.SynchronizationFolderIds = synchronizationFolderIds; } else { // At this point it's a mix of everything. Do full sync. options.Type = SynchronizationType.Full; } return options; } public virtual bool DelaySendOperationSynchronization() => false; public virtual IEnumerable> Move(BatchMoveRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType())); public virtual IEnumerable> ChangeFlag(BatchChangeFlagRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType())); public virtual IEnumerable> MarkRead(BatchMarkReadRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType())); public virtual IEnumerable> Delete(BatchDeleteRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType())); public virtual IEnumerable> AlwaysMoveTo(BatchAlwaysMoveToRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType())); public virtual IEnumerable> MoveToFocused(BatchMoveToFocusedRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType())); public virtual IEnumerable> CreateDraft(BatchCreateDraftRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType())); public virtual IEnumerable> SendDraft(BatchSendDraftRequestRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType())); public virtual IEnumerable> RenameFolder(RenameFolderRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType())); public virtual IEnumerable> EmptyFolder(EmptyFolderRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType())); public virtual IEnumerable> MarkFolderAsRead(MarkFolderAsReadRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType())); public virtual IEnumerable> Archive(BatchArchiveRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType())); /// /// Downloads a single missing message from synchronizer and saves it to given FileId from IMailItem. /// /// Mail item that its mime file does not exist on the disk. /// Optional download progress for IMAP synchronizer. /// Cancellation token. public virtual Task DownloadMissingMimeMessageAsync(IMailItem mailItem, ITransferProgress transferProgress = null, CancellationToken cancellationToken = default) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType())); public bool CancelActiveSynchronization() { // TODO: What if account is deleted during synchronization? return true; } } }