Implemented cache reset for Gmail history id expiration. (#581)

Wino.Core.Domain/Enums/AccountCacheResetReason.cs

@@ -0,0 +1,7 @@
+namespace Wino.Core.Domain.Enums;
+
+public enum AccountCacheResetReason
+{
+    AccountRemoval,
+    ExpiredCache
+}

Wino.Core.Domain/Interfaces/IAccountService.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Threading.Tasks;
 using Wino.Core.Domain.Entities.Mail;
 using Wino.Core.Domain.Entities.Shared;
+using Wino.Core.Domain.Enums;
 using Wino.Core.Domain.Models.Accounts;
 
 namespace Wino.Core.Domain.Interfaces;
@@ -156,4 +157,11 @@ public interface IAccountService
     /// <returns>Primary alias for the account.</returns>
     Task<MailAccountAlias> GetPrimaryAccountAliasAsync(Guid accountId);
     Task<bool> IsAccountFocusedEnabledAsync(Guid accountId);
+
+    /// <summary>
+    /// Deletes mail cache in the database for the given account.
+    /// </summary>
+    /// <param name="accountId">Account id.</param>
+    /// <param name="AccountCacheResetReason">Reason for the cache reset.</param>
+    Task DeleteAccountMailCacheAsync(Guid accountId, AccountCacheResetReason accountCacheResetReason);
 }

Wino.Core.Domain/Interfaces/IMailService.cs

@@ -135,4 +135,10 @@ public interface IMailService
     /// <param name="package">Mail creation package.</param>
     /// <returns></returns>
     Task CreateMailRawAsync(MailAccount account, MailItemFolder mailItemFolder, NewMailItemPackage package);
+
+    /// <summary>
+    /// Checks whether the account has any draft mail locally.
+    /// </summary>
+    /// <param name="accountId">Account id.</param>
+    Task<bool> HasAccountAnyDraftAsync(Guid accountId);
 }

Wino.Core.Domain/Interfaces/IMimeFileService.cs

@@ -66,4 +66,10 @@ public interface IMimeFileService
     /// <param name="mimeLocalPath">File path that physical MimeMessage is located.</param>
     /// <param name="options">Rendering options</param>
     MailRenderModel GetMailRenderModel(MimeMessage message, string mimeLocalPath, MailRenderingOptions options = null);
+
+    /// <summary>
+    /// Deletes every file in the mime cache for the given account.
+    /// </summary>
+    /// <param name="accountId">Account id.</param>
+    Task DeleteUserMimeCacheAsync(Guid accountId);
 }

Wino.Core.Domain/Translations/en_US/resources.json

@@ -4,6 +4,8 @@
     "AccountAlias_Column_Verified": "Verified",
     "AccountAlias_Disclaimer_FirstLine": "Wino can only import aliases for your Gmail accounts.",
     "AccountAlias_Disclaimer_SecondLine": "If you want to use aliases for your Outlook or IMAP account, please add them yourself.",
+    "AccountCacheReset_Title": "Account Cache Reset",
+    "AccountCacheReset_Message": "This account requires full re-sychronization to continue working. Please wait while Wino re-synchronizes your messages...",
     "AccountContactNameYou": "You",
     "AccountCreationDialog_Completed": "all done",
     "AccountCreationDialog_FetchingEvents": "Fetching calendar events.",

Wino.Core.UWP/Services/WinoServerConnectionManager.cs

@@ -258,7 +258,10 @@ public class WinoServerConnectionManager :
                 WeakReferenceMessenger.Default.Send(JsonSerializer.Deserialize(messageJson, CommunicationMessagesContext.Default.CopyAuthURLRequested));
                 break;
             case nameof(NewMailSynchronizationRequested):
-                WeakReferenceMessenger.Default.Send(JsonSerializer.Deserialize<NewMailSynchronizationRequested>(messageJson));
+                WeakReferenceMessenger.Default.Send(JsonSerializer.Deserialize(messageJson, CommunicationMessagesContext.Default.NewMailSynchronizationRequested));
+                break;
+            case nameof(AccountCacheResetMessage):
+                WeakReferenceMessenger.Default.Send(JsonSerializer.Deserialize(messageJson, CommunicationMessagesContext.Default.AccountCacheResetMessage));
                 break;
             default:
                 throw new Exception("Invalid data type name passed to client.");

Wino.Core/Integration/Processors/DefaultChangeProcessor.cs

@@ -6,6 +6,7 @@ using MimeKit;
 using Wino.Core.Domain.Entities.Calendar;
 using Wino.Core.Domain.Entities.Mail;
 using Wino.Core.Domain.Entities.Shared;
+using Wino.Core.Domain.Enums;
 using Wino.Core.Domain.Interfaces;
 using Wino.Core.Domain.Models.MailItem;
 using Wino.Core.Domain.Models.Synchronization;
@@ -61,10 +62,12 @@ public interface IDefaultChangeProcessor
     Task UpdateCalendarDeltaSynchronizationToken(Guid calendarId, string deltaToken);
     Task<MailCopy> GetMailCopyAsync(string mailCopyId);
     Task CreateMailRawAsync(MailAccount account, MailItemFolder mailItemFolder, NewMailItemPackage package);
+    Task DeleteUserMailCacheAsync(Guid accountId);
 }
 
 public interface IGmailChangeProcessor : IDefaultChangeProcessor
 {
+    Task<bool> HasAccountAnyDraftAsync(Guid accountId);
     Task MapLocalDraftAsync(string mailCopyId, string newDraftId, string newThreadId);
     Task CreateAssignmentAsync(Guid accountId, string mailCopyId, string remoteFolderId);
     Task ManageCalendarEventAsync(Event calendarEvent, AccountCalendar assignedCalendar, MailAccount organizerAccount);
@@ -214,4 +217,10 @@ public class DefaultChangeProcessor(IDatabaseService databaseService,
 
     public Task UpdateCalendarDeltaSynchronizationToken(Guid calendarId, string deltaToken)
         => CalendarService.UpdateCalendarDeltaSynchronizationToken(calendarId, deltaToken);
+
+    public async Task DeleteUserMailCacheAsync(Guid accountId)
+    {
+        await _mimeFileService.DeleteUserMimeCacheAsync(accountId).ConfigureAwait(false);
+        await AccountService.DeleteAccountMailCacheAsync(accountId, AccountCacheResetReason.ExpiredCache).ConfigureAwait(false);
+    }
 }

Wino.Core/Integration/Processors/GmailChangeProcessor.cs

@@ -308,4 +308,6 @@ public class GmailChangeProcessor : DefaultChangeProcessor, IGmailChangeProcesso
         };
     }
 
+    public Task<bool> HasAccountAnyDraftAsync(Guid accountId)
+        => MailService.HasAccountAnyDraftAsync(accountId);
 }

Wino.Core/Synchronizers/GmailSynchronizer.cs

@@ -43,7 +43,13 @@ namespace Wino.Core.Synchronizers.Mail;
 public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message, Event>, IHttpClientFactory
 {
     public override uint BatchModificationSize => 1000;
-    public override uint InitialMessageDownloadCountPerFolder => 1200;
+
+    /// <summary>
+    /// This is NOT the initial message download count per folder.
+    /// Gmail doesn't have per-folder sync. Therefore this represents to total amount that 1 page query returns until
+    /// there are no pages to get. Max allowed is 500.
+    /// </summary>
+    public override uint InitialMessageDownloadCountPerFolder => 500;
 
     // It's actually 100. But Gmail SDK has internal bug for Out of Memory exception.
     // https://github.com/googleapis/google-api-dotnet-client/issues/2603
@@ -143,6 +149,7 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
 
         if (options.Type == MailSynchronizationType.FoldersOnly) return MailSynchronizationResult.Empty;
 
+        retry:
         cancellationToken.ThrowIfCancellationRequested();
 
         bool isInitialSync = string.IsNullOrEmpty(Account.SynchronizationDeltaIdentifier);
@@ -204,25 +211,43 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
             var historyRequest = _gmailService.Users.History.List("me");
             historyRequest.StartHistoryId = startHistoryId;
 
-            while (!string.IsNullOrEmpty(nextPageToken))
+            try
             {
-                // If this is the first delta check, start from the last history id.
+                while (!string.IsNullOrEmpty(nextPageToken))
-                // Otherwise start from the next page token. We set them both to the same value for start.
+                {
-                // For each different page we set the page token to the next page token.
+                    // If this is the first delta check, start from the last history id.
+                    // Otherwise start from the next page token. We set them both to the same value for start.
+                    // For each different page we set the page token to the next page token.
 
-                bool isFirstDeltaCheck = nextPageToken == startHistoryId.ToString();
+                    bool isFirstDeltaCheck = nextPageToken == startHistoryId.ToString();
 
-                if (!isFirstDeltaCheck)
+                    if (!isFirstDeltaCheck)
-                    historyRequest.PageToken = nextPageToken;
+                        historyRequest.PageToken = nextPageToken;
 
-                var historyResponse = await historyRequest.ExecuteAsync(cancellationToken);
+                    var historyResponse = await historyRequest.ExecuteAsync(cancellationToken);
 
-                nextPageToken = historyResponse.NextPageToken;
+                    nextPageToken = historyResponse.NextPageToken;
 
-                if (historyResponse.History == null)
+                    if (historyResponse.History == null)
-                    continue;
+                        continue;
 
-                deltaChanges.Add(historyResponse);
+                    deltaChanges.Add(historyResponse);
+                }
+            }
+            catch (GoogleApiException ex) when (ex.HttpStatusCode == System.Net.HttpStatusCode.NotFound)
+            {
+                // History ID is too old or expired, need to do a full sync.
+                // Theoratically we need to delete the local cache and start from scratch.
+
+                _logger.Warning("History ID {StartHistoryId} is expired for {Name}. Will remove user's mail cache and do full sync.", startHistoryId, Account.Name);
+
+                await _gmailChangeProcessor.DeleteUserMailCacheAsync(Account.Id).ConfigureAwait(false);
+
+                Account.SynchronizationDeltaIdentifier = string.Empty;
+
+                await _gmailChangeProcessor.UpdateAccountAsync(Account).ConfigureAwait(false);
+
+                goto retry;
             }
         }
 
@@ -612,7 +637,7 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
         // async callback is finished. Therefore we need to wrap all local database processings into task list and wait all of them to finish
         // Batch execution finishes after response parsing is done.
 
-        var batchProcessCallbacks = new List<Task>();
+        var batchProcessCallbacks = new List<Task<Message>>();
 
         foreach (var batchBundle in batchedDownloadRequests)
         {
@@ -626,8 +651,9 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
                 batchRequest.Queue<Message>(request, (content, error, index, message) =>
                 {
                     var downloadingMessageId = messageIds.ElementAt(index);
+                    var downloadTask = HandleSingleItemDownloadedCallbackAsync(content, error, downloadingMessageId, cancellationToken);
 
-                    batchProcessCallbacks.Add(HandleSingleItemDownloadedCallbackAsync(content, error, downloadingMessageId, cancellationToken));
+                    batchProcessCallbacks.Add(downloadTask);
 
                     downloadedItemCount++;
 
@@ -650,6 +676,15 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
 
         // Wait for all processing to finish.
         await Task.WhenAll(batchProcessCallbacks).ConfigureAwait(false);
+
+        // Try to update max history id.
+        var maxHistoryId = batchProcessCallbacks.Select(a => a.Result).Where(a => a.HistoryId != null).Max(a => a.HistoryId.Value);
+
+        if (maxHistoryId != 0)
+        {
+            Account.SynchronizationDeltaIdentifier = await _gmailChangeProcessor.UpdateAccountDeltaSynchronizationIdentifierAsync(Account.Id, maxHistoryId.ToString()).ConfigureAwait(false);
+        }
+
     }
 
     /// <summary>
@@ -1100,7 +1135,7 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
     /// <param name="error"></param>
     /// <param name="httpResponseMessage"></param>
     /// <param name="cancellationToken"></param>
-    private async Task HandleSingleItemDownloadedCallbackAsync(Message message,
+    private async Task<Message> HandleSingleItemDownloadedCallbackAsync(Message message,
                                                                RequestError error,
                                                                string downloadingMessageId,
                                                                CancellationToken cancellationToken = default)
@@ -1126,7 +1161,7 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
         {
             _logger.Warning("Skipped GMail message download for {DownloadingMessageId}", downloadingMessageId);
 
-            return;
+            return null;
         }
 
         // Gmail has LabelId property for each message.
@@ -1135,20 +1170,27 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
 
         // If CreateNewMailPackagesAsync returns null it means local draft mapping is done.
         // We don't need to insert anything else.
-        if (mailPackage == null)
+        if (mailPackage == null) return message;
-            return;
 
         foreach (var package in mailPackage)
         {
             await _gmailChangeProcessor.CreateMailAsync(Account.Id, package).ConfigureAwait(false);
         }
 
+        return message;
+    }
+
+    private async Task UpdateAccountSyncIdentifierFromMessageAsync(Message message)
+    {
         // Try updating the history change identifier if any.
         if (message.HistoryId == null) return;
 
-        // Delta changes also has history id but the maximum id is preserved in the account service.
+        if (ulong.TryParse(Account.SynchronizationDeltaIdentifier, out ulong currentIdentifier) &&
-        // TODO: This is not good. Centralize the identifier fetch and prevent direct access here.
+            ulong.TryParse(message.HistoryId.Value.ToString(), out ulong messageIdentifier) &&
-        Account.SynchronizationDeltaIdentifier = await _gmailChangeProcessor.UpdateAccountDeltaSynchronizationIdentifierAsync(Account.Id, message.HistoryId.ToString());
+            messageIdentifier > currentIdentifier)
+        {
+            Account.SynchronizationDeltaIdentifier = await _gmailChangeProcessor.UpdateAccountDeltaSynchronizationIdentifierAsync(Account.Id, message.HistoryId.ToString());
+        }
     }
 
     private async Task ProcessSingleNativeRequestResponseAsync(IRequestBundle<IClientServiceRequest> bundle,
@@ -1164,7 +1206,8 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
 
             if (gmailMessage == null) return;
 
-            await HandleSingleItemDownloadedCallbackAsync(gmailMessage, error, "unknown", cancellationToken);
+            await HandleSingleItemDownloadedCallbackAsync(gmailMessage, error, string.Empty, cancellationToken);
+            await UpdateAccountSyncIdentifierFromMessageAsync(gmailMessage).ConfigureAwait(false);
         }
         else if (bundle is HttpRequestBundle<IClientServiceRequest, Label> folderBundle)
         {
@@ -1210,9 +1253,12 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
     /// </summary>
     private async Task MapDraftIdsAsync(CancellationToken cancellationToken = default)
     {
-        // TODO: This call is not necessary if we don't have any local drafts.
+        // Check if account has any draft locally.
-        // Remote drafts will be downloaded in missing message batches anyways.
+        // There is no point to send this query if there are no local drafts.
-        // Fix it by checking whether we need to do this or not.
+
+        bool hasLocalDrafts = await _gmailChangeProcessor.HasAccountAnyDraftAsync(Account.Id).ConfigureAwait(false);
+
+        if (!hasLocalDrafts) return;
 
         var drafts = await _gmailService.Users.Drafts.List("me").ExecuteAsync(cancellationToken);
 

Wino.Mail.ViewModels/MailListPageViewModel.cs

@@ -40,7 +40,8 @@ public partial class MailListPageViewModel : MailBaseViewModel,
     IRecipient<MailItemSelectionRemovedEvent>,
     IRecipient<AccountSynchronizationCompleted>,
     IRecipient<NewMailSynchronizationRequested>,
-    IRecipient<AccountSynchronizerStateChanged>
+    IRecipient<AccountSynchronizerStateChanged>,
+    IRecipient<AccountCacheResetMessage>
 {
     private bool isChangingFolder = false;
 
@@ -1117,4 +1118,22 @@ public partial class MailListPageViewModel : MailBaseViewModel,
 
         await ExecuteUIThread(() => { IsAccountSynchronizerInSynchronization = isAnyAccountSynchronizing; });
     }
+
+    public void Receive(AccountCacheResetMessage message)
+    {
+        if (message.Reason == AccountCacheResetReason.ExpiredCache &&
+            ActiveFolder.HandlingFolders.Any(a => a.MailAccountId == message.AccountId))
+        {
+            var handlingFolder = ActiveFolder.HandlingFolders.FirstOrDefault(a => a.MailAccountId == message.AccountId);
+
+            if (handlingFolder == null) return;
+
+            _ = ExecuteUIThread(() =>
+            {
+                MailCollection.Clear();
+
+                _mailDialogService.InfoBarMessage(Translator.AccountCacheReset_Title, Translator.AccountCacheReset_Message, InfoBarMessageType.Warning);
+            });
+        }
+    }
 }

Wino.Messages/CommunicationMessagesContext.cs

@@ -1,4 +1,5 @@
 using System.Text.Json.Serialization;
+using Wino.Messaging.Server;
 using Wino.Messaging.UI;
 
 namespace Wino.Messaging;
@@ -23,4 +24,6 @@ namespace Wino.Messaging;
 [JsonSerializable(typeof(AccountSynchronizationProgressUpdatedMessage))]
 [JsonSerializable(typeof(AccountFolderConfigurationUpdated))]
 [JsonSerializable(typeof(CopyAuthURLRequested))]
+[JsonSerializable(typeof(NewMailSynchronizationRequested))]
+[JsonSerializable(typeof(AccountCacheResetMessage))]
 public partial class CommunicationMessagesContext : JsonSerializerContext;

Wino.Messages/UI/AccountCacheResetMessage.cs

@@ -0,0 +1,7 @@
+using System;
+using Wino.Core.Domain.Enums;
+
+namespace Wino.Messaging.UI;
+
+// Raised when the account's mail cache is reset.
+public record AccountCacheResetMessage(Guid AccountId, AccountCacheResetReason Reason) : UIMessageBase<AccountCacheResetMessage>;

Wino.Server/ServerContext.cs

@@ -44,7 +44,8 @@ public class ServerContext :
     IRecipient<AccountFolderConfigurationUpdated>,
     IRecipient<CopyAuthURLRequested>,
     IRecipient<NewMailSynchronizationRequested>,
-    IRecipient<OnlineSearchRequested>
+    IRecipient<OnlineSearchRequested>,
+    IRecipient<AccountCacheResetMessage>
 {
     private readonly System.Timers.Timer _timer;
     private static object connectionLock = new object();
@@ -147,6 +148,8 @@ public class ServerContext :
 
     public async void Receive(OnlineSearchRequested message) => await SendMessageAsync(MessageType.UIMessage, message);
 
+    public async void Receive(AccountCacheResetMessage message) => await SendMessageAsync(MessageType.UIMessage, message);
+
     #endregion
 
     private string GetAppPackagFamilyName()

Wino.Services/AccountService.cs

@@ -22,15 +22,18 @@ public class AccountService : BaseDatabaseService, IAccountService
     public IAuthenticator ExternalAuthenticationAuthenticator { get; set; }
 
     private readonly ISignatureService _signatureService;
+    private readonly IMimeFileService _mimeFileService;
     private readonly IPreferencesService _preferencesService;
 
     private readonly ILogger _logger = Log.ForContext<AccountService>();
 
     public AccountService(IDatabaseService databaseService,
                           ISignatureService signatureService,
+                          IMimeFileService mimeFileService,
                           IPreferencesService preferencesService) : base(databaseService)
     {
         _signatureService = signatureService;
+        _mimeFileService = mimeFileService;
         _preferencesService = preferencesService;
     }
 
@@ -262,12 +265,26 @@ public class AccountService : BaseDatabaseService, IAccountService
     private Task<MergedInbox> GetMergedInboxInformationAsync(Guid mergedInboxId)
         => Connection.Table<MergedInbox>().FirstOrDefaultAsync(a => a.Id == mergedInboxId);
 
+    public async Task DeleteAccountMailCacheAsync(Guid accountId, AccountCacheResetReason accountCacheResetReason)
+    {
+        var deleteQuery = new Query("MailCopy")
+                .WhereIn("Id", q => q
+                .From("MailCopy")
+                .Select("Id")
+                .WhereIn("FolderId", q2 => q2
+                    .From("MailItemFolder")
+                    .Select("Id")
+                    .Where("MailAccountId", accountId)
+                )).AsDelete();
+
+        await Connection.ExecuteAsync(deleteQuery.GetRawQuery());
+
+        WeakReferenceMessenger.Default.Send(new AccountCacheResetMessage(accountId, accountCacheResetReason));
+    }
+
     public async Task DeleteAccountAsync(MailAccount account)
     {
-        // TODO: Delete mime messages and attachments.
+        await DeleteAccountMailCacheAsync(account.Id, AccountCacheResetReason.AccountRemoval);
-        // TODO: Delete token cache by underlying provider.
-
-        await Connection.ExecuteAsync("DELETE FROM MailCopy WHERE Id IN(SELECT Id FROM MailCopy WHERE FolderId IN (SELECT Id from MailItemFolder WHERE MailAccountId == ?))", account.Id);
 
         await Connection.Table<MailItemFolder>().DeleteAsync(a => a.MailAccountId == account.Id);
         await Connection.Table<AccountSignature>().DeleteAsync(a => a.MailAccountId == account.Id);
@@ -302,6 +319,8 @@ public class AccountService : BaseDatabaseService, IAccountService
 
         await Connection.DeleteAsync(account);
 
+        await _mimeFileService.DeleteUserMimeCacheAsync(account.Id).ConfigureAwait(false);
+
         // Clear out or set up a new startup entity id.
         // Next account after the deleted one will be the startup account.
 
@@ -319,8 +338,6 @@ public class AccountService : BaseDatabaseService, IAccountService
             }
         }
 
-
-
         ReportUIChange(new AccountRemovedMessage(account));
     }
 

Wino.Services/MailService.cs

@@ -119,6 +119,18 @@ public class MailService : BaseDatabaseService, IMailService
         return mails;
     }
 
+    public async Task<bool> HasAccountAnyDraftAsync(Guid accountId)
+    {
+        // Get the draft folder.
+        var draftFolder = await _folderService.GetSpecialFolderByAccountIdAsync(accountId, SpecialFolderType.Draft);
+
+        if (draftFolder == null) return false;
+
+        var draftCount = await Connection.Table<MailCopy>().Where(a => a.FolderId == draftFolder.Id).CountAsync();
+
+        return draftCount > 0;
+    }
+
     public async Task<List<MailCopy>> GetUnreadMailsByFolderIdAsync(Guid folderId)
     {
         var unreadMails = await Connection.QueryAsync<MailCopy>("SELECT * FROM MailCopy WHERE FolderId = ? AND IsRead = 0", folderId);
@@ -143,7 +155,7 @@ public class MailService : BaseDatabaseService, IMailService
         //}
 
         // SQLite PCL doesn't support joins.
-        // We make the query using SqlKatka and execute it directly on SQLite-PCL.
+        // We make the query using SqlKata and execute it directly on SQLite-PCL.
 
         var query = new Query("MailCopy")
                     .Join("MailItemFolder", "MailCopy.FolderId", "MailItemFolder.Id")

Wino.Services/MimeFileService.cs

@@ -176,4 +176,22 @@ public class MimeFileService : IMimeFileService
 
         return renderingModel;
     }
+
+    public async Task DeleteUserMimeCacheAsync(Guid accountId)
+    {
+        var mimeFolderPath = await _nativeAppService.GetMimeMessageStoragePath().ConfigureAwait(false);
+        var mimeDirectory = Path.Combine(mimeFolderPath, accountId.ToString());
+
+        try
+        {
+            if (Directory.Exists(mimeDirectory))
+            {
+                Directory.Delete(mimeDirectory, true);
+            }
+        }
+        catch (Exception ex)
+        {
+            Log.Error(ex, "Failed to remove user's mime cache folder.");
+        }
+    }
 }